diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1bc32ba8f..88c57e1ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,7 @@ on: push: branches: [main] pull_request: + workflow_dispatch: env: CARGO_TERM_COLOR: always @@ -20,25 +21,106 @@ jobs: components: clippy - uses: Swatinem/rust-cache@v2 - run: cargo clippy --workspace --all-targets -- -D warnings + - run: cargo clippy -p codegraph-graph --features postgres,mysql,redis --tests -- -D warnings test: name: test (${{ matrix.os }}) runs-on: ${{ matrix.os }} + services: + redis: + image: redis:7-alpine + ports: ["6379:6379"] + options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 3 + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: codegraph + ports: ["5432:5432"] + options: --health-cmd "pg_isready -U postgres" --health-interval 10s --health-timeout 5s --health-retries 3 + mysql: + image: mysql:8 + env: + MYSQL_ROOT_PASSWORD: postgres + MYSQL_DATABASE: codegraph + ports: ["3306:3306"] + options: --health-cmd "mysqladmin ping -h 127.0.0.1 -u root -ppostgres" --health-interval 10s --health-timeout 5s --health-retries 3 strategy: fail-fast: false matrix: os: [ubuntu-latest] steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable + with: + components: llvm-tools-preview - uses: Swatinem/rust-cache@v2 - - run: cargo test --workspace --no-fail-fast + - name: Install DB clients + run: sudo apt-get update && sudo apt-get install -y postgresql-client mysql-client + - name: Apply schema (postgres) + run: | + psql "postgres://postgres:postgres@127.0.0.1:5432/codegraph" -f sql/postgres/001-initial-schema.sql + psql "postgres://postgres:postgres@127.0.0.1:5432/codegraph" -f sql/postgres/002-add-repos-registry.sql + - name: Apply schema (mysql) + run: | + mysql -h 127.0.0.1 -P 3306 -u root -ppostgres codegraph < sql/mysql/001-initial-schema.sql + mysql -h 127.0.0.1 -P 3306 -u root -ppostgres codegraph < sql/mysql/002-add-repos-registry.sql + - name: Install grcov + uses: taiki-e/install-action@grcov + - name: Run tests with coverage instrumentation + env: + RUSTFLAGS: "-Cinstrument-coverage" + LLVM_PROFILE_FILE: "codegraph-%p-%m.profraw" + run: cargo test --workspace --no-fail-fast + - name: Storage integration tests (postgres) + env: + RUSTFLAGS: "-Cinstrument-coverage" + LLVM_PROFILE_FILE: "codegraph-%p-%m.profraw" + TEST_RDBMS_DSN: "postgres://postgres:postgres@127.0.0.1:5432/codegraph" + TEST_RDBMS_REPO_ID: "1" + run: cargo test -p codegraph-graph --features postgres --test rdbms -- --ignored --nocapture --test-threads=1 + - name: Storage integration tests (mysql) + env: + RUSTFLAGS: "-Cinstrument-coverage" + LLVM_PROFILE_FILE: "codegraph-%p-%m.profraw" + TEST_RDBMS_DSN: "mysql://root:postgres@127.0.0.1:3306/codegraph" + TEST_RDBMS_REPO_ID: "1" + run: cargo test -p codegraph-graph --features mysql --test rdbms -- --ignored --nocapture --test-threads=1 + - name: Storage integration tests (redis) + env: + RUSTFLAGS: "-Cinstrument-coverage" + LLVM_PROFILE_FILE: "codegraph-%p-%m.profraw" + TEST_REDIS_DSN: "redis://127.0.0.1:6379" + run: cargo test -p codegraph-graph --features redis --test redis -- --ignored --nocapture + - name: Generate coverage report (lcov) + run: | + mkdir -p ./target/coverage + grcov . \ + --binary-path ./target/debug/ \ + --source-dir . \ + --output-type lcov \ + --branch \ + --ignore-not-existing \ + --ignore "/*" \ + --ignore "*/tests/*" \ + --ignore "*/benches/*" \ + --output-path ./target/coverage/lcov.info + + - name: Upload to Codecov + uses: codecov/codecov-action@v5 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + files: ./target/coverage/lcov.info + verbose: true + fail_ci_if_error: false slim: name: slim build (no visualize) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable with: components: clippy diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml deleted file mode 100644 index f080e9040..000000000 --- a/.github/workflows/integration.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Integration (Postgres / MySQL / Redis) - -permissions: - contents: read - -# Chạy test tích hợp trên backend thật (Postgres/MySQL/Redis) qua service -# container của GitHub Actions. Schema được apply thủ công (`sql//*`) -# trước khi chạy test — khớp thiết kế "migration thủ công" của repo. -# -# Test trong `tests/rdbms.rs` / `tests/redis.rs` bị `#[ignore]` và chỉ chạy khi -# có DSN tương ứng → không ảnh hưởng `cargo test` thường (CI chính ở ci.yml). -on: - push: - branches: [main] - pull_request: - workflow_dispatch: - -env: - CARGO_TERM_COLOR: always - -jobs: - clippy-gated: - name: clippy (rdbms + redis test targets) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy - - uses: Swatinem/rust-cache@v2 - # Đảm bảo các file test gated (tests/rdbms.rs, tests/redis.rs) vẫn - # clippy-sạch dù CI chính chỉ build với default features. - - run: cargo clippy -p codegraph-graph --features postgres,mysql,redis --tests -- -D warnings - - postgres: - name: postgres - runs-on: ubuntu-latest - services: - postgres: - image: postgres:16 - env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: codegraph - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - env: - TEST_RDBMS_DSN: "postgres://postgres:postgres@127.0.0.1:5432/codegraph" - TEST_RDBMS_REPO_ID: "1" - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Install postgresql-client - run: sudo apt-get update && sudo apt-get install -y postgresql-client - - name: Apply schema (manual migration) - run: | - psql "$TEST_RDBMS_DSN" -f sql/postgres/001-initial-schema.sql - psql "$TEST_RDBMS_DSN" -f sql/postgres/002-add-repos-registry.sql - - name: Run integration tests - run: cargo test -p codegraph-graph --features postgres --test rdbms -- --ignored --nocapture - - mysql: - name: mysql - runs-on: ubuntu-latest - services: - mysql: - image: mysql:8 - env: - MYSQL_ROOT_PASSWORD: postgres - MYSQL_DATABASE: codegraph - ports: - - 3306:3306 - options: >- - --health-cmd "mysqladmin ping -h 127.0.0.1 -u root -ppostgres" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - env: - TEST_RDBMS_DSN: "mysql://root:postgres@127.0.0.1:3306/codegraph" - TEST_RDBMS_REPO_ID: "1" - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - - name: Install mysql-client - run: sudo apt-get update && sudo apt-get install -y mysql-client - - name: Apply schema (manual migration) - run: | - mysql -h 127.0.0.1 -P 3306 -u root -ppostgres codegraph < sql/mysql/001-initial-schema.sql - mysql -h 127.0.0.1 -P 3306 -u root -ppostgres codegraph < sql/mysql/002-add-repos-registry.sql - - name: Run integration tests - run: cargo test -p codegraph-graph --features mysql --test rdbms -- --ignored --nocapture - - redis: - name: redis - runs-on: ubuntu-latest - services: - redis: - image: redis:7 - ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - env: - TEST_REDIS_DSN: "redis://127.0.0.1:6379" - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - uses: Swatinem/rust-cache@v2 - # Unit test nội bộ (storage/redis.rs) chạy trên DB 15. - - name: Run storage unit tests - run: cargo test -p codegraph-graph --features redis - # Integration test (GraphIndex roundtrip) chạy trên DB 0 (DSN mặc định). - - name: Run integration tests - run: cargo test -p codegraph-graph --features redis --test redis -- --ignored --nocapture diff --git a/Cargo.lock b/Cargo.lock index 63e185ca5..a4364f6ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -12,6 +18,7 @@ dependencies = [ "const-random", "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -25,6 +32,24 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" +dependencies = [ + "as-slice", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -123,6 +148,32 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "as-slice" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "async-lock" version = "3.4.2" @@ -166,6 +217,49 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "av-scenechange" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" +dependencies = [ + "aligned", + "anyhow", + "arg_enum_proc_macro", + "arrayvec", + "log", + "num-rational", + "num-traits", + "pastey 0.1.1", + "rayon", + "thiserror 2.0.18", + "v_frame", + "y4m", +] + +[[package]] +name = "av1-grain" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom 8.0.0", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7178fe5f7d460b13895ebb9dcb28a3a6216d2df2574a0806cb51b555d297f38" +dependencies = [ + "arrayvec", +] + [[package]] name = "axum" version = "0.8.9" @@ -218,6 +312,12 @@ dependencies = [ "tracing", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -245,6 +345,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + [[package]] name = "bitflags" version = "1.3.2" @@ -260,6 +366,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bitstream-io" +version = "4.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" +dependencies = [ + "no_std_io2", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -279,6 +394,12 @@ dependencies = [ "serde", ] +[[package]] +name = "built" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" + [[package]] name = "bumpalo" version = "3.20.3" @@ -288,12 +409,24 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.11.1" @@ -315,6 +448,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.62" @@ -547,8 +689,10 @@ dependencies = [ "codegraph-extract", "criterion", "dashmap", + "fastembed", "libsqlite3-sys", "lmdb-rkv", + "ort", "parking_lot", "redis", "rusqlite", @@ -569,7 +713,7 @@ version = "1.2.0" dependencies = [ "anyhow", "camino", - "dirs", + "dirs 5.0.1", "jsonc-parser", "serde", "serde_json", @@ -666,7 +810,7 @@ dependencies = [ "codspeed", "criterion-plot", "is-terminal", - "itertools", + "itertools 0.10.5", "num-traits", "once_cell", "oorandom", @@ -680,6 +824,12 @@ dependencies = [ "walkdir", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.5" @@ -709,6 +859,21 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "console" version = "0.16.4" @@ -747,6 +912,55 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "cookie" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -918,6 +1132,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "criterion" version = "0.5.1" @@ -931,7 +1154,7 @@ dependencies = [ "criterion-plot", "futures", "is-terminal", - "itertools", + "itertools 0.10.5", "num-traits", "once_cell", "oorandom", @@ -953,7 +1176,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools", + "itertools 0.10.5", ] [[package]] @@ -1006,14 +1229,38 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88490bf1b990d87eaaa7ac8aa887f629a08e7359765b4911faf63c3763347d23" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.24.0", + "darling_macro 0.24.0", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", ] [[package]] @@ -1029,17 +1276,37 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.117", +] + [[package]] name = "darling_macro" version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f5792fa0d41cd2325ce0ffa64f0a340eaebd4971a3a0c5e1ffd2cc488a355e" dependencies = [ - "darling_core", + "darling_core 0.24.0", "quote", "syn 3.0.3", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + [[package]] name = "dashmap" version = "6.2.1" @@ -1061,10 +1328,57 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid", - "pem-rfc7468", + "pem-rfc7468 0.7.0", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +dependencies = [ + "pem-rfc7468 1.0.0", "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.117", +] + [[package]] name = "digest" version = "0.10.7" @@ -1083,7 +1397,16 @@ version = "5.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" dependencies = [ - "dirs-sys", + "dirs-sys 0.4.1", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys 0.5.0", ] [[package]] @@ -1094,10 +1417,22 @@ checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" dependencies = [ "libc", "option-ext", - "redox_users", + "redox_users 0.4.6", "windows-sys 0.48.0", ] +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users 0.5.2", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -1109,6 +1444,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -1136,12 +1480,41 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "env_home" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7f84e12ccf0a7ddc17a6c41c93326024c42920d7ee630d04950e6926645c0fe" +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1158,6 +1531,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + [[package]] name = "etcetera" version = "0.8.0" @@ -1190,23 +1569,72 @@ dependencies = [ ] [[package]] -name = "fallible-iterator" -version = "0.3.0" +name = "exr" +version = "1.74.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - +checksum = "711fe42c9964295e01ee3fba3f9fe0e1d24b98886950d68efe81b1c76e21adf3" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "num-complex", + "pulp", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fallible-streaming-iterator" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fastembed" +version = "5.17.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4539f4a2c4472269adc227587b935c0a973e6b5fc4a03e14bbe62608e06c2298" +dependencies = [ + "anyhow", + "hf-hub", + "image", + "ndarray", + "ort", + "safetensors", + "serde", + "serde_json", + "tokenizers", +] + [[package]] name = "fastrand" version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + [[package]] name = "file-id" version = "0.2.3" @@ -1232,6 +1660,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.11.1" @@ -1243,12 +1681,39 @@ dependencies = [ "spin 0.9.9", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1413,6 +1878,16 @@ dependencies = [ "wasip3", ] +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.31.1" @@ -1443,6 +1918,25 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "h2" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "half" version = "2.7.1" @@ -1471,7 +1965,20 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", + "serde", + "serde_core", ] [[package]] @@ -1516,6 +2023,27 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hf-hub" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef3982638978efa195ff11b305f51f1f22f4f0a6cabee7af79b383ebee6a213" +dependencies = [ + "dirs 6.0.0", + "http", + "indicatif", + "libc", + "log", + "native-tls", + "rand 0.9.5", + "reqwest", + "serde", + "serde_json", + "thiserror 2.0.18", + "ureq", + "windows-sys 0.61.2", +] + [[package]] name = "hkdf" version = "0.12.4" @@ -1534,6 +2062,12 @@ dependencies = [ "digest", ] +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + [[package]] name = "home" version = "0.5.12" @@ -1598,6 +2132,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -1606,6 +2141,38 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", ] [[package]] @@ -1614,13 +2181,23 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64 0.22.1", "bytes", + "futures-channel", + "futures-util", "http", "http-body", "hyper", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", + "socket2", + "system-configuration", "tokio", "tower-service", + "tracing", + "windows-registry", ] [[package]] @@ -1778,6 +2355,46 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imgref" +version = "1.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89194689a993ab15268672e99e7b0e19da2da3268ac682e8f02d29d4d1434cd7" + [[package]] name = "indexmap" version = "2.14.0" @@ -1832,6 +2449,23 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + [[package]] name = "is-terminal" version = "0.4.17" @@ -1858,6 +2492,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1926,12 +2569,28 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libm" version = "0.2.16" @@ -1973,6 +2632,12 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "lmdb-rkv" version = "0.14.0" @@ -2011,6 +2676,21 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + [[package]] name = "mach2" version = "0.4.3" @@ -2020,6 +2700,22 @@ dependencies = [ "libc", ] +[[package]] +name = "macro_rules_attribute" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c" +dependencies = [ + "macro_rules_attribute-proc_macro", + "pastey 0.2.3", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c" + [[package]] name = "matchers" version = "0.2.0" @@ -2035,6 +2731,26 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + [[package]] name = "md-5" version = "0.10.6" @@ -2057,6 +2773,22 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.0" @@ -2070,47 +2802,151 @@ dependencies = [ ] [[package]] -name = "nix" -version = "0.31.3" +name = "monostate" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" dependencies = [ - "bitflags 2.11.1", - "cfg-if", - "cfg_aliases", - "libc", + "monostate-impl", + "serde", + "serde_core", ] [[package]] -name = "no-std-compat" -version = "0.4.1" +name = "monostate-impl" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" dependencies = [ - "spin 0.5.2", + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] -name = "notify" -version = "7.0.0" +name = "moxcms" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" dependencies = [ - "bitflags 2.11.1", - "filetime", - "fsevent-sys", - "inotify", - "kqueue", - "libc", - "log", - "mio", - "notify-types", - "walkdir", - "windows-sys 0.52.0", + "num-traits", + "pxfm", ] [[package]] -name = "notify-debouncer-full" +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +dependencies = [ + "spin 0.5.2", +] + +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + +[[package]] +name = "notify" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009" +dependencies = [ + "bitflags 2.11.1", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.52.0", +] + +[[package]] +name = "notify-debouncer-full" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dcf855483228259b2353f89e99df35fc639b2b2510d1166e4858e3f67ec1afb" @@ -2166,6 +3002,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "bytemuck", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -2185,6 +3048,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -2210,18 +3084,107 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags 2.11.1", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "oorandom" version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ort" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b" +dependencies = [ + "ndarray", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "parking" version = "2.2.1" @@ -2251,6 +3214,18 @@ dependencies = [ "windows-link", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + [[package]] name = "pastey" version = "0.2.3" @@ -2266,6 +3241,15 @@ dependencies = [ "base64ct", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -2284,7 +3268,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", + "der 0.7.10", "pkcs8", "spki", ] @@ -2295,7 +3279,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", + "der 0.7.10", "spki", ] @@ -2339,12 +3323,34 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags 2.11.1", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "portable-atomic" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -2354,6 +3360,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2382,6 +3394,69 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4488a4a36b9a4ba6b9334a32a39971f77c1436ec82c38707bce707699cc3bbcb" +dependencies = [ + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pulp" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "046aa45b989642ec2e4717c8e72d677b13edd831a4d3b6cf37d9a3e54912496a" +dependencies = [ + "bytemuck", + "cfg-if", + "libm", + "num-complex", + "paste", + "pulp-wasm-simd-flag", + "raw-cpuid", + "reborrow", + "version_check", +] + +[[package]] +name = "pulp-wasm-simd-flag" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d8f70e07b9c3962945a74e59ca1c511bba65b6419468acc217c457d93f3c740" + +[[package]] +name = "pxfm" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.45" @@ -2410,10 +3485,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", - "rand_chacha", + "rand_chacha 0.3.1", "rand_core 0.6.4", ] +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + [[package]] name = "rand" version = "0.10.2" @@ -2435,6 +3520,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -2444,12 +3539,86 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "rand_core" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rav1e" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" +dependencies = [ + "aligned-vec", + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av-scenechange", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.14.0", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "paste", + "profiling", + "rand 0.9.5", + "rand_chacha 0.9.0", + "simd_helpers", + "thiserror 2.0.18", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.11.1", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.12.0" @@ -2460,6 +3629,17 @@ dependencies = [ "rayon-core", ] +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools 0.14.0", + "rayon", +] + [[package]] name = "rayon-core" version = "1.13.0" @@ -2470,6 +3650,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "redis" version = "1.5.0" @@ -2524,6 +3710,17 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", +] + [[package]] name = "ref-cast" version = "1.0.26" @@ -2599,6 +3796,55 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" + [[package]] name = "rhai" version = "1.25.1" @@ -2628,6 +3874,20 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rmcp" version = "3.1.2" @@ -2642,7 +3902,7 @@ dependencies = [ "http", "http-body", "http-body-util", - "pastey", + "pastey 0.2.3", "pin-project-lite", "rand 0.10.2", "rmcp-macros", @@ -2665,7 +3925,7 @@ version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6898e24cd16342b59bfa8a53c2c04b9cf62fc8a2cfea57b9c038b09984bfc521" dependencies = [ - "darling", + "darling 0.24.0", "proc-macro2", "quote", "serde_json", @@ -2725,6 +3985,41 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -2737,6 +4032,19 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safetensors" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b079b829cb27a1c3c374341345ed2e8b2c0c839034522cee576c140bd7f846" +dependencies = [ + "hashbrown 0.16.1", + "libc", + "serde", + "serde_json", + "tempfile", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2746,6 +4054,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "1.2.2" @@ -2778,6 +4095,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.1", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.28" @@ -2924,6 +4264,21 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "slab" version = "0.4.12" @@ -2960,6 +4315,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "spin" version = "0.5.2" @@ -2982,7 +4348,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", ] [[package]] @@ -3264,6 +4642,9 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -3276,6 +4657,27 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.11.1", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "target-lexicon" version = "0.13.5" @@ -3360,6 +4762,50 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -3404,6 +4850,39 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "itertools 0.14.0", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.5", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 2.0.18", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.52.3" @@ -3430,6 +4909,26 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.19" @@ -3511,6 +5010,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.11.1", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -3745,6 +5262,12 @@ dependencies = [ "tree-sitter-language", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.20.0" @@ -3772,12 +5295,27 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-width" version = "0.2.2" @@ -3790,12 +5328,60 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "unit-prefix" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" +dependencies = [ + "base64 0.23.1", + "cookie_store", + "der 0.8.1", + "flate2", + "log", + "native-tls", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-root-certs", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" +dependencies = [ + "base64 0.23.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -3808,6 +5394,12 @@ dependencies = [ "serde", ] +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3831,6 +5423,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "v_frame" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -3859,6 +5462,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -3902,6 +5514,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.126" @@ -3956,6 +5578,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -4000,6 +5635,30 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "which" version = "7.0.3" @@ -4022,6 +5681,22 @@ dependencies = [ "wasite", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -4031,6 +5706,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" @@ -4072,6 +5753,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -4442,6 +6134,12 @@ version = "0.8.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" +[[package]] +name = "y4m" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" + [[package]] name = "yoke" version = "0.8.3" @@ -4578,3 +6276,27 @@ dependencies = [ "cc", "pkg-config", ] + +[[package]] +name = "zune-core" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/README.md b/README.md index 50716daf3..2d9b36c58 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,15 @@ # CodeGraph -[![CI](https://github.com/cleboost/codegraph/actions/workflows/ci.yml/badge.svg)](https://github.com/cleboost/codegraph/actions/workflows/ci.yml) +[![CI](https://github.com/hungpham10/codegraph-rs/actions/workflows/ci.yml/badge.svg)](https://github.com/Cleboost/codegraph-rs/actions/workflows/ci.yml) +[![CodSpeed Badge](https://img.shields.io/endpoint?url=https://app.codspeed.io//badge.json)](https://app.codspeed.io//hungpham10/codegraph-rs?utm_source=badge) +[![codecov](https://codecov.io/gh/hungpham10/codegraph-rs/graph/badge.svg?token=PUSMFF0CM8)](https://codecov.io/gh/hungpham10/codegraph-rs) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) > Local-first code intelligence for AI agents. Built in Rust. Single static -> binary, ~5 MB. Tree-sitter **semantic graph** (semgraph) in SQLite, served over MCP. +> binary. Tree-sitter **semantic graph** (semgraph) in SQLite (or LMDB / +> Postgres / MySQL / Redis), served over MCP. -CodeGraph parses your codebase with tree-sitter, builds a **semantic graph** where every symbol gets a global ID and every function has a **call chain** (markers + callee IDs), stores everything in a single `.codegraph/db.sqlite`, and exposes the graph to AI agents — Claude Code, Cursor, Codex CLI, opencode, Hermes — over the Model Context Protocol (MCP). +CodeGraph parses your codebase with tree-sitter, builds a **semantic graph** where every symbol gets a global ID and every function has a **call chain** (markers + callee IDs), stores everything under `.codegraph/` (SQLite by default), and exposes the graph to AI agents — Claude Code, Cursor, Codex CLI, opencode, Hermes — over the Model Context Protocol (MCP). Agents that consult the semantic graph instead of grepping the filesystem make **fewer tool calls**, **explore faster**, and **stay within context**. @@ -14,12 +17,13 @@ Agents that consult the semantic graph instead of grepping the filesystem make * - **Semgraph model**: Symbols have global IDs (≥100); call chains mix markers (`LOOP`, `IF_TRUE`, `RETURN`, …) and callee IDs. Edges derived from chains. No more `NodeKind`/`EdgeKind` — wire breaking to `SymbolKind`. - **One binary.** Rust + statically-linked SQLite + native tree-sitter grammars. No Node runtime, no `.wasm`, no `node_modules`. -- **Small.** ~5 MB stripped (vs ~140 MB for the previous TypeScript build). +- **Compact.** ~58 MB release build with every storage backend (SQLite, LMDB, Redis, Postgres/MySQL) and the embedding runtime bundled in one file (vs ~140 MB for the previous TypeScript build). - **Fast.** Full re-index a 139-file project in ~190 ms (release, parallel rayon). -- **Local.** Index lives in `.codegraph/db.sqlite` next to your code. Nothing leaves the machine. +- **Local.** Index lives in `.codegraph/` next to your code (SQLite by default; LMDB / Postgres / MySQL / Redis optional). Nothing leaves the machine. - **Full re-index always.** No incremental sync — watcher debounces and re-indexes completely (simpler, no stale state). - **Multi-agent.** One binary serves any MCP client (Claude Code, Cursor, Codex, opencode, Hermes, Antigravity) over stdio or Streamable HTTP (`--http`) — the agent binds the workspace with `codegraph_init` and drives everything through tools. -- **30 MCP tools** including `codegraph_flow` (call chain), `codegraph_search_flow` (pattern search), `codegraph_references` (library call consumers), `codegraph_diff` (MR impact draft), and a behavior sandbox (`codegraph_sandbox`). +- **Optional semantic search.** Enable `[embedding] backend = "fastembed"` in config to get vector KNN / hybrid symbol search — BGE-small embeddings running locally, backend already bundled in the release binary. +- **27 MCP tools** including `codegraph_flow` (call chain), `codegraph_search_flow` (pattern search), `codegraph_references` (library call consumers), `codegraph_diff` (MR impact draft), and a behavior sandbox (`codegraph_sandbox`). ## Install @@ -29,7 +33,7 @@ Agents that consult the semantic graph instead of grepping the filesystem make * **Linux / macOS** ```sh -curl -fsSL https://raw.githubusercontent.com/Cleboost/codegraph-rs/main/scripts/install.sh | sh +curl -fsSL https://raw.githubusercontent.com/hungpham10/codegraph-rs/main/scripts/install.sh | sh ``` Drops `codegraph` into `~/.local/bin`. Override with `CODEGRAPH_INSTALL_DIR`. @@ -37,7 +41,7 @@ Drops `codegraph` into `~/.local/bin`. Override with `CODEGRAPH_INSTALL_DIR`. **Windows (PowerShell)** ```powershell -irm https://raw.githubusercontent.com/Cleboost/codegraph-rs/main/scripts/install.ps1 | iex +irm https://raw.githubusercontent.com/hungpham10/codegraph-rs/main/scripts/install.ps1 | iex ``` Installs to `%LOCALAPPDATA%\codegraph\bin` and adds it to the user PATH. @@ -53,7 +57,7 @@ yay -S codegraph-rs-bin
Manual -1. Download the archive for your platform from the [latest release](https://github.com/Cleboost/codegraph-rs/releases/latest): +1. Download the archive for your platform from the [latest release](https://github.com/hungpham10/codegraph-rs/releases/latest): | Platform | File | |---|---| @@ -70,10 +74,10 @@ yay -S codegraph-rs-bin
From source -Requires Rust stable (≥ 1.80). +Requires Rust stable (≥ 1.85 — `codegraph-graph` uses edition 2024). ```sh -git clone https://github.com/Cleboost/codegraph-rs +git clone https://github.com/hungpham10/codegraph-rs cd codegraph-rs cargo build --release -p codegraph # binary at target/release/codegraph @@ -82,7 +86,7 @@ cargo build --release -p codegraph Or via Cargo directly: ```sh -cargo install --git https://github.com/Cleboost/codegraph-rs codegraph +cargo install --git https://github.com/hungpham10/codegraph-rs codegraph ```
@@ -103,7 +107,7 @@ codegraph serve --mcp --http --addr 0.0.0.0:8123 ``` The agent then binds the workspace with `codegraph_init {"path": ...}` and gets -tools like `codegraph_search`, `codegraph_symbol`, `codegraph_callers`, +tools like `codegraph_search_symbol`, `codegraph_symbol`, `codegraph_callers`, `codegraph_flow`, `codegraph_search_flow`, `codegraph_impact`, `codegraph_context` — all querying is done **over MCP**, not via CLI commands. The file watcher debounces changes and triggers full re-indexes while you edit. @@ -122,10 +126,11 @@ runs the MCP server. All reading/interacting goes through MCP tools. | Command | What it does | |---|---| -| `codegraph init [--no-index]` | Create `.codegraph/` and full re-index (skip with `--no-index`) | +| `codegraph init [--no-index]` | Create `.codegraph/` and full re-index (skip with `--no-index`); live progress bar on by default (`--no-progress` to disable) | | `codegraph deinit` | Remove `.codegraph/` | +| `codegraph embed [--model ] [--cache-dir ]` | Pre-download an embedding model into the global cache so semantic search works offline (requires the `fastembed` feature; default model `bge-small-en-v1.5`) | | `codegraph serve --mcp` | Run as MCP server over stdio (used by agents) | -| `codegraph serve --mcp --http` | Run as MCP server over Streamable HTTP (SSE); `--addr` (default `0.0.0.0:8123`), `--allow-host ` (repeatable, LAN), `--allow-any-host` | +| `codegraph serve --mcp --http` | Run as MCP server over Streamable HTTP (SSE); `--addr` (default `0.0.0.0:8123`), `--allow-host ` (repeatable, LAN), `--allow-any-host`, `--format minimize\|medium` (response encoding for LLM token tuning, default `minimize`) | Global flag `--path ` overrides the workspace root. @@ -143,14 +148,15 @@ Each language emits: ## MCP tools -Agents see **30 tools** through the MCP server (search, callers/callees/impact/ -flow, class queries, annotations, dependencies, diff draft/simulation, behavior -sandbox, usage report, plus the session tools `codegraph_init` / -`codegraph_deinit` / `codegraph_index`). Key ones: +Agents see **27 tools** through the MCP server (search with match modes +including opt-in semantic/hybrid, callers/callees/impact/flow, class queries, +annotations, dependencies, diff draft/simulation, behavior sandbox, usage +report, plus the session tools `codegraph_init` / `codegraph_deinit` / +`codegraph_index`). Key ones: | Tool | Use case | |---|---| -| `codegraph_search` | Find symbols by name (substring, case-insensitive) | +| `codegraph_search_symbol` | Find symbols by name with match modes: `contains` (default), `prefix`, `suffix`, `exact`, plus opt-in `semantic` (vector KNN over embeddings) and `hybrid` (contains + semantic merged via Reciprocal Rank Fusion) | | `codegraph_symbol` | Look up a symbol by id or exact name; duplicate names → `ambiguous=true` with full match list; retry with `id` | | `codegraph_callers` | What (transitively) calls this function? (BFS on chain engine) | | `codegraph_callees` | What does this function call directly? (read chain, skip markers) | @@ -181,7 +187,7 @@ Tokens can be: marker names (`LOOP`, `IF_TRUE`, `IF_FALSE`, `BRANCH_END`, `RETUR ### Disambiguation -When `codegraph_symbol` or `codegraph_search` returns duplicate names: +When `codegraph_symbol` or `codegraph_search_symbol` returns duplicate names: ```json { "ambiguous": true, @@ -196,12 +202,13 @@ When `codegraph_symbol` or `codegraph_search` returns duplicate names: crates/ codegraph-core/ Error + semgraph model (Symbol, SymbolKind, Chain, CallRecord, EffectType, ScopeLevel, markers) codegraph-extract/ tree-sitter native + 14 LangSpec declarative extractors + 5 hand-written - codegraph-graph/ GraphIndex (semgraph): registry + 2 engines (chain Search + name Search) + sqlite storage + codegraph-graph/ GraphIndex (semgraph): registry + 2 engines (chain Search + name Search) + pluggable storage (SQLite / LMDB / Redis / Postgres / MySQL) + optional embedding vector index codegraph-context/ Markdown/JSON context formatter (symbol + callers + callees + source) codegraph-api/ GraphApi wrapper on SharedGraphIndex (async query surface) - codegraph-mcp/ MCP server on the rmcp SDK (stdio + Streamable HTTP) + 30-tool dispatch, session-driven - codegraph-installer/ Agent config targets (Claude/Cursor/Codex/opencode/Hermes) - codegraph/ CLI lifecycle (init/deinit/serve --mcp) + watcher (notify + debounced full re-index) + codegraph-sboxes/ Behavior sandbox: Cranelift JIT compile of function groups + Rhai mock runtime + codegraph-mcp/ MCP server on the rmcp SDK (stdio + Streamable HTTP) + 27-tool dispatch, session-driven + codegraph-bench/ Benchmarks (criterion search benches, storage benches, codspeed) + codegraph/ CLI lifecycle (init/deinit/embed/serve --mcp) + watcher (notify + debounced full re-index) ``` Pipeline: @@ -229,8 +236,8 @@ A `.codegraph/` directory is created next to your project: ``` .codegraph/ - db.sqlite SQLite v1 (WAL mode, single file — entities + radix streams) - config.toml Language enable/disable, walker include/exclude + db.sqlite SQLite (WAL mode, single file — entities + radix streams); db.lmdb/ directory when the LMDB backend is selected + config.toml Language toggles, walker filters, storage backend, embedding settings .gitignore Pre-filled so the index is never committed version Codegraph version that created the directory ``` @@ -266,8 +273,36 @@ exclude = [ "*.min.js", "*.lock" ] + +# Storage backend — "sqlite" (default) | "lmdb" | "redis" | "memory" | "postgres" | "mysql" +[storage] +type = "sqlite" +# DSN override. Defaults: sqlite → sqlite:///.codegraph/db.sqlite, +# lmdb → lmdb:///.codegraph/db.lmdb (directory). Redis REQUIRES a dsn. +# dsn = "redis://localhost:6379" +# Postgres/MySQL use `dsns` (shard list) + `repo_id` — see below. + +# Semantic search (vector KNN) — OFF by default. See "Semantic search" below. +[embedding] +# backend = "fastembed" +# model = "bge-small-en-v1.5" +# cache_dir = "~/.cache/codegraph/embeddings" ``` +### Storage backends + +The `[storage]` section selects where the index lives: + +| `type` | Notes | +|---|---| +| `sqlite` | Default. Single-file `db.sqlite` (WAL) inside `.codegraph/`. | +| `lmdb` | Memory-mapped KV (`db.lmdb/` directory inside `.codegraph/`). Same local-first workflow, mmap-friendly for large indexes. Enabled by default in the `codegraph` binary. | +| `redis` | Requires an explicit `dsn` (e.g. `redis://localhost:6379`) — there is no sensible local default. | +| `memory` | Ephemeral in-process index; nothing is persisted. | +| `postgres` / `mysql` | Multi-tenant, sharded — see below. | + +`dsn` (when set) overrides the derived default for any backend. + ### Postgres / MySQL (multi-tenant, sharded) CodeGraph can store the index in PostgreSQL or MySQL instead of the local @@ -316,6 +351,47 @@ Then `codegraph init` (CLI) or `codegraph_init` (MCP tool) generates the `repo_id` and stores the index on the right shard automatically. See `sql/README.md` for the full multi-tenant + sharding design. +### Semantic search (optional, opt-in) + +Vector similarity search over symbol embeddings is **off by default** — no +embedding model runs unless you enable it in config. The release binary +already bundles the fastembed (ONNX sentence-transformer) backend, so +enabling it is config-only — no rebuild required: + +1. Enable it in `.codegraph/config.toml`: + + ```toml + [embedding] + backend = "fastembed" # "hashing"/unset = off + model = "bge-small-en-v1.5" # 384-dim, default + cache_dir = "~/.cache/codegraph/embeddings" # global model cache (default) + # SQLite-only: point at a sqlite-vss (vector0/vss0) extension directory to + # run KNN through HNSW ANN inside the database: + # vss_extension = "~/.cache/codegraph/embeddings/vss" + # execution_provider = "coreml" # macOS hardware acceleration + ``` + +2. Optionally pre-download the model so indexing works offline: + + ```sh + codegraph embed --model bge-small-en-v1.5 + ``` + + The `codegraph embed` subcommand is compiled in when the binary is built + with `--features fastembed`. + +With embeddings enabled, `codegraph_search_symbol` gains the `match` modes +`"semantic"` (vector KNN — find symbols by similar/approximate names) and +`"hybrid"` (substring + semantic merged via Reciprocal Rank Fusion). Vectors +are persisted with the index, so restarts reuse them without re-embedding. + +Notes: +- If the model fails to load (no network, missing ONNX runtime), opening the + index **errors out** — there is no silent fallback to a lexical baseline. +- On macOS you can build with `--features fastembed,apple-accel` to run + embeddings on the Apple Neural Engine / GPU via the CoreML execution + provider. That feature is macOS-only and fails to build elsewhere. + ### C vs C++ headers (`.h`) By default, `.h` files are resolved automatically: @@ -342,7 +418,9 @@ The Rust port: - Parses in parallel via `rayon` - Builds with `lto="fat"`, `codegen-units=1`, `strip`, `panic=abort` -Result: **~5 MB** stripped, **sub-second** startup, **~5× faster** indexing on the same workspace. +Result: a single **~58 MB** stripped binary with every backend bundled +(SQLite, LMDB, Redis, Postgres/MySQL drivers, ONNX embedding runtime), +**sub-second** startup, and **~5× faster** indexing on the same workspace. ## Semgraph model (wire-breaking) @@ -376,7 +454,8 @@ cargo test -p codegraph-extract # 30 tests: 10 lib + 16 chains + 2 cpp + 2 ex cargo test -p codegraph-graph # 60+ tests: search, storage, ingest, flow, reopen cargo test -p codegraph-api cargo test -p codegraph-mcp -cargo test -p codegraph-viz +cargo test -p codegraph-sboxes # sandbox JIT: control flow + end-to-end traces +cargo test -p codegraph-bench # pipeline integration cargo test -p codegraph-installer ``` @@ -390,20 +469,34 @@ cargo test -p codegraph-extract --features lang-python ``` Feature flags on `codegraph-graph`: -- `sqlite` — sqlite storage backend (enabled on `codegraph`, `codegraph-mcp`, `codegraph-viz`) +- `sqlite` — sqlite storage backend (enabled on `codegraph`, `codegraph-mcp`) +- `lmdb` — LMDB storage backend, memory-mapped KV bundled C library (enabled on `codegraph`) - `redis` — redis storage backend (compile-only verify, runtime needs server) - `postgres` — PostgreSQL storage backend (multi-tenant, sharded) - `mysql` — MySQL storage backend (multi-tenant, sharded) +- `bloom-search` — bloom-filter acceleration for chain searches (enabled on `codegraph`) +- `fastembed` — ONNX embedding backend for semantic search (currently also pulled in unconditionally by `codegraph-api`, so it is present in release builds) +- `apple-accel` — macOS-only CoreML execution provider for ONNX Runtime (pair with `fastembed`; build fails on non-macOS) + +Feature flags on the `codegraph` binary: +- `rdbms` (default) — turns on `postgres` + `mysql` for the CLI and MCP server +- `fastembed` — compiles in the `codegraph embed` CLI command (the embedding backend itself is already bundled via `codegraph-api`) +- `apple-accel` — macOS-only hardware acceleration for embeddings + +The `codegraph-mcp` crate exposes the same `rdbms` convenience feature (not +enabled by default there). -The `codegraph` and `codegraph-mcp` binaries expose a convenience `rdbms` -feature that turns on both `postgres` and `mysql` (it is **on by default** -for `codegraph`): +Note: `codegraph-api` currently enables every `codegraph-graph` feature, so +`cargo build -p codegraph --no-default-features` verifies the CLI compiles +without `rdbms` wiring but does **not** produce a slimmer binary — all +storage drivers and the embedding backend are still compiled in. ```sh # Full feature verification cargo check --workspace --features sqlite cargo check -p codegraph-graph --features redis cargo check -p codegraph --features rdbms +cargo check -p codegraph --features fastembed ``` ## License @@ -414,4 +507,4 @@ MIT. See [LICENSE](LICENSE). - The original TypeScript implementation by [@colbymchenry](https://github.com/colbymchenry). - `tree-sitter` and all language grammar authors. -- `rusqlite`, `notify`, `clap`, `tokio`, `rayon`, `ignore`, `dashmap`, `parking_lot`. \ No newline at end of file +- `rusqlite`, `notify`, `clap`, `tokio`, `rayon`, `ignore`, `dashmap`, `parking_lot`. diff --git a/crates/codegraph-api/Cargo.toml b/crates/codegraph-api/Cargo.toml index 2940fbe23..5609fbcca 100644 --- a/crates/codegraph-api/Cargo.toml +++ b/crates/codegraph-api/Cargo.toml @@ -7,7 +7,7 @@ repository.workspace = true [dependencies] codegraph-core = { path = "../codegraph-core" } -codegraph-graph = { path = "../codegraph-graph", features = ["sqlite"] } +codegraph-graph = { path = "../codegraph-graph", features = ["sqlite", "lmdb","redis","postgres","mysql", "bloom-search","fastembed"] } codegraph-context = { path = "../codegraph-context" } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/codegraph-api/src/lib.rs b/crates/codegraph-api/src/lib.rs index 551bedf7a..3d2f7e9ea 100644 --- a/crates/codegraph-api/src/lib.rs +++ b/crates/codegraph-api/src/lib.rs @@ -219,54 +219,7 @@ impl GraphApi { self.shared_index.ensure_fresh().await } - /// Search symbol theo tên (substring, case-insensitive). - pub async fn search(&self, query: &str, limit: u32) -> Result> { - self.index() - .await - .search_symbol(query, None, limit as usize) - .await - } - - /// Resumable + deadline-aware của [`Self::search`] — nền cho - /// `codegraph_search`. `timeout_ms = 0` = không giới hạn thời gian. - /// `timeout_ms = u64::MAX` ([`TIMEOUT_EXPIRE_IMMEDIATELY`]) = deadline đã - /// hết hạn ngay → chắc chắn `timed_out` (dùng cho test xác định). - /// `resume` = id trả về từ lần timeout trước (phải cùng query). - pub async fn search_resumable( - &self, - query: &str, - limit: u32, - resume: Option, - timeout_ms: u64, - ) -> Result { - self.search_symbol_paged_resumable( - query, - None, - SymbolMatch::Contains, - Pagination { limit, offset: 0 }, - resume, - timeout_ms, - ) - .await - } - - /// Search symbol nâng cao — kind filter + match mode + phân trang. - /// Trả về (page, total). - pub async fn search_symbol_paged( - &self, - query: &str, - kind: Option, - mode: SymbolMatch, - limit: u32, - offset: u32, - ) -> Result<(Vec, usize)> { - self.index() - .await - .search_symbol_paged(query, kind, mode, limit as usize, offset as usize) - .await - } - - /// Resumable + deadline-aware của [`Self::search_symbol_paged`] — nền cho + /// Search symbol nâng cao (resumable + deadline-aware) — nền cho /// `codegraph_search_symbol`. `timeout_ms = 0` = không giới hạn; /// `timeout_ms = u64::MAX` ([`TIMEOUT_EXPIRE_IMMEDIATELY`]) = chắc chắn /// `timed_out` (dùng cho test xác định). diff --git a/crates/codegraph-api/tests/api.rs b/crates/codegraph-api/tests/api.rs index d75595524..14a6d5c08 100644 --- a/crates/codegraph-api/tests/api.rs +++ b/crates/codegraph-api/tests/api.rs @@ -77,8 +77,22 @@ async fn search_and_symbol_by_id() { let (caller, _, _) = seed_index(&db_str).await; let api = api(&db_str).await; - // Substring search. - let hits = api.search("call", 10).await.unwrap(); + // Substring search (resumable path, no deadline). + let hits = api + .search_symbol_paged_resumable( + "call", + None, + SymbolMatch::Contains, + Pagination { + limit: 10, + offset: 0, + }, + None, + 0, + ) + .await + .unwrap() + .page; assert!(hits.iter().any(|s| s.id == caller)); // Symbol by id. assert_eq!(api.symbol_by_id(caller).await.unwrap().name, "caller"); @@ -276,7 +290,17 @@ async fn search_resumable_timeout_retry_roundtrip() { // total = 5000 vì name engine chặn cứng MAX_RESULTS tên distinct. let capped = 5000; let first = api - .search_resumable("order", 20, None, codegraph_api::TIMEOUT_EXPIRE_IMMEDIATELY) + .search_symbol_paged_resumable( + "order", + None, + SymbolMatch::Contains, + Pagination { + limit: 20, + offset: 0, + }, + None, + codegraph_api::TIMEOUT_EXPIRE_IMMEDIATELY, + ) .await .unwrap(); assert!(first.timed_out, "expired deadline must time out"); @@ -284,7 +308,17 @@ async fn search_resumable_timeout_retry_roundtrip() { // Retry: cùng args + resume, không giới hạn thời gian → hoàn tất. let out = api - .search_resumable("order", 20, Some(resume_id.clone()), 0) + .search_symbol_paged_resumable( + "order", + None, + SymbolMatch::Contains, + Pagination { + limit: 20, + offset: 0, + }, + Some(resume_id.clone()), + 0, + ) .await .unwrap(); assert!(!out.timed_out); @@ -299,16 +333,36 @@ async fn search_resumable_timeout_retry_roundtrip() { // Resume id không tồn tại → lỗi (LLM nên retry không resume). assert!( - api.search_resumable("order", 20, Some("deadbeef00000000".into()), 0) - .await - .is_err(), + api.search_symbol_paged_resumable( + "order", + None, + SymbolMatch::Contains, + Pagination { + limit: 20, + offset: 0 + }, + Some("deadbeef00000000".into()), + 0, + ) + .await + .is_err(), "unknown resume id must be rejected" ); // Resume id không khớp query → lỗi. assert!( - api.search_resumable("totally_different", 20, Some(resume_id), 0) - .await - .is_err(), + api.search_symbol_paged_resumable( + "totally_different", + None, + SymbolMatch::Contains, + Pagination { + limit: 20, + offset: 0 + }, + Some(resume_id), + 0, + ) + .await + .is_err(), "resume id for a different query must be rejected" ); } diff --git a/crates/codegraph-bench/src/lib.rs b/crates/codegraph-bench/src/lib.rs index b2db08943..9daed8d52 100644 --- a/crates/codegraph-bench/src/lib.rs +++ b/crates/codegraph-bench/src/lib.rs @@ -136,7 +136,21 @@ pub fn run_queries( let start = Instant::now(); let mut ops = 0usize; for name in names { - if let Ok(hits) = idx.search_symbol(name, None, 5).await { + if let Ok(out) = idx + .search_symbol_paged_resumable( + name, + None, + codegraph_core::SymbolMatch::Contains, + codegraph_graph::Pagination { + limit: 5, + offset: 0, + }, + None, + None, + ) + .await + { + let hits = out.page; ops += 1; let Some(h) = hits.first() else { continue }; // callees + flow = 2 phép đọc chain engine + flow. diff --git a/crates/codegraph-context/src/lib.rs b/crates/codegraph-context/src/lib.rs index 29c92040c..5f1a9c290 100644 --- a/crates/codegraph-context/src/lib.rs +++ b/crates/codegraph-context/src/lib.rs @@ -4,8 +4,8 @@ //! còn `Db`/`Traversal` cũ — query surface mới của `GraphIndex`: //! `search_symbol` → `callers`/`callees` (BFS trên chain engine). -use codegraph_core::{Result, Symbol}; -use codegraph_graph::SharedGraphIndex; +use codegraph_core::{Result, Symbol, SymbolMatch}; +use codegraph_graph::{Pagination, SharedGraphIndex}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt::Write; @@ -74,8 +74,19 @@ pub async fn build_response( ) -> Result { let idx = index.ensure_fresh().await; let candidates = idx - .search_symbol(&req.query, None, req.limit as usize) - .await?; + .search_symbol_paged_resumable( + &req.query, + None, + SymbolMatch::Contains, + Pagination { + limit: req.limit as usize, + offset: 0, + }, + None, + None, + ) + .await? + .page; // Pre-load mỗi file một lần khi cần source. let file_cache: HashMap> = if req.include_source { diff --git a/crates/codegraph-core/src/semgraph.rs b/crates/codegraph-core/src/semgraph.rs index 3bc6a0d3d..bacd71da0 100644 --- a/crates/codegraph-core/src/semgraph.rs +++ b/crates/codegraph-core/src/semgraph.rs @@ -462,6 +462,12 @@ pub enum SymbolMatch { Suffix, /// Tên trùng chính xác (case-insensitive). Exact, + /// Semantic (vector): query → embedding → KNN over symbol embeddings — + /// tìm symbol **tên tương tự / cùng ý nghĩa** kể cả khi không khớp substring. + Semantic, + /// Hybrid: chạy cả `Contains` (lexical) lẫn `Semantic` (vector), gộp kết + /// quả bằng Reciprocal Rank Fusion (RRF). + Hybrid, } impl SymbolMatch { @@ -472,6 +478,8 @@ impl SymbolMatch { "prefix" => Self::Prefix, "suffix" => Self::Suffix, "exact" => Self::Exact, + "semantic" => Self::Semantic, + "hybrid" => Self::Hybrid, _ => return None, }) } diff --git a/crates/codegraph-extract/src/config.rs b/crates/codegraph-extract/src/config.rs index f112e624a..ee627ef85 100644 --- a/crates/codegraph-extract/src/config.rs +++ b/crates/codegraph-extract/src/config.rs @@ -61,6 +61,9 @@ struct ConfigFile { /// Backend storage (mặc định sqlite). #[serde(default)] storage: StorageSection, + /// Embedding backend cho semantic search (fastembed / hashing) + cache model. + #[serde(default)] + embedding: EmbeddingSection, } #[derive(Debug, Default, Deserialize)] @@ -87,6 +90,30 @@ struct LanguagesSection { headers: Option, } +#[derive(Debug, Default, Deserialize)] +struct EmbeddingSection { + /// `"fastembed"` | `"hashing"`. + #[serde(default)] + backend: Option, + /// Tên model fastembed (alias hoặc variant name). + #[serde(default)] + model: Option, + /// Thư mục cache model (global). Mặc định `~/.cache/codegraph/embeddings`. + #[serde(default)] + cache_dir: Option, + /// Thư mục chứa extension sqlite-vss (`vector0`/`vss0`) — CHỈ backend SQLite. + /// Khi set (và file tồn tại), KNN semantic chạy qua `vss0` (HNSW ANN trong + /// SQLite). Thiếu file → fallback brute-force. `None` → tự dò `/vss`. + #[serde(default)] + vss_extension: Option, + /// Execution provider cho ONNX Runtime — `"cpu"` (mặc định) | `"coreml"` + /// (Apple Neural Engine / GPU, macOS) | `"metal"` (GPU, macOS). Chỉ có hiệu + /// lực khi build `--features fastembed,apple-accel` trên macOS; ngược lại bỏ + /// qua (chạy CPU). Platform khác macOS luôn CPU. + #[serde(default)] + execution_provider: Option, +} + /// Raw rule — `effect` để string để rule lỗi (unknown) bị skip + warn, không /// làm hỏng toàn bộ config; parse lại bằng `EffectType::parse`. #[derive(Debug, Deserialize)] @@ -104,6 +131,8 @@ pub struct ExtractConfig { pub effect_classifier: EffectClassifier, /// Backend storage được chọn trong config (mặc định sqlite). pub storage: StorageConfig, + /// Cấu hình embedding backend (semantic search) — đọc từ `[embedding]`. + pub embedding: codegraph_graph::embeddings::EmbeddingConfig, } /// Storage backend đã parse từ `[storage]` trong config. @@ -135,6 +164,13 @@ impl ExtractConfig { Self { header_language: parse_header_language(file.languages.headers.as_deref()), effect_classifier: build_classifier(file.effect_rules), + embedding: codegraph_graph::embeddings::EmbeddingConfig::from_raw( + file.embedding.backend.as_deref(), + file.embedding.model.as_deref(), + file.embedding.cache_dir.as_deref(), + file.embedding.vss_extension.as_deref(), + file.embedding.execution_provider.as_deref(), + ), storage: StorageConfig { kind: file .storage @@ -178,6 +214,10 @@ impl ExtractConfig { /// - `postgres` / `mysql` → `Sharded { dsns, repo_id, root }` /// (`repo_id` phải đã được sinh bởi `ensure_repo_id`; nếu thiếu → `None`) pub fn storage_route(&self, root: &Utf8Path) -> Option { + // Áp dụng config embedding (backend/model/cache) cho process trước khi + // mở index — `GraphIndex::new_with_storage` đọc global này để quyết định + // có bật vector index hay không (opt-in: chỉ khi backend = "fastembed"). + codegraph_graph::embeddings::set_embedding_config(self.embedding.clone()); match self.storage.kind { StorageKind::Memory => Some(StorageRoute::Memory), StorageKind::Postgres | StorageKind::MySql => { @@ -289,6 +329,30 @@ type = "sqlite" # dsns = ["postgres://user:pass@db1:5432/codegraph", "postgres://user:pass@db2:5432/codegraph"] # repo_id = 14028493579208694412 # sinh bởi `codegraph init` (partition key) # dsn = "sqlite:///tmp/codegraph.db" + +[embedding] +# Semantic search (vector KNN/k-means) là OPT-IN — MẶC ĐỊNH TẮT. +# Bỏ comment + set "fastembed" để bật vector search (cần compile `--features fastembed` +# và tải model ONNX lúc chạy). Nếu tắt, semantic/hybrid search sẽ báo lỗi rõ ràng +# (KHÔNG fallback silent sang lexical). +# backend = "fastembed" +# Model fastembed — alias thân thiện hoặc variant name, VD: +# bge-small-en-v1.5 (mặc định, 384-dim), bge-base-en-v1.5, bge-large-en-v1.5, +# all-minilm-l6-v2, all-mpnet-base-v2, nomic-embed-text-v1.5, multilingual-e5-small. +# model = "bge-small-en-v1.5" +# Thư mục cache model (global, chia sẻ mọi project) — pre-download bằng +# `codegraph embed --model ` để chạy offline. Mặc định ~/.cache/codegraph/embeddings. +# cache_dir = "~/.cache/codegraph/embeddings" +# SQLite-only: dùng sqlite-vss (vector0/vss0) để KNN chạy HNSW ANN ngay trong +# SQLite thay vì brute-force in-memory. Cần 2 file extension trong thư mục này +# (tự build hoặc tải prebuilt). Có mặt → bật; thiếu → fallback brute-force. +# vss_extension = "~/.cache/codegraph/embeddings/vss" +# Execution provider cho ONNX Runtime (chỉ macOS, build `--features fastembed,apple-accel`): +# "cpu" (mặc định) → CPU + Accelerate/vecLib SIMD, mọi core +# "coreml" → Core ML EP (Apple Neural Engine / GPU trên Apple Silicon) +# "metal" → Metal EP (GPU) +# Build thiếu `apple-accel`, hoặc platform khác macOS → bỏ qua, chạy CPU. +# execution_provider = "cpu" "#; /// Quick project scan: returns a hint when the tree is clearly C-only or C++-only. diff --git a/crates/codegraph-extract/src/walker.rs b/crates/codegraph-extract/src/walker.rs index 1de52529a..3c11b1128 100644 --- a/crates/codegraph-extract/src/walker.rs +++ b/crates/codegraph-extract/src/walker.rs @@ -219,6 +219,7 @@ mod tests { header_language: HeaderLanguage::Cpp, effect_classifier: Default::default(), storage: Default::default(), + embedding: Default::default(), }; let matches = walk(&root, &parsers, &config); let h = matches diff --git a/crates/codegraph-extract/tests/extract.rs b/crates/codegraph-extract/tests/extract.rs index f8bc95163..330ae5386 100644 --- a/crates/codegraph-extract/tests/extract.rs +++ b/crates/codegraph-extract/tests/extract.rs @@ -1,8 +1,28 @@ //! Integration: Orchestrator walk + parse → GraphIndex::ingest → search. use camino::Utf8PathBuf; +use codegraph_core::Symbol; use codegraph_extract::Orchestrator; -use codegraph_graph::GraphIndex; +use codegraph_graph::{GraphIndex, Pagination}; + +/// Tiện ích: search substring (resumable) trả `Vec` — thay thế +/// `GraphIndex::search_symbol` đã xoá (mọi search đều qua resumable path). +async fn search_symbol(idx: &GraphIndex, q: &str) -> Vec { + idx.search_symbol_paged_resumable( + q, + None, + codegraph_core::SymbolMatch::Contains, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) + .await + .unwrap() + .page +} fn fixture_root() -> Utf8PathBuf { Utf8PathBuf::from_path_buf( @@ -32,50 +52,50 @@ async fn index_fixtures_dir() { assert!(stats.calls > 0, "expected calls"); // Java - let hits = index.search_symbol("UserService", None, 10).await.unwrap(); + let hits = search_symbol(&index, "UserService").await; assert!( hits.iter().any(|s| s.language == "java"), "expected java hit, got {hits:?}" ); // Ruby - let hits = index.search_symbol("UserService", None, 10).await.unwrap(); + let hits = search_symbol(&index, "UserService").await; assert!( hits.iter().any(|s| s.language == "ruby"), "expected ruby hit" ); // Python - let hits = index.search_symbol("process_user", None, 10).await.unwrap(); + let hits = search_symbol(&index, "process_user").await; assert!( hits.iter().any(|s| s.language == "python"), "expected python hit" ); // Go - let hits = index.search_symbol("ProcessUser", None, 10).await.unwrap(); + let hits = search_symbol(&index, "ProcessUser").await; assert!(hits.iter().any(|s| s.language == "go"), "expected go hit"); // JS - let hits = index.search_symbol("processUser", None, 10).await.unwrap(); + let hits = search_symbol(&index, "processUser").await; assert!( hits.iter().any(|s| s.language == "javascript"), "expected js hit" ); // TS - let hits = index.search_symbol("processUser", None, 10).await.unwrap(); + let hits = search_symbol(&index, "processUser").await; assert!( hits.iter().any(|s| s.name == "processUser"), "missing processUser, got {hits:?}" ); // Rust - let hits = index.search_symbol("process_user", None, 10).await.unwrap(); + let hits = search_symbol(&index, "process_user").await; assert!(hits.iter().any(|s| s.name == "process_user")); // UserService from TS class + Rust struct - let hits = index.search_symbol("UserService", None, 10).await.unwrap(); + let hits = search_symbol(&index, "UserService").await; assert!( hits.len() >= 2, "expected UserService from both TS and Rust, got {}", @@ -90,7 +110,7 @@ async fn chains_are_built_for_each_function() { assert!(stats.chains > 0, "expected chains in index"); // Flow của một function trả về chain có marker hoặc ít nhất là chính nó. - let hits = index.search_symbol("process_user", None, 10).await.unwrap(); + let hits = search_symbol(&index, "process_user").await; let py = hits .iter() .find(|s| s.language == "python") diff --git a/crates/codegraph-graph/Cargo.toml b/crates/codegraph-graph/Cargo.toml index f68858738..9b09bc87a 100644 --- a/crates/codegraph-graph/Cargo.toml +++ b/crates/codegraph-graph/Cargo.toml @@ -36,6 +36,17 @@ lmdb-rkv = { workspace = true, optional = true } # Bundled sqlite cho sqlx (giống rusqlite của codegraph-db) — feature # unification khiến sqlx dùng chung bản build bundled này, không cần system lib. libsqlite3-sys = { version = "0.30", features = ["bundled"], optional = true } +fastembed = { version = "5.17.4", optional = true } + +# macOS-only: ONNX Runtime compile với feature `coreml` để chạy embedding trên +# Apple Neural Engine / GPU (Apple Silicon). Chỉ được pull vào khi feature +# `apple-accel` bật (macOS) → build non-macOS KHÔNG kéo native build này. +# NOTE: phiên bản `ort` hiện tại (2.0.0-rc.13) KHÔNG expose feature `metal` (Metal +# EP) — chỉ `coreml`. CoreML trên Apple Silicon đã tận dụng GPU/ANE, nên đây là +# đường truyền tăng tốc phần cứng duy nhất khả dụng. `ort = "=2.0.0-rc.13"` phải +# khớp phiên bản fastembed đang dùng để 2 dep unify thành 1 bản build. +[target.'cfg(target_os = "macos")'.dependencies] +ort = { version = "=2.0.0-rc.13", features = ["coreml"], optional = true } [features] default = [] @@ -47,6 +58,16 @@ lmdb = ["dep:lmdb-rkv"] postgres = ["dep:sqlx"] mysql = ["dep:sqlx"] bloom-search = [] +# Embedding backend fastembed (ONNX / sentence-transformers) cho semantic search. +# OPT-IN: không bật mặc định. Chỉ khi config `[embedding].backend = "fastembed"` +# (và crate compile `--features fastembed`) thì vector index mới được xây. Nếu +# bật mà model tải thất bại → init index lỗi (KHÔNG fallback silent sang hashing). +fastembed = ["dep:fastembed"] +# macOS-only: bật CoreML/Metal execution provider cho ONNX Runtime (embed trên +# ANE/GPU). Chỉ có nghĩa khi build trên macOS + `--features fastembed,apple-accel`. +# Trên non-macOS, bật feature này SẼ LỖI (ort chỉ compile được trên macOS với +# coreml/metal) → build bình thường `--features fastembed` (CPU + Accelerate SIMD). +apple-accel = ["dep:ort"] [dev-dependencies] tempfile = "3" diff --git a/crates/codegraph-graph/src/embeddings.rs b/crates/codegraph-graph/src/embeddings.rs new file mode 100644 index 000000000..fcb5e521b --- /dev/null +++ b/crates/codegraph-graph/src/embeddings.rs @@ -0,0 +1,534 @@ +//! Embedding backend cho semantic search (KNN / k-means). +//! +//! Thiết kế **pluggable**: `GraphIndex` chỉ biết [`EmbeddingBackend`] (trait) — +//! backend cụ thể sinh vector từ text. Có hai backend: +//! +//! - [`FastEmbedBackend`] (feature `fastembed`): dùng crate `fastembed` chạy +//! model ONNX **BAAI/bge-small-en-v1.5** (384-dim, đa ngôn ngữ) — sinh vector +//! **semantic thật** (sentence-transformer). Đây là backend duy nhất sinh +//! vector dùng được; **phải được bật tường minh** qua `[embedding].backend`. +//! - [`HashingEmbeddings`]: dependency-free, thuần Rust — baseline lexical khi +//! không bật fastembed. **KHÔNG** được dùng làm fallback silent: nếu +//! `[embedding].backend = "fastembed"` mà model tải thất bại (thiếu mạng, +//! thiếu ONNX runtime...), init index sẽ **báo lỗi** chứ không lặng lẽ chuyển +//! sang hashing. +//! +//! ## Opt-in (mặc định TẮT) +//! +//! Embedding **không bật mặc định**. Chỉ khi `[embedding].backend = "fastembed"` +//! (trong `.codegraph/config.toml`) được set vào lúc `init`/`open` thì vector +//! index mới được xây + persist. Nếu không set (hoặc set `"hashing"`), semantic +//! search không khả dụng và `GraphIndex` không chạy embedding gì cả. +//! +//! ```toml +//! [embedding] +//! backend = "fastembed" # "fastembed" (bật) | "hashing"/unset (tắt) +//! model = "bge-small-en-v1.5" # alias thân thiện hoặc variant name +//! cache_dir = "~/.cache/codegraph/embeddings" # thư mục global chứa model +//! ``` +//! +//! `cache_dir` là **thư mục global** — model được tải/đệm vào đây một lần, chia +//! sẻ cho mọi project. Dùng `codegraph embed --model ` để pre-download trước. +//! Xem [`EmbeddingConfig`] / [`set_embedding_config`] / [`warm_model_cache`]. +//! +//! Các vector được **persist vào storage** (qua `Storage::save_embedding`) để +//! KNN/k-means tái dùng qua các lần restart mà không phải re-embed. +//! +//! Cả hai backend đều trả vector đã **L2-normalize** (cosine similarity = dot +//! product) để `VectorIndex` hoạt động nhất quán. + +use std::collections::hash_map::DefaultHasher; +use std::hash::{Hash, Hasher}; +use std::path::PathBuf; +use std::sync::OnceLock; + +/// Số chiều mặc định của vector embedding. +/// +/// Bằng đúng dim của model fastembed mặc định (BGE-small-en-v1.5 = 384) để +/// `VectorIndex` khởi tạo cùng chiều với backend mặc định; `rebuild_vector_index` +/// vẫn lấy `dim()` từ backend thực tế nên khác biệt nhẹ không gây lỗi. +pub const VECTOR_DIM: usize = 384; + +/// Backend sinh embedding: ánh xạ một đoạn text → vector f32 (đã L2-normalize +/// để cosine similarity = dot product). +pub trait EmbeddingBackend: Send + Sync { + /// Số chiều vector. + fn dim(&self) -> usize; + /// Embed `text` → vector f32 đã chuẩn hóa (norm = 1). + fn embed(&self, text: &str) -> Vec; + /// Embed một batch text → vector. Mặc định lặp `embed` từng phần tử (chậm + /// với model ONNX). `FastEmbedBackend` override để batch (tận dụng tính toán + /// vector hoá hàng loạt — nhanh gấp bội so với gọi `embed` tuần tự, nhất là + /// khi index hàng chục ngàn symbol). + fn embed_batch(&self, texts: &[String]) -> Vec> { + texts.iter().map(|t| self.embed(t)).collect() + } +} + +/// L2-normalize một vector (norm = 1). Trả nguyên `v` nếu là zero-vector. +/// Sau khi normalize, cosine similarity = dot product. +fn normalize(mut v: Vec) -> Vec { + let norm = v.iter().map(|x| x * x).sum::().sqrt(); + if norm > 0.0 { + for x in &mut v { + *x /= norm; + } + } + v +} + +/// Loại embedding backend (từ `[embedding].backend`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum EmbeddingBackendKind { + /// fastembed (ONNX, semantic thật). Phải bật tường minh qua config. + Fastembed, + /// HashingEmbeddings (dependency-free, lexical overlap). Không sinh vector + /// semantic; `[embedding]` unset hoặc `backend = "hashing"` → embedding TẮT. + #[default] + Hashing, +} + +impl std::str::FromStr for EmbeddingBackendKind { + type Err = String; + fn from_str(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "fastembed" | "fast" | "onnx" | "embedding" => Ok(Self::Fastembed), + "hashing" | "hash" | "lexical" => Ok(Self::Hashing), + other => Err(format!("unknown embedding backend: {other}")), + } + } +} + +/// Cấu hình embedding backend — đọc từ `[embedding]` trong `.codegraph/config.toml`. +#[derive(Debug, Clone)] +pub struct EmbeddingConfig { + /// Có bật embedding (vector index) không. **OPT-IN**: chỉ `true` khi + /// `[embedding]` được khai báo tường minh trong config (thậm chí chỉ cần + /// có key `backend`). Mặc định `false` → không chạy embedding, không xây + /// vector index, semantic search báo lỗi rõ ràng. + pub enabled: bool, + /// Loại backend (`fastembed` | `hashing`) — chỉ có nghĩa khi `enabled`. + /// - `hashing`: dependency-free, lexical (không tải model). + /// - `fastembed`: ONNX sentence-transformer; **lỗi init nếu tải model thất + /// bại** (KHÔNG fallback silent sang hashing). + pub backend: EmbeddingBackendKind, + /// Tên model fastembed (alias thân thiện hoặc variant name, VD + /// `"bge-small-en-v1.5"` / `"BGESmallENV15"`). Mặc định BGE-small-en-v1.5. + pub model: String, + /// Thư mục cache model (global). `None` → `~/.cache/codegraph/embeddings`. + pub cache_dir: Option, + /// Thư mục chứa extension sqlite-vss (`vector0`/`vss0`). **Chỉ cho backend + /// SQLite**: khi được set (và file tồn tại), KNN semantic chạy qua `vss0` + /// (HNSW ANN trong chính SQLite) thay vì brute-force in-memory. Thiếu file → + /// fallback brute-force (KHÔNG lỗi). `None` → tự dò `/vss`. + pub vss_extension: Option, + /// Execution provider cho ONNX Runtime (fastembed) — chỉ có nghĩa khi backend + /// = `fastembed` VÀ crate compile với feature `apple-accel` (macOS). Giá trị: + /// `None`/"cpu" (mặc định) → ONNX Runtime chạy trên CPU (Accelerate/vecLib + /// SIMD + mọi core); `"coreml"` → Core ML EP (ANE/GPU, Apple Silicon); + /// `"metal"` → Metal EP (GPU). Yêu cầu build `--features fastembed,apple-accel` + /// trên macOS; nếu set `"coreml"`/`"metal"` mà build thiếu `apple-accel` → + /// bỏ qua (chạy CPU), KHÔNG lỗi. Platform khác macOS → luôn CPU. + pub execution_provider: Option, +} + +impl Default for EmbeddingConfig { + /// Mặc định: embedding **TẮT** (`enabled = false`). Phải khai báo `[embedding]` + /// trong config mới kích hoạt. `model`/`cache_dir` vẫn giữ sẵn để khi bật + /// lên không phải set thêm. + fn default() -> Self { + Self { + enabled: false, + backend: EmbeddingBackendKind::Hashing, + model: "bge-small-en-v1.5".to_string(), + cache_dir: default_cache_dir(), + vss_extension: None, + execution_provider: None, + } + } +} + +impl EmbeddingConfig { + /// Parse từ raw config strings (từ `[embedding]` trong `config.toml`). + /// + /// `enabled = true` **chỉ khi** `backend` được khai báo tường minh (dù là + /// `"hashing"` hay `"fastembed"`) — đảm bảo opt-in: bỏ qua `[embedding]` + /// hoàn toàn = tắt. `backend` parse sai → coi như `"hashing"` (vẫn enabled, + /// nhưng dùng backend rẻ, không tải model). + pub fn from_raw( + backend: Option<&str>, + model: Option<&str>, + cache_dir: Option<&str>, + vss_extension: Option<&str>, + execution_provider: Option<&str>, + ) -> Self { + let enabled = backend.is_some(); + let backend = backend + .and_then(|s| s.parse::().ok()) + .unwrap_or_default(); + let model = model.unwrap_or("bge-small-en-v1.5").to_string(); + let cache_dir = cache_dir.and_then(expand_tilde); + let vss_extension = vss_extension.and_then(expand_tilde); + let execution_provider = execution_provider.map(|s| s.trim().to_ascii_lowercase()); + Self { + enabled, + backend, + model, + cache_dir, + vss_extension, + execution_provider, + } + } +} + +/// Cache dir mặc định: `~/.cache/codegraph/embeddings` (global, cross-project). +fn default_cache_dir() -> Option { + expand_tilde("~/.cache/codegraph/embeddings") +} + +/// Expand `~` thành home dir (best-effort). Trả `Some` nếu không bắt đầu bằng `~`. +fn expand_tilde(path: &str) -> Option { + if !path.starts_with('~') { + return Some(PathBuf::from(path)); + } + let home = std::env::var("HOME") + .ok() + .or_else(|| std::env::var("USERPROFILE").ok())?; + let rest = path.strip_prefix('~').unwrap_or(""); + Some(PathBuf::from(home).join(rest.trim_start_matches('/'))) +} + +/// Suffix file extension của sqlite-vss theo OS (`.dylib` / `.so` / `.dll`). +fn vss_lib_suffix() -> &'static str { + if cfg!(target_os = "macos") { + "dylib" + } else if cfg!(target_os = "windows") { + "dll" + } else { + "so" + } +} + +/// Giải đường dẫn tới 2 extension sqlite-vss (`vector0`, `vss0`). +/// +/// Trả `Some((vector0, vss0))` nếu cả hai file tồn tại, `None` nếu chưa cấu +/// hình hoặc thiếu file. Ưu tiên `vss_extension` trong config; nếu `None` → tự +/// dò `/vss`. Chỉ trả `Some` khi file thực sự tồn tại → caller có +/// thể yên tâm thêm extension vào kết nối SQLite mà không làm hỏng `open` +/// khi thiếu binary. +pub fn resolve_vss_extensions() -> Option<(PathBuf, PathBuf)> { + let cfg = embedding_config(); + let dir = cfg + .vss_extension + .clone() + .or_else(|| cfg.cache_dir.as_ref().map(|c| c.join("vss")))?; + let ext = vss_lib_suffix(); + let v0 = dir.join(format!("vector0.{ext}")); + let vss = dir.join(format!("vss0.{ext}")); + (v0.exists() && vss.exists()).then_some((v0, vss)) +} + +/// Global embedding config — set 1 lần lúc startup (từ project config) qua +/// [`set_embedding_config`]; các `GraphIndex` đọc qua [`embedding_config`]. +/// +/// Quan trọng: [`embedding_config`] KHÔNG tự khởi tạo OnceLock này (chỉ đọc, +/// fallback về [`DEFAULT_EMBEDDING_CONFIG`]) — để tránh race trong test: nếu +/// `embedding_config()` tự `get_or_init(default)` thì config mặc định (tắt) sẽ +/// bị "khoá" trước khi test gọi `set_embedding_config`, làm opt-in bị ignore. +static EMBEDDING_CONFIG: OnceLock = OnceLock::new(); + +/// Config mặc định (embedding TẮT) — lazily init, KHÔNG ảnh hưởng `EMBEDDING_CONFIG`. +static DEFAULT_EMBEDDING_CONFIG: OnceLock = OnceLock::new(); + +/// Áp dụng config embedding (chỉ có tác dụng lần đầu; các lần sau bị bỏ qua). +/// Gọi ở nơi mở index (VD `ExtractConfig::storage_route`). +pub fn set_embedding_config(cfg: EmbeddingConfig) { + EMBEDDING_CONFIG.get_or_init(|| cfg); +} + +/// Đọc config embedding hiện tại (mặc định TẮT nếu chưa set bởi [`set_embedding_config`]). +pub fn embedding_config() -> &'static EmbeddingConfig { + match EMBEDDING_CONFIG.get() { + Some(c) => c, + None => DEFAULT_EMBEDDING_CONFIG.get_or_init(EmbeddingConfig::default), + } +} + +/// Embedding có được kích hoạt không. +/// +/// Chỉ `true` khi `[embedding]` được khai báo tường minh trong config (tức +/// `EmbeddingConfig.enabled`). Khi `false`: `GraphIndex` không chạy embedding, +/// không xây vector index, và semantic search báo lỗi rõ thay vì fallback silent. +pub fn embedding_enabled() -> bool { + embedding_config().enabled +} + +/// Backend dependency-free (fallback): feature-hashing bag-of-words + character +/// n-gram vào vector chiều `dim`, rồi L2-normalize. Deterministic, rất nhanh. +/// +/// Hai symbol chia sẻ nhiều token/substring → vector gần nhau (cosine cao) → +/// KNN trả về gần nhau. "Lexical similarity" chứ không phải semantic sâu, nhưng +/// phục vụ tốt việc "search tên tương tự" và không cần tải model. +pub struct HashingEmbeddings { + dim: usize, +} + +impl HashingEmbeddings { + pub fn new(dim: usize) -> Self { + Self { dim: dim.max(1) } + } + + /// Hash một token → bin [0, dim). + fn bin(&self, token: &str) -> usize { + let mut h = DefaultHasher::new(); + token.hash(&mut h); + (h.finish() as usize) % self.dim + } +} + +impl EmbeddingBackend for HashingEmbeddings { + fn dim(&self) -> usize { + self.dim + } + + fn embed(&self, text: &str) -> Vec { + let mut v = vec![0.0f32; self.dim]; + // Token (alphanumeric) + character trigram → capture cả word và + // substring overlap. + for raw in text.split(|c: char| !c.is_alphanumeric()) { + if raw.is_empty() { + continue; + } + let tok = raw.to_lowercase(); + v[self.bin(&tok)] += 1.0; + let chars: Vec = tok.chars().collect(); + for w in chars.windows(3) { + let trigram: String = w.iter().collect(); + v[self.bin(&trigram)] += 0.5; + } + } + normalize(v) + } +} + +/// Backend fastembed (model ONNX) — chỉ compile khi bật feature `fastembed`. +/// +/// `fastembed::TextEmbedding::embed` yêu cầu `&mut self`, nên giữ model trong +/// `Mutex` (interior mutability) và share một instance process-wide qua +/// `OnceLock` để không tải model (~130MB, cache `cache_dir`) nhiều lần. +#[cfg(feature = "fastembed")] +mod fastembed_backend { + use super::*; + use fastembed::{EmbeddingModel, TextEmbedding, TextInitOptions}; + use parking_lot::Mutex; + use std::sync::Arc; + + /// Process-wide ONNX model, share bởi mọi `GraphIndex`. Lần đầu gọi sẽ tải + /// và cache model; các lần sau reuse instance đã tải. Nếu init lỗi (thiếu + /// mạng / ONNX runtime), lỗi được cache trong `OnceLock` và mọi lần gọi sau + /// đều trả lại lỗi đó (KHÔNG fallback silent sang [`HashingEmbeddings`]). + pub(crate) fn global_model( + model: EmbeddingModel, + cache_dir: Option, + ) -> Result<&'static Arc>, String> { + static MODEL_CELL: OnceLock>, String>> = OnceLock::new(); + MODEL_CELL + .get_or_init(|| { + // Dùng mọi core vật lý — ONNX Runtime sẽ chạy inference SIMD + // (trên macOS là Accelerate/vecLib) đa luồng. Đặt tường minh để + // không bị default lệch trên máy ít core ảo. + let intra = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(4); + let mut opt = TextInitOptions::new(model) + .with_show_download_progress(true) + .with_intra_threads(intra); + if let Some(dir) = cache_dir { + opt = opt.with_cache_dir(dir); + } + // Execution provider (CoreML/Metal) — CHỈ compile khi feature + // `apple-accel` bật (macOS, fastembed). Các type CoreML/Metal EP + // chỉ tồn tại khi `ort` compile với feature tương ứng. Nếu config + // yêu cầu "coreml"/"metal" mà build thiếu `apple-accel` → block này + // bị loại bỏ → chạy CPU (KHÔNG lỗi silent, chỉ không dùng GPU). + #[cfg(all(feature = "fastembed", feature = "apple-accel"))] + { + use ort::execution_providers::CoreML; + // `ort` 2.0.0-rc.13 chỉ expose CoreML EP (Metal EP chưa có + // feature). CoreML trên Apple Silicon đã chạy trên GPU/ANE, + // nên cả "coreml" và "metal" đều dùng CoreML EP. Nếu config + // ghi "metal" → in note rõ thay vì lặng lẹ. + let ep = match embedding_config().execution_provider.as_deref() { + Some("coreml") => Some(CoreML::default().build()), + Some("metal") => { + eprintln!( + "[codegraph-graph] Metal EP is not exposed by ONNX Runtime 2.0.0-rc.13; using CoreML (Apple GPU/ANE) instead" + ); + Some(CoreML::default().build()) + } + _ => None, + }; + if let Some(ep) = ep { + opt = opt.with_execution_providers(vec![ep]); + } + } + TextEmbedding::try_new(opt) + .map(|m| Arc::new(Mutex::new(m))) + .map_err(|e| e.to_string()) + }) + .as_ref() + .map_err(|e| e.clone()) + } + + /// Map tên model thân thiện (hoặc variant name) → `EmbeddingModel`. + pub(crate) fn resolve_model(s: &str) -> EmbeddingModel { + let t = s.trim(); + let by_alias = match t.to_ascii_lowercase().as_str() { + "bge-small-en-v1.5" | "bge-small" => Some(EmbeddingModel::BGESmallENV15), + "bge-base-en-v1.5" | "bge-base" => Some(EmbeddingModel::BGEBaseENV15), + "bge-large-en-v1.5" | "bge-large" => Some(EmbeddingModel::BGELargeENV15), + "all-minilm-l6-v2" | "all-minilm" => Some(EmbeddingModel::AllMiniLML6V2), + "all-mpnet-base-v2" => Some(EmbeddingModel::AllMpnetBaseV2), + "nomic-embed-text-v1.5" | "nomic-embed-text" => Some(EmbeddingModel::NomicEmbedTextV15), + "multilingual-e5-small" => Some(EmbeddingModel::MultilingualE5Small), + _ => None, + }; + if let Some(m) = by_alias { + return m; + } + // Thử variant name thô (VD "BGESmallENV15"). + if let Ok(m) = t.parse::() { + return m; + } + eprintln!( + "[codegraph-graph] unknown embedding model '{s}', falling back to BGE-small-en-v1.5" + ); + EmbeddingModel::BGESmallENV15 + } + + pub struct FastEmbedBackend { + model: Arc>, + dim: usize, + } + + impl FastEmbedBackend { + /// Khởi tạo backend từ [`EmbeddingConfig`], tái sử dụng model đã tải + /// (nếu có). Trả `Err` nếu fastembed không init được (thiếu mạng tải + /// model, thiếu ONNX runtime…). + pub fn try_new(cfg: &EmbeddingConfig) -> Result { + let model = resolve_model(&cfg.model); + let cache = cfg.cache_dir.clone().or_else(default_cache_dir); + let model = global_model(model, cache)?; + // Xác định dim thực tế bằng một embedding probe (robust với mọi model). + let dim = { + let mut g = model.lock(); + g.embed(vec!["__probe__"], None) + .map(|v| v[0].len()) + .map_err(|e| e.to_string())? + }; + Ok(Self { + model: model.clone(), + dim, + }) + } + } + + impl EmbeddingBackend for FastEmbedBackend { + fn dim(&self) -> usize { + self.dim + } + + fn embed(&self, text: &str) -> Vec { + let mut g = self.model.lock(); + let v = g + .embed(vec![text], None) + .expect("fastembed embed failed (model unloaded?)"); + normalize(v.into_iter().next().unwrap()) + } + + /// Batch embedding — fastembed tính toán vector hoá hàng loạt (SIMD/đa + /// luồng qua ONNX Runtime), nhanh hơn rất nhiều so với gọi `embed` tuần + /// tự từng symbol. Truyền toàn bộ chunk làm 1 batch ONNX (`batch_size = + /// texts.len()`) để tận dụng tối đa throughput. + fn embed_batch(&self, texts: &[String]) -> Vec> { + let mut g = self.model.lock(); + let v = g + .embed(texts, Some(texts.len().max(1))) + .expect("fastembed embed_batch failed (model unloaded?)"); + v.into_iter().map(normalize).collect() + } + } +} + +/// Pre-download (warm) một model fastembed vào `cache_dir` (global) — để semantic +/// search chạy offline sau này. Dùng bởi CLI `codegraph embed --model `. +#[cfg(feature = "fastembed")] +pub fn warm_model_cache(model: &str, cache_dir: Option<&std::path::Path>) -> Result<(), String> { + let m = fastembed_backend::resolve_model(model); + let cache = cache_dir.map(PathBuf::from).or_else(default_cache_dir); + fastembed_backend::global_model(m, cache)?; + eprintln!("[codegraph-graph] embedding model '{model}' cached"); + Ok(()) +} + +/// Tạo backend từ config. Trả về `Err` (không fallback silent) khi: +/// +/// - `backend = "fastembed"` mà feature `fastembed` chưa bật compile-time, hoặc +/// - `backend = "fastembed"` mà model tải thất bại (thiếu mạng / ONNX runtime). +/// +/// Caller phải handle error (mở index sẽ báo lỗi rõ ràng nếu model không tải được). +pub fn make_backend() -> Result, String> { + let cfg = embedding_config(); + match cfg.backend { + EmbeddingBackendKind::Hashing => Ok(Box::new(HashingEmbeddings::new(VECTOR_DIM))), + EmbeddingBackendKind::Fastembed => { + #[cfg(feature = "fastembed")] + { + fastembed_backend::FastEmbedBackend::try_new(cfg) + .map(|b| Box::new(b) as Box) + } + #[cfg(not(feature = "fastembed"))] + { + Err( + "embedding backend 'fastembed' requested but crate not compiled with 'fastembed' feature".to_string() + ) + } + } + } +} + +/// Backend mặc định cho `GraphIndex` — CHỈ dùng khi config tắt (`backend = "hashing"` +/// hoặc không set `[embedding]`). Nếu `[embedding].backend = "fastembed"` thì +/// caller PHẢI dùng `make_backend()` để handle error explicit. +pub fn default_backend() -> Box { + Box::new(HashingEmbeddings::new(VECTOR_DIM)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dim_and_normalized() { + let b = HashingEmbeddings::new(64); + let v = b.embed("authenticateUser"); + assert_eq!(v.len(), 64); + let norm = v.iter().map(|x| x * x).sum::().sqrt(); + assert!((norm - 1.0).abs() < 1e-5, "vector phải L2-normalize"); + } + + #[test] + fn similar_text_higher_cosine() { + let b = HashingEmbeddings::new(256); + let a = b.embed("user authentication service"); + let b2 = b.embed("authentication User service"); + let c = b.embed("render frame buffer"); + let dot = |x: &[f32], y: &[f32]| x.iter().zip(y).map(|(p, q)| p * q).sum::(); + let sim_ab = dot(&a, &b2); + let sim_ac = dot(&a, &c); + assert!( + sim_ab > sim_ac, + "hai text gần nhau phải có cosine > text khác biệt ({sim_ab} vs {sim_ac})" + ); + } +} diff --git a/crates/codegraph-graph/src/lib.rs b/crates/codegraph-graph/src/lib.rs index 6880902c9..a008a7add 100644 --- a/crates/codegraph-graph/src/lib.rs +++ b/crates/codegraph-graph/src/lib.rs @@ -34,6 +34,7 @@ //! same-file +3) → `build_edges_from_calls` (edge = chain[position], CallSite + //! var-type alias, gom SaveCallRecords) → files → rebuild engines → bump version. +use crate::embeddings::{EmbeddingBackend, default_backend, embedding_enabled, make_backend}; pub use crate::search::Search; use crate::search::SearchResume; #[cfg(feature = "lmdb")] @@ -45,6 +46,7 @@ pub use crate::storage::postgres::PostgresStorage; #[cfg(feature = "sqlite")] pub use crate::storage::sqlite::SqliteStorage; pub use crate::storage::{InMemoryStorage, Storage, Tx}; +use crate::vector_index::VectorIndex; use codegraph_core::{ CallRecord, CallSite, CallSiteResult, ClassInfo, DependenciesReport, Dependency, EdgeMeta, EffectType, Error, FileInfo, FlowCall, FlowResult, FunctionScope, MemberInfo, ResolveResult, @@ -60,10 +62,12 @@ use tokio::sync::RwLock; #[cfg(feature = "bloom-search")] mod bloom; pub mod diff; +pub mod embeddings; mod radix; mod search; mod shared; mod storage; +pub mod vector_index; pub use shared::SharedGraphIndex; @@ -163,6 +167,17 @@ pub struct GraphIndex { next_id: u64, /// index version (bump mỗi lần ingest — SharedGraphIndex dò stale). version: u64, + /// Embedding backend cho semantic search (KNN/k-means). Chỉ được khởi tạo + /// thực sự khi `[embedding].backend = "fastembed"` (config opt-in); nếu + /// embedding tắt (`embedding_enabled = false`) đây là `HashingEmbeddings` + /// placeholder và KHÔNG bao giờ được gọi để sinh vector. + embedding_backend: Arc, + /// `true` khi semantic search được bật (config `[embedding].backend = "fastembed"`). + /// Khi `false`, vector index rỗng và semantic search báo lỗi rõ ràng. + embedding_enabled: bool, + /// Vector index: symbol id → embedding. Build từ embeddings **persist trong + /// storage** (load khi open; compute+save cho symbol thiếu khi ingest). + vector_index: VectorIndex, } // ── Search resumable (deadline-aware, checkpointable) ── @@ -191,6 +206,14 @@ pub enum SearchCursorPhase { /// Search đã hoàn tất: `collected` + `total` giữ để phân trang tiếp mà /// không quét lại. Không chứa query — `SearchCursor.query` lo phần đó. Paged { collected: Vec, total: usize }, + /// Mode `Semantic`/`Hybrid`: candidate ids đã sort theo relevance (giảm + /// dần) — Phase B lọc `kind` + gom vào `collected` giống `Expand` nhưng + /// duyệt trực tiếp list id (không qua name engine). + Candidates { + ids: Vec, + idx: usize, + collected: Vec, + }, } /// Server-side cursor cho search resumable — validate theo (query, mode, @@ -230,12 +253,47 @@ pub struct Pagination { pub offset: usize, } +/// Text đầu vào cho embedding: gộp tên + signature + doc + annotations — capture +/// cả ý nghĩa lẫn loại của symbol để semantic search hữu dụng. +fn embedding_text(sym: &Symbol) -> String { + let mut parts: Vec<&str> = Vec::new(); + parts.push(&sym.name); + if let Some(sig) = &sym.signature { + parts.push(sig); + } + if let Some(doc) = &sym.doc { + parts.push(doc); + } + if !sym.annotations.is_empty() { + parts.extend(sym.annotations.iter().map(|a| a.name.as_str())); + } + parts.join(" ") +} + +/// Reciprocal Rank Fusion: gộp nhiều list (mỗi list đã sort theo relevance giảm +/// dần) thành một list fused. Điểm mỗi id = Σ 1/(k + rank). `k = 60` chuẩn. +/// Dùng cho `Hybrid` (lexical + vector). +fn rrf_fuse(lists: &[Vec]) -> Vec { + const K: f32 = 60.0; + let mut scores: HashMap = HashMap::new(); + for list in lists { + for (rank, &id) in list.iter().enumerate() { + *scores.entry(id).or_default() += 1.0 / (K + (rank as f32) + 1.0); + } + } + let mut out: Vec<(u64, f32)> = scores.into_iter().collect(); + out.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + out.into_iter().map(|(id, _)| id).collect() +} + impl GraphIndex { - /// Index in-memory (test/dev, không persist). + /// Index in-memory (test/dev, không persist). Embedding mặc định TẮT + /// (config chưa set → `[embedding].backend = "hashing"`), nên không load + /// model. Nếu process đã set config fastembed trước đó mà model lỗi → panic. pub fn in_memory() -> Self { let storage = Arc::new(RwLock::new(InMemoryStorage::default())) as Arc>; - Self::new_with_storage(storage) + Self::new_with_storage(storage).expect("in_memory embedding backend init failed") } /// Mở index từ một backend persistent bằng DSN — rebuild từ entity store. @@ -351,7 +409,7 @@ impl GraphIndex { .await .map_err(serr)?; let storage = Arc::new(RwLock::new(storage)) as Arc>; - let mut idx = Self::new_with_storage(storage); + let mut idx = Self::new_with_storage(storage)?; idx.rebuild().await?; Ok(idx) } @@ -362,7 +420,7 @@ impl GraphIndex { .await .map_err(serr)?; let storage = Arc::new(RwLock::new(storage)) as Arc>; - let mut idx = Self::new_with_storage(storage); + let mut idx = Self::new_with_storage(storage)?; idx.rebuild().await?; Ok(idx) } @@ -393,7 +451,7 @@ impl GraphIndex { .await .map_err(serr)?; let storage = Arc::new(RwLock::new(storage)) as Arc>; - let mut idx = Self::new_with_storage(storage); + let mut idx = Self::new_with_storage(storage)?; idx.rebuild().await?; Ok(idx) } @@ -437,7 +495,7 @@ impl GraphIndex { .map_err(serr)?; let storage = Arc::new(RwLock::new(storage)) as Arc>; - let mut idx = Self::new_with_storage(storage); + let mut idx = Self::new_with_storage(storage)?; idx.rebuild().await?; Ok(idx) } @@ -457,7 +515,7 @@ impl GraphIndex { .map_err(serr)?; let storage = Arc::new(RwLock::new(storage)) as Arc>; - let mut idx = Self::new_with_storage(storage); + let mut idx = Self::new_with_storage(storage)?; idx.rebuild().await?; Ok(idx) } @@ -473,12 +531,22 @@ impl GraphIndex { } } - fn new_with_storage(storage: Arc>) -> Self { + fn new_with_storage(storage: Arc>) -> Result { // Name engine luôn in-memory (như semgraph SearchIndex) — storage riêng // để record id (1..N) không đụng record của chain engine (func ids). let name_storage = Arc::new(RwLock::new(InMemoryStorage::default())) as Arc>; - Self { + // Embedding chỉ bật khi config `[embedding].backend = "fastembed"` (opt-in). + // Nếu bật mà model tải thất bại → lỗi rõ ràng (KHÔNG fallback silent). + let (backend, enabled) = if embedding_enabled() { + let b = make_backend() + .map_err(|e| Error::Db(format!("embedding backend init failed: {e}")))?; + (Arc::from(b), true) + } else { + // Tắt → placeholder (không bao giờ gọi embed; vector index rỗng). + (Arc::from(default_backend()), false) + }; + Ok(Self { chains: Search::new(CHAIN_SHARDING, storage.clone()), names: Search::new(CHAIN_SHARDING, name_storage), storage, @@ -493,7 +561,10 @@ impl GraphIndex { files: Vec::new(), next_id: SYMBOL_BASE, version: 0, - } + embedding_backend: backend, + embedding_enabled: enabled, + vector_index: VectorIndex::new(crate::embeddings::VECTOR_DIM), + }) } // ── Build / rebuild ── @@ -578,6 +649,7 @@ impl GraphIndex { // Engines. self.rebuild_chain_engine(None).await?; self.rebuild_name_engine(None).await?; + self.rebuild_vector_index(None).await?; Ok(()) } @@ -698,6 +770,67 @@ impl GraphIndex { Ok(()) } + /// Rebuild vector index (semantic search) từ symbols hiện tại. + /// + /// - Nếu embedding **TẮT** (`embedding_enabled = false`): vector index để + /// rỗng, semantic search sẽ báo lỗi rõ ràng (không fallback silent). + /// - Nếu bật: ưu tiên load vector đã **persist trong storage** (tái dùng cho + /// KNN/k-means, không re-embed). Chỉ những symbol thiếu vector mới được + /// embed + `save_embedding` (incremental). Chạy sau khi registry + name + /// engine đã sẵn sàng. + async fn rebuild_vector_index(&mut self, progress: Option<&dyn IngestProgress>) -> Result<()> { + if !self.embedding_enabled { + // Embedding tắt → không build vector index. + self.vector_index = VectorIndex::new(crate::embeddings::VECTOR_DIM); + return Ok(()); + } + let dim = self.embedding_backend.dim(); + let mut vi = VectorIndex::new(dim); + // Load các vector đã persist (tái dùng, không re-embed). + let stored = { + let st = self.storage.read().await; + st.load_all_embeddings().await.map_err(serr)? + }; + // Symbol thiếu vector → compute + save. + let mut missing: Vec<(u64, String)> = Vec::new(); + for sym in self.symbols.values() { + match stored.get(&sym.id) { + Some(v) if v.len() == dim => { + vi.insert(sym.id, v.clone()); + } + _ => missing.push((sym.id, embedding_text(sym))), + } + } + if !missing.is_empty() { + if let Some(p) = progress { + p.phase("embed vectors", missing.len()); + } + eprintln!( + "codegraph: embedding {} symbols (batch) — this may take a while for large repos", + missing.len(), + ); + // Batch-embed theo chunk để tận dụng tính toán hàng loạt (fastembed) + // và báo tiến độ từng bước — tránh quét tuần tự chậm + bar đứng im. + let mut st = self.storage.write().await; + let texts: Vec = missing.iter().map(|(_, t)| t.clone()).collect(); + const CHUNK: usize = 512; + let mut base = 0usize; + for chunk in texts.chunks(CHUNK) { + let vecs = self.embedding_backend.embed_batch(chunk); + for (v, (id, _)) in vecs.into_iter().zip(&missing[base..base + chunk.len()]) { + vi.insert(*id, v.clone()); + st.save_embedding(*id, &v).await.map_err(serr)?; + } + base += chunk.len(); + if let Some(p) = progress { + p.advance(chunk.len()); + } + } + } + self.vector_index = vi; + Ok(()) + } + // ── Ingest (full re-index — pipeline 2 phase như semgraph) ── /// Ingest toàn bộ parse results — **full re-index**: xoá dữ liệu cũ, register @@ -816,6 +949,7 @@ impl GraphIndex { // ── Phase 5: engines + version bump ── self.rebuild_chain_engine(p).await?; self.rebuild_name_engine(p).await?; + self.rebuild_vector_index(p).await?; self.version += 1; { let mut st = self.storage.write().await; @@ -1193,77 +1327,9 @@ impl GraphIndex { out } - // ── Queries ── - - /// Tìm symbol theo tên (substring, case-insensitive) qua name engine; lọc - /// theo kind nếu `Some`. `limit = 0` = không giới hạn (vẫn chặn bởi engine). - pub async fn search_symbol( - &self, - query: &str, - kind: Option, - limit: usize, - ) -> Result> { - self.search_symbol_filtered(query, limit, |s| kind.is_none() || s.kind == kind.unwrap()) - .await - } - - /// Như `search_symbol` nhưng chấp nhận NHIỀU kind — dùng cho sandbox (entry - /// có thể là `Function` free function (Rust/Go/...) hoặc `Method` (Java/...)). - pub async fn search_symbol_kinds( - &self, - query: &str, - kinds: &[SymbolKind], - limit: usize, - ) -> Result> { - self.search_symbol_filtered(query, limit, |s| kinds.contains(&s.kind)) - .await - } - - async fn search_symbol_filtered( - &self, - query: &str, - limit: usize, - filter: F, - ) -> Result> - where - F: Fn(&Symbol) -> bool, - { - let q = query.to_lowercase(); - let hits = match self.names.search(q.as_bytes(), None).await { - Ok(h) => h, - Err(_) => return Ok(Vec::new()), - }; - let limit = if limit == 0 { usize::MAX } else { limit }; - let mut out = Vec::new(); - let mut seen = HashSet::new(); - for (record, _) in hits { - if record == 0 { - continue; - } - let Some(name) = self.name_records.get(record - 1) else { - continue; - }; - let Some(ids) = self.name_index.get(name) else { - continue; - }; - for &id in ids { - if !seen.insert(id) { - continue; - } - let Some(s) = self.symbols.get(&id) else { - continue; - }; - if !filter(s) { - continue; - } - out.push(s.clone()); - if out.len() >= limit { - return Ok(out); - } - } - } - Ok(out) - } + // ── Queries (mọi search đều qua `search_symbol_paged_resumable` — resumable, + // deadline-aware; các hàm tiện ích còn lại chỉ wrap nó, không có implementation + // song song) ── /// Symbol theo id. pub fn symbol_by_id(&self, id: u64) -> Option { @@ -1909,30 +1975,8 @@ impl GraphIndex { } } - /// Search symbol nâng cao: lọc theo kind + match mode (contains/prefix/ - /// suffix/exact) + phân trang. Trả về (page, total) — total là số khớp - /// trước phân trang, page sort theo (name, id) cho pagination ổn định. - pub async fn search_symbol_paged( - &self, - query: &str, - kind: Option, - mode: SymbolMatch, - limit: usize, - offset: usize, - ) -> Result<(Vec, usize)> { - let out = self - .search_symbol_paged_resumable( - query, - kind, - mode, - Pagination { limit, offset }, - None, - None, - ) - .await?; - Ok((out.page, out.total)) - } - /// Phiên bản resumable + deadline-aware của [`search_symbol_paged`]: ngắt + /// Phiên bản resumable + deadline-aware của search symbol nâng cao (kind + /// filter + match mode contains/prefix/suffix/exact + phân trang): ngắt /// giữa chừng khi `deadline` hết hạn, trả `PagedSearchOutcome { timed_out: /// true, cursor: Some(phase dở) }` — caller gọi lại với `resume = /// Some(cursor)` để tiếp tục từ đúng vị trí (không lặp phần đã duyệt). @@ -1946,6 +1990,18 @@ impl GraphIndex { /// - Hoàn tất + còn page sau → `cursor = Some(Paged)` để phân trang tiếp /// không cần quét lại. /// + /// KNN: ưu tiên backend-native (SQLite + sqlite-vss `vss0` HNSW ANN) nếu + /// khả dụng, ngược lại brute-force in-memory `VectorIndex` (đúng cho mọi + /// backend). Trả `Vec<(symbol_id, sim)>` với `sim` cao = gần hơn. + async fn knn_hits(&self, qvec: &[f32], k: usize) -> Vec<(u64, f32)> { + if let Ok(guard) = self.storage.try_read() + && let Ok(Some(hits)) = guard.knn(qvec, k).await + { + return hits; + } + self.vector_index.knn(qvec, k) + } + /// `resume` phải khớp (query, mode, kind) — sai → `InvalidArgument`. pub async fn search_symbol_paged_resumable( &self, @@ -1968,86 +2024,163 @@ impl GraphIndex { )); } + // Semantic/Hybrid cần embedding bật (config `[embedding].backend = "fastembed"`). + // Nếu tắt → lỗi rõ ràng (KHÔNG fallback silent sang lexical/hashing). + if matches!(mode, SymbolMatch::Semantic | SymbolMatch::Hybrid) && !self.embedding_enabled { + return Err(Error::Invalid( + "semantic/hybrid search requires embedding; enable `[embedding] backend = \"fastembed\"` in config".into(), + )); + } + // ── Khôi phục / khởi tạo phase ── let (mut phase, mut timed_out) = match resume.map(|c| c.phase) { Some(p) => (p, false), None => ( match mode { SymbolMatch::Contains => SearchCursorPhase::Engine(SearchResume::default()), + // Semantic/Hybrid dùng placeholder — Phase A ghi đè thành + // `Candidates` (tính KNN/RRF). + SymbolMatch::Semantic | SymbolMatch::Hybrid => { + SearchCursorPhase::Engine(SearchResume::default()) + } _ => SearchCursorPhase::ScanNames { name_pos: 0 }, }, false, ), }; - // ── Phase A: sinh danh sách tên khớp (sort) ── - match &mut phase { - SearchCursorPhase::Engine(sr) => { - let page = self - .names - .search_resumable(q.as_bytes(), None, Some(sr.clone()), deadline) - .await?; - if page.timed_out { - phase = SearchCursorPhase::Engine(page.resume.unwrap_or_default()); - timed_out = true; - } else { - // record → tên, sort → Expand. - let mut names: Vec = page - .record_ids - .iter() - .filter_map(|&r| { - if r == 0 { - return None; - } - self.name_records.get(r - 1).cloned() - }) - .collect(); - names.sort(); - phase = SearchCursorPhase::Expand { - names, - name_idx: 0, - id_idx: 0, - collected: Vec::new(), - }; + // ── Phase A: sinh danh sách candidate (sort) — dispatch theo mode ── + match mode { + SymbolMatch::Contains => { + if let SearchCursorPhase::Engine(sr) = &mut phase { + let page = self + .names + .search_resumable(q.as_bytes(), None, Some(sr.clone()), deadline) + .await?; + if page.timed_out { + phase = SearchCursorPhase::Engine(page.resume.unwrap_or_default()); + timed_out = true; + } else { + // record → tên, sort → Expand. + let mut names: Vec = page + .record_ids + .iter() + .filter_map(|&r| { + if r == 0 { + return None; + } + self.name_records.get(r - 1).cloned() + }) + .collect(); + names.sort(); + phase = SearchCursorPhase::Expand { + names, + name_idx: 0, + id_idx: 0, + collected: Vec::new(), + }; + } } } - SearchCursorPhase::ScanNames { name_pos } => { - let mut matched: Vec = Vec::new(); - let mut pos = *name_pos; - loop { - if let Some(dl) = deadline - && Instant::now() >= dl - { - phase = SearchCursorPhase::ScanNames { name_pos: pos }; - timed_out = true; - break; + SymbolMatch::Prefix | SymbolMatch::Suffix | SymbolMatch::Exact => { + if let SearchCursorPhase::ScanNames { name_pos } = &mut phase { + let mut matched: Vec = Vec::new(); + let mut pos = *name_pos; + loop { + if let Some(dl) = deadline + && Instant::now() >= dl + { + phase = SearchCursorPhase::ScanNames { name_pos: pos }; + timed_out = true; + break; + } + if pos >= self.sorted_name_keys.len() { + break; + } + let name = &self.sorted_name_keys[pos]; + let ok = match mode { + SymbolMatch::Prefix => name.starts_with(&q), + SymbolMatch::Suffix => name.ends_with(&q), + SymbolMatch::Exact => name == &q, + _ => false, + }; + if ok { + matched.push(name.clone()); + } + pos += 1; } - if pos >= self.sorted_name_keys.len() { - break; + if !timed_out { + phase = SearchCursorPhase::Expand { + names: matched, + name_idx: 0, + id_idx: 0, + collected: Vec::new(), + }; } - let name = &self.sorted_name_keys[pos]; - let ok = match mode { - SymbolMatch::Prefix => name.starts_with(&q), - SymbolMatch::Suffix => name.ends_with(&q), - SymbolMatch::Exact => name == &q, - _ => false, + } + } + SymbolMatch::Semantic => { + if !matches!( + phase, + SearchCursorPhase::Candidates { .. } | SearchCursorPhase::Paged { .. } + ) { + let qvec = self.embedding_backend.embed(&q); + let k = (pagination.limit * 4).max(32); + let ids: Vec = self + .knn_hits(&qvec, k) + .await + .into_iter() + .map(|(id, _)| id) + .collect(); + phase = SearchCursorPhase::Candidates { + ids, + idx: 0, + collected: Vec::new(), }; - if ok { - matched.push(name.clone()); - } - pos += 1; } - if !timed_out { - phase = SearchCursorPhase::Expand { - names: matched, - name_idx: 0, - id_idx: 0, + } + SymbolMatch::Hybrid => { + if !matches!( + phase, + SearchCursorPhase::Candidates { .. } | SearchCursorPhase::Paged { .. } + ) { + // Lexical (Contains) trên name engine. + let lex: Vec = self + .names + .search(q.as_bytes(), None) + .await + .map(|recs| { + let mut out = Vec::new(); + for (r, _) in recs { + if r == 0 { + continue; + } + if let Some(name) = self.name_records.get(r - 1) + && let Some(ids) = self.name_index.get(name) + { + out.extend_from_slice(ids); + } + } + out + }) + .unwrap_or_default(); + // Vector (Semantic). + let qvec = self.embedding_backend.embed(&q); + let k = (pagination.limit * 4).max(32); + let vec_ids: Vec = self + .knn_hits(&qvec, k) + .await + .into_iter() + .map(|(id, _)| id) + .collect(); + let ids = rrf_fuse(&[lex, vec_ids]); + phase = SearchCursorPhase::Candidates { + ids, + idx: 0, collected: Vec::new(), }; } } - // Phase A xong rồi (timed out ở phase B trước) — không làm gì. - _ => {} } // ── Phase B: stream ids theo tên đã sort → collected ── @@ -2092,12 +2225,43 @@ impl GraphIndex { } } + // ── Phase B (Candidates): stream id vector (Semantic/Hybrid) → collected ── + if !timed_out + && let SearchCursorPhase::Candidates { + ids, + idx, + collected, + } = &mut phase + { + loop { + if let Some(dl) = deadline + && Instant::now() >= dl + { + timed_out = true; + break; + } + if *idx >= ids.len() { + break; + } + let id = ids[*idx]; + *idx += 1; + if let Some(k) = kind { + if self.symbols.get(&id).is_some_and(|s| s.kind == k) { + collected.push(id); + } + } else { + collected.push(id); + } + } + } + // ── Trang kết quả + cursor ── if timed_out { let progress = match &phase { SearchCursorPhase::Engine(sr) => sr.record_ids.len(), SearchCursorPhase::ScanNames { name_pos } => *name_pos, SearchCursorPhase::Expand { collected, .. } => collected.len(), + SearchCursorPhase::Candidates { collected, .. } => collected.len(), SearchCursorPhase::Paged { .. } => 0, }; return Ok(PagedSearchOutcome { @@ -2114,12 +2278,16 @@ impl GraphIndex { }); } - // Hoàn tất: lấy collected + total từ Expand, hoặc dùng thẳng từ Paged. + // Hoàn tất: lấy collected + total từ Expand/Candidates, hoặc dùng thẳng từ Paged. let (collected, total) = match &phase { SearchCursorPhase::Expand { collected, .. } => { let total = collected.len(); (collected.clone(), total) } + SearchCursorPhase::Candidates { collected, .. } => { + let total = collected.len(); + (collected.clone(), total) + } SearchCursorPhase::Paged { collected, total } => (collected.clone(), *total), // Không thể tới đây khi chưa hoàn tất phase A. _ => (Vec::new(), 0), @@ -2153,27 +2321,6 @@ impl GraphIndex { }) } - /// Resumable + deadline-aware của `search_symbol_filtered` (mode Contains, - /// không lọc kind) — nền cho `codegraph_search`. `limit` chặn số symbol - /// trả về; kết quả sort theo (name, id). - pub async fn search_symbol_resumable( - &self, - query: &str, - limit: usize, - resume: Option, - deadline: Option, - ) -> Result { - self.search_symbol_paged_resumable( - query, - None, - SymbolMatch::Contains, - Pagination { limit, offset: 0 }, - resume, - deadline, - ) - .await - } - /// Số liệu tổng hợp. pub fn stats(&self) -> SemgraphStats { SemgraphStats { @@ -2264,11 +2411,41 @@ mod tests { ); idx.ingest(&[r]).await.unwrap(); - // search_symbol (substring, case-insensitive). - let hits = idx.search_symbol("b", None, 10).await.unwrap(); + // search_symbol (substring, case-insensitive) — qua resumable. + let hits = idx + .search_symbol_paged_resumable( + "b", + None, + SymbolMatch::Contains, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) + .await + .unwrap() + .page; assert_eq!(hits.len(), 1); assert_eq!(hits[0].name, "b"); - assert!(idx.search_symbol("zzz", None, 10).await.unwrap().is_empty()); + assert!( + idx.search_symbol_paged_resumable( + "zzz", + None, + SymbolMatch::Contains, + Pagination { + limit: 10, + offset: 0 + }, + None, + None, + ) + .await + .unwrap() + .page + .is_empty() + ); // callees của a = [b]; của b = [c]. let cees = idx.callees(SYMBOL_BASE).await.unwrap(); @@ -2450,8 +2627,22 @@ mod tests { assert!(!res2.ambiguous); assert_eq!(res2.symbol.unwrap().id, SYMBOL_BASE); - // search_symbol mở rộng cả 2 symbol trùng tên. - let hits = idx.search_symbol("process", None, 10).await.unwrap(); + // search_symbol mở rộng cả 2 symbol trùng tên — qua resumable. + let hits = idx + .search_symbol_paged_resumable( + "process", + None, + SymbolMatch::Contains, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) + .await + .unwrap() + .page; assert_eq!(hits.len(), 2); } @@ -2572,6 +2763,70 @@ mod tests { assert_eq!(idx.symbol_by_id(SYMBOL_BASE + 1).unwrap().file, "b.ts"); } + /// Embedding là OPT-IN + PERSIST: bật backend "hashing", ingest (lưu vector + /// vào sqlite), reopen → vector index được rebuild TỪ embeddings đã lưu + /// (không re-embed), semantic search vẫn chạy. + #[tokio::test] + async fn sqlite_embeddings_persist_and_reopen() { + crate::embeddings::set_embedding_config(crate::embeddings::EmbeddingConfig::from_raw( + Some("hashing"), + None, + None, + None, + None, + )); + let dir = tempfile::tempdir().unwrap(); + let path = format!("sqlite://{}/db.sqlite", dir.path().to_string_lossy()); + let r = result( + "a.ts", + vec![ + sym("auth.rs", "authenticate_user", SYMBOL_BASE), + sym("db.rs", "query_database", SYMBOL_BASE + 1), + ], + HashMap::new(), + vec![], + ); + { + let mut idx = GraphIndex::open(&path).await.unwrap(); + idx.ingest(&[r]).await.unwrap(); + // Semantic chạy được (embedding enabled). + let sem = idx + .search_symbol_paged_resumable( + "authenticte", + None, + SymbolMatch::Semantic, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) + .await + .unwrap() + .page; + assert_eq!(sem[0].name, "authenticate_user"); + } + // Reopen — vector index phải được load từ embeddings đã persist. + let idx = GraphIndex::open(&path).await.unwrap(); + let sem = idx + .search_symbol_paged_resumable( + "authenticte", + None, + SymbolMatch::Semantic, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) + .await + .unwrap() + .page; + assert_eq!(sem[0].name, "authenticate_user"); + } + /// Ingest 2 lần = full re-index — dữ liệu cũ biến mất, id gán lại từ đầu. #[tokio::test] async fn ingest_twice_is_full_reindex() { @@ -2695,41 +2950,83 @@ mod tests { let (_, total, _) = idx.search_by_annotation("controller", Some(SymbolKind::Class), 0, 1); assert_eq!(total, 1, "kind filter loại bỏ match không đúng kind"); - // search_symbol_paged — prefix/suffix/exact + kind filter. - let (hits, total) = idx - .search_symbol_paged("order", Some(SymbolKind::Class), SymbolMatch::Prefix, 10, 0) + // search_symbol_paged — prefix/suffix/exact + kind filter (resumable). + let out = idx + .search_symbol_paged_resumable( + "order", + Some(SymbolKind::Class), + SymbolMatch::Prefix, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) .await .unwrap(); + let hits = out.page; + let total = out.total; assert_eq!( total, 2, "OrderService + OrderController khớp prefix 'order' + kind class" ); assert_eq!(hits[0].name, "OrderController"); assert_eq!(hits[1].name, "OrderService"); - let (hits, total) = idx - .search_symbol_paged( + let out = idx + .search_symbol_paged_resumable( "service", Some(SymbolKind::Class), SymbolMatch::Suffix, - 10, - 0, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, ) .await .unwrap(); + let hits = out.page; + let total = out.total; assert_eq!(total, 1); assert_eq!(hits[0].name, "OrderService"); - let (hits, total) = idx - .search_symbol_paged("validate", None, SymbolMatch::Exact, 10, 0) + let out = idx + .search_symbol_paged_resumable( + "validate", + None, + SymbolMatch::Exact, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) .await .unwrap(); + let hits = out.page; + let total = out.total; assert_eq!(total, 1); assert_eq!(hits[0].name, "validate"); // contains + pagination. Sort theo tên lowercase (nhất quán với search // case-insensitive): "getorders" đứng trước "order*". - let (page0, total) = idx - .search_symbol_paged("order", None, SymbolMatch::Contains, 2, 0) + let out = idx + .search_symbol_paged_resumable( + "order", + None, + SymbolMatch::Contains, + Pagination { + limit: 2, + offset: 0, + }, + None, + None, + ) .await .unwrap(); + let page0 = out.page; + let total = out.total; assert_eq!( total, 4, "OrderService, OrderController, OrderRepository + getOrders" @@ -2737,10 +3034,21 @@ mod tests { assert_eq!(page0.len(), 2); assert_eq!(page0[0].name, "getOrders"); assert_eq!(page0[1].name, "OrderController"); - let (page1, _) = idx - .search_symbol_paged("order", None, SymbolMatch::Contains, 2, 2) + let out = idx + .search_symbol_paged_resumable( + "order", + None, + SymbolMatch::Contains, + Pagination { + limit: 2, + offset: 2, + }, + None, + None, + ) .await .unwrap(); + let page1 = out.page; assert_eq!(page1.len(), 2); assert_eq!(page1[0].name, "OrderRepository"); assert_eq!(page1[1].name, "OrderService"); @@ -2853,10 +3161,19 @@ mod tests { ), ]; for (q, kind, mode, limit, offset) in cases { - let (direct_page, direct_total) = idx - .search_symbol_paged(q, kind, mode, limit, offset) + let direct = idx + .search_symbol_paged_resumable( + q, + kind, + mode, + Pagination { limit, offset }, + None, + None, + ) .await .unwrap(); + let direct_page = direct.page; + let direct_total = direct.total; let chained = chained(&idx, q, kind, mode, limit, offset).await; assert_eq!( chained.total, direct_total, @@ -2892,10 +3209,22 @@ mod tests { } idx.ingest(&results).await.unwrap(); - let (direct_page, direct_total) = idx - .search_symbol_paged("order", None, SymbolMatch::Contains, 10, 0) + let direct = idx + .search_symbol_paged_resumable( + "order", + None, + SymbolMatch::Contains, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) .await .unwrap(); + let direct_page = direct.page; + let direct_total = direct.total; assert_eq!(direct_total, 4000); // Call đầu deadline hết hạn → chắc chắn timed_out (tạo checkpoint). @@ -3011,4 +3340,67 @@ mod tests { assert!(external_names.contains(&"fmt")); assert!(external_names.contains(&"requests")); } + + #[tokio::test] + async fn semantic_and_hybrid_search() { + // Embedding là OPT-IN — test này bật tường minh backend "hashing" + // (dependency-free, không tải model) để semantic/hybrid search chạy. + crate::embeddings::set_embedding_config(crate::embeddings::EmbeddingConfig::from_raw( + Some("hashing"), + None, + None, + None, + None, + )); + let mut idx = GraphIndex::in_memory(); + // Tên gần giống nhau → hashing embeddings sinh vector tương tự → + // KNN cosine tìm được dù query sai chính tả. + let syms = vec![ + sym("auth.rs", "authenticate_user", SYMBOL_BASE), + sym("auth.rs", "authorize_request", SYMBOL_BASE + 1), + sym("db.rs", "query_database", SYMBOL_BASE + 2), + sym("db.rs", "parse_config", SYMBOL_BASE + 3), + ]; + let r = result("f.rs", syms, HashMap::new(), vec![]); + idx.ingest(&[r]).await.unwrap(); + + // Semantic: query lệch chính tả "authenticte" vẫn phải rank + // authenticate_user lên đầu (KNN cosine trên embedding). + let sem = idx + .search_symbol_paged_resumable( + "authenticte", + None, + SymbolMatch::Semantic, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) + .await + .unwrap() + .page; + assert!(!sem.is_empty(), "semantic search must return candidates"); + assert_eq!(sem[0].name, "authenticate_user"); + + // Hybrid: "auth" (lexical) + vector → vẫn phải có authenticate_user. + let hyb = idx + .search_symbol_paged_resumable( + "auth", + None, + SymbolMatch::Hybrid, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) + .await + .unwrap() + .page; + assert!(!hyb.is_empty()); + assert!(hyb.iter().any(|s| s.name == "authenticate_user")); + } } diff --git a/crates/codegraph-graph/src/storage.rs b/crates/codegraph-graph/src/storage.rs index eeeaac0c0..13ed55b16 100644 --- a/crates/codegraph-graph/src/storage.rs +++ b/crates/codegraph-graph/src/storage.rs @@ -19,6 +19,29 @@ use codegraph_core::{FileInfo, Symbol}; #[cfg(feature = "sqlite")] pub mod sqlite; +/// Mã hoá vector f32 thành BLOB little-endian (4 byte/phần tử) — chia sẻ cho +/// mọi backend persist (sqlite/lmdb/rdbms/redis) để lưu embedding vào storage. +pub(crate) fn encode_vector(v: &[f32]) -> Vec { + let mut out = Vec::with_capacity(v.len() * 4); + for x in v { + out.extend_from_slice(&x.to_le_bytes()); + } + out +} + +/// Giải mã BLOB little-endian thành vector f32. Trả `None` nếu độ dài không +/// chia hết cho 4 (corrupt). +pub(crate) fn decode_vector(b: &[u8]) -> Option> { + if !b.len().is_multiple_of(4) { + return None; + } + let mut out = Vec::with_capacity(b.len() / 4); + for chunk in b.chunks_exact(4) { + out.push(f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])); + } + Some(out) +} + #[cfg(feature = "redis")] pub mod redis; @@ -326,6 +349,34 @@ pub trait Storage: Send + Sync { Ok(()) } + // ── Embeddings (vector per symbol id) ── + /// Lưu vector embedding cho một symbol (keyed theo symbol id). Vector đã + /// L2-normalize (cosine = dot product). Mặc định: no-op. + async fn save_embedding(&mut self, _symbol_id: u64, _vector: &[f32]) -> Result<()> { + Ok(()) + } + /// Đọc vector embedding của symbol — `None` nếu chưa có. Mặc định: `None`. + async fn load_embedding(&self, _symbol_id: u64) -> Result>> { + Ok(None) + } + /// Đọc toàn bộ embeddings (symbol_id → vector) — rebuild VectorIndex khi + /// open. Mặc định: rỗng. + async fn load_all_embeddings(&self) -> Result>> { + Ok(HashMap::new()) + } + /// Xoá toàn bộ embeddings — dùng khi full re-index. Mặc định: no-op. + async fn clear_embeddings(&mut self) -> Result<()> { + Ok(()) + } + /// KNN backend-native (SQLite + sqlite-vss). Trả `Some(hits)` nếu backend + /// hỗ trợ ANN, `None` để caller fallback sang `VectorIndex` in-memory + /// (brute-force, đúng cho mọi backend). `hits` = `Vec<(symbol_id, sim)>` + /// với `sim` cao = gần hơn (đã đảo dấu distance để đồng nhất với + /// `VectorIndex::knn`). Mặc định: `None` (không backend-native). + async fn knn(&self, _query_vec: &[f32], _k: usize) -> Result>> { + Ok(None) + } + // ── Transaction ── /// Bắt đầu một transaction (sync, không await — đúng theo cách radix gọi). /// Buffer ops; mọi thay đổi chỉ lộ ra khi `commit`. @@ -370,6 +421,8 @@ struct MemoryData { files: HashMap, /// index version. version: u64, + /// symbol id → embedding vector (L2-normalized f32). + embeddings: HashMap>, } /// In-memory radix storage. Thread-safe: toàn bộ state nằm sau 1 RwLock; @@ -401,6 +454,7 @@ impl InMemoryStorage { call_names: HashMap::new(), files: HashMap::new(), version: 0, + embeddings: HashMap::new(), })), next_id: Arc::new(AtomicUsize::new(1)), } @@ -847,6 +901,41 @@ impl Storage for InMemoryStorage { d.call_names.clear(); d.files.clear(); d.version = 0; + d.embeddings.clear(); + Ok(()) + } + + async fn save_embedding(&mut self, symbol_id: u64, vector: &[f32]) -> Result<()> { + let mut d = self + .data + .write() + .map_err(|_| StorageError::Internal("poison".into()))?; + d.embeddings.insert(symbol_id, vector.to_vec()); + Ok(()) + } + + async fn load_embedding(&self, symbol_id: u64) -> Result>> { + let d = self + .data + .read() + .map_err(|_| StorageError::Internal("poison".into()))?; + Ok(d.embeddings.get(&symbol_id).cloned()) + } + + async fn load_all_embeddings(&self) -> Result>> { + let d = self + .data + .read() + .map_err(|_| StorageError::Internal("poison".into()))?; + Ok(d.embeddings.clone()) + } + + async fn clear_embeddings(&mut self) -> Result<()> { + let mut d = self + .data + .write() + .map_err(|_| StorageError::Internal("poison".into()))?; + d.embeddings.clear(); Ok(()) } diff --git a/crates/codegraph-graph/src/storage/lmdb.rs b/crates/codegraph-graph/src/storage/lmdb.rs index 686264d32..cfc74f231 100644 --- a/crates/codegraph-graph/src/storage/lmdb.rs +++ b/crates/codegraph-graph/src/storage/lmdb.rs @@ -23,7 +23,10 @@ use codegraph_core::{FileInfo, Symbol}; use lmdb::EnvironmentFlags; use lmdb::{Cursor, Database, DatabaseFlags, Environment, Transaction, WriteFlags}; -use super::{EMPTY, Result, Storage, StorageError, Tx, TxOp, decode_chain, encode_chain}; +use super::{ + EMPTY, Result, Storage, StorageError, Tx, TxOp, decode_chain, decode_vector, encode_chain, + encode_vector, +}; /// Map lỗi LMDB → `StorageError`. fn e(err: impl std::fmt::Display) -> StorageError { @@ -150,6 +153,7 @@ const D_CALL_RECORDS: &str = "sg_call_records"; const D_CALL_NAMES: &str = "sg_call_names"; const D_FILES: &str = "sg_files"; const D_VERSION: &str = "sg_meta"; +const D_EMBEDDINGS: &str = "sg_embeddings"; /// Key duy nhất cho các "row đơn" (counter / next_id / version) — mỗi DBI chỉ có 1 row. const KEY_ONE: [u8; 8] = [0u8; 8]; @@ -245,6 +249,7 @@ pub struct LmdbStorage { call_names: Database, files: Database, version: Database, + embeddings: Database, } impl LmdbStorage { @@ -310,6 +315,9 @@ impl LmdbStorage { let version = env .create_db(Some(D_VERSION), DatabaseFlags::empty()) .map_err(e)?; + let embeddings = env + .create_db(Some(D_EMBEDDINGS), DatabaseFlags::empty()) + .map_err(e)?; Ok(Self { env, nodes, @@ -330,6 +338,7 @@ impl LmdbStorage { call_names, files, version, + embeddings, }) } @@ -579,6 +588,49 @@ impl Storage for LmdbStorage { Ok(out) } + async fn save_embedding(&mut self, symbol_id: u64, vector: &[f32]) -> Result<()> { + let mut tx = self.env.begin_rw_txn().map_err(e)?; + tx.put( + self.embeddings, + &ku64(symbol_id), + &encode_vector(vector), + WriteFlags::empty(), + ) + .map_err(e)?; + tx.commit().map_err(e)?; + Ok(()) + } + + async fn load_embedding(&self, symbol_id: u64) -> Result>> { + let tx = self.env.begin_ro_txn().map_err(e)?; + Ok(self + .get_opt(&tx, self.embeddings, &ku64(symbol_id))? + .and_then(decode_vector)) + } + + async fn load_all_embeddings(&self) -> Result>> { + let tx = self.env.begin_ro_txn().map_err(e)?; + let mut cur = tx.open_ro_cursor(self.embeddings).map_err(e)?; + let mut out = HashMap::new(); + for item in cur.iter() { + let (k, v) = item.map_err(e)?; + if k.len() == 8 { + let id = de_u64(k); + if let Some(vec) = decode_vector(v) { + out.insert(id, vec); + } + } + } + Ok(out) + } + + async fn clear_embeddings(&mut self) -> Result<()> { + let mut tx = self.env.begin_rw_txn().map_err(e)?; + tx.clear_db(self.embeddings).map_err(e)?; + tx.commit().map_err(e)?; + Ok(()) + } + async fn save_next_id(&mut self, next: u64) -> Result<()> { let mut tx = self.env.begin_rw_txn().map_err(e)?; tx.put(self.next_id, &KEY_ONE, &ku64(next), WriteFlags::empty()) @@ -709,7 +761,13 @@ impl Storage for LmdbStorage { async fn clear_entities(&mut self) -> Result<()> { let mut tx = self.env.begin_rw_txn().map_err(e)?; - for db in [self.symbols, self.call_records, self.call_names, self.files] { + for db in [ + self.symbols, + self.call_records, + self.call_names, + self.files, + self.embeddings, + ] { tx.clear_db(db).map_err(e)?; } tx.put(self.next_id, &KEY_ONE, &ku64(100), WriteFlags::empty()) @@ -981,6 +1039,29 @@ mod tests { assert_eq!(record, 42); } + #[tokio::test] + async fn test_embeddings_roundtrip() { + let (_d, path) = tmp_path(); + let mut s = LmdbStorage::open(&path).await.unwrap(); + let v1 = vec![0.1f32, 0.2, 0.3, -0.4]; + let v2 = vec![1.0f32, -1.0, 0.0, 0.5]; + s.save_embedding(100, &v1).await.unwrap(); + s.save_embedding(101, &v2).await.unwrap(); + // upsert overwrite cho 100. + s.save_embedding(100, &v2).await.unwrap(); + + let all = s.load_all_embeddings().await.unwrap(); + assert_eq!(all.len(), 2); + assert_eq!(all.get(&100).unwrap(), &v2); + assert_eq!(all.get(&101).unwrap(), &v2); + assert_eq!(s.load_embedding(100).await.unwrap().unwrap(), v2); + assert_eq!(s.load_embedding(101).await.unwrap().unwrap(), v2); + + s.clear_embeddings().await.unwrap(); + assert!(s.load_all_embeddings().await.unwrap().is_empty()); + assert_eq!(s.load_embedding(101).await.unwrap(), None); + } + /// Node trong tx chưa lộ ra reader cho tới `commit`. #[tokio::test] async fn test_tx_atomic() { diff --git a/crates/codegraph-graph/src/storage/mysql.rs b/crates/codegraph-graph/src/storage/mysql.rs index e5b3d40a7..296da3793 100644 --- a/crates/codegraph-graph/src/storage/mysql.rs +++ b/crates/codegraph-graph/src/storage/mysql.rs @@ -1,4 +1,8 @@ -use super::{Result, Storage, StorageError, Tx, decode_chain, encode_chain}; +use std::collections::HashMap; + +use super::{ + Result, Storage, StorageError, Tx, decode_chain, decode_vector, encode_chain, encode_vector, +}; use async_trait::async_trait; use codegraph_core::{Annotation, FileInfo, ScopeLevel, Symbol, SymbolKind}; use sqlx::mysql::{MySqlPoolOptions, MySqlRow}; @@ -59,6 +63,19 @@ impl MySqlStorage { .execute(&self.pool) .await .map_err(db_err)?; + // Embeddings: (repo_id, symbol_id) → vector BLOB. Tạo bảng nếu chưa có + // (idempotent) để không bắt buộc chạy migration thủ công cho tính năng này. + sqlx::query( + "CREATE TABLE IF NOT EXISTS sg_embeddings ( + repo_id BIGINT NOT NULL, + symbol_id BIGINT NOT NULL, + vector LONGBLOB NOT NULL, + PRIMARY KEY (repo_id, symbol_id) + )", + ) + .execute(&self.pool) + .await + .map_err(db_err)?; Ok(()) } @@ -467,6 +484,53 @@ impl Storage for MySqlStorage { rows.iter().map(row_to_symbol).collect() } + async fn save_embedding(&mut self, symbol_id: u64, vector: &[f32]) -> Result<()> { + sqlx::query( + "INSERT INTO sg_embeddings (repo_id, symbol_id, vector) VALUES (?, ?, ?) \ + ON DUPLICATE KEY UPDATE vector = VALUES(vector)", + ) + .bind(self.repo_id as i64) + .bind(symbol_id as i64) + .bind(encode_vector(vector)) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn load_embedding(&self, symbol_id: u64) -> Result>> { + let row: Option<(Vec,)> = + sqlx::query_as("SELECT vector FROM sg_embeddings WHERE repo_id = ? AND symbol_id = ?") + .bind(self.repo_id as i64) + .bind(symbol_id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.and_then(|(b,)| decode_vector(&b))) + } + + async fn load_all_embeddings(&self) -> Result>> { + let rows: Vec<(i64, Vec)> = + sqlx::query_as("SELECT symbol_id, vector FROM sg_embeddings WHERE repo_id = ?") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + Ok(rows + .into_iter() + .filter_map(|(id, b)| decode_vector(&b).map(|v| (id as u64, v))) + .collect()) + } + + async fn clear_embeddings(&mut self) -> Result<()> { + sqlx::query("DELETE FROM sg_embeddings WHERE repo_id = ?") + .bind(self.repo_id as i64) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + async fn save_next_id(&mut self, next: u64) -> Result<()> { sqlx::query( "INSERT INTO sg_next_id (repo_id, next) VALUES (?, ?) \ @@ -649,6 +713,7 @@ impl Storage for MySqlStorage { let mut tx = self.pool.begin().await.map_err(db_err)?; for t in [ "sg_symbols", + "sg_embeddings", "sg_files", "sg_call_records", "sg_call_names", diff --git a/crates/codegraph-graph/src/storage/postgres.rs b/crates/codegraph-graph/src/storage/postgres.rs index 0692902f4..234f4f3ab 100644 --- a/crates/codegraph-graph/src/storage/postgres.rs +++ b/crates/codegraph-graph/src/storage/postgres.rs @@ -1,4 +1,8 @@ -use super::{Result, Storage, StorageError, Tx, decode_chain, encode_chain}; +use std::collections::HashMap; + +use super::{ + Result, Storage, StorageError, Tx, decode_chain, decode_vector, encode_chain, encode_vector, +}; use async_trait::async_trait; use codegraph_core::{Annotation, FileInfo, ScopeLevel, Symbol, SymbolKind}; use sqlx::postgres::{PgPoolOptions, PgRow}; @@ -71,6 +75,19 @@ impl PostgresStorage { .execute(&self.pool) .await .map_err(db_err)?; + // Embeddings: (repo_id, symbol_id) → vector BLOB. Tạo bảng nếu chưa có + // (idempotent) để không bắt buộc chạy migration thủ công cho tính năng này. + sqlx::query( + "CREATE TABLE IF NOT EXISTS sg_embeddings ( + repo_id BIGINT NOT NULL, + symbol_id BIGINT NOT NULL, + vector BYTEA NOT NULL, + PRIMARY KEY (repo_id, symbol_id) + )", + ) + .execute(&self.pool) + .await + .map_err(db_err)?; Ok(()) } @@ -478,6 +495,54 @@ impl Storage for PostgresStorage { rows.iter().map(row_to_symbol).collect() } + async fn save_embedding(&mut self, symbol_id: u64, vector: &[f32]) -> Result<()> { + sqlx::query( + "INSERT INTO sg_embeddings (repo_id, symbol_id, vector) VALUES ($1,$2,$3) \ + ON CONFLICT (repo_id, symbol_id) DO UPDATE SET vector = EXCLUDED.vector", + ) + .bind(self.repo_id as i64) + .bind(symbol_id as i64) + .bind(encode_vector(vector)) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + + async fn load_embedding(&self, symbol_id: u64) -> Result>> { + let row: Option<(Vec,)> = sqlx::query_as( + "SELECT vector FROM sg_embeddings WHERE repo_id = $1 AND symbol_id = $2", + ) + .bind(self.repo_id as i64) + .bind(symbol_id as i64) + .fetch_optional(&self.pool) + .await + .map_err(db_err)?; + Ok(row.and_then(|(b,)| decode_vector(&b))) + } + + async fn load_all_embeddings(&self) -> Result>> { + let rows: Vec<(i64, Vec)> = + sqlx::query_as("SELECT symbol_id, vector FROM sg_embeddings WHERE repo_id = $1") + .bind(self.repo_id as i64) + .fetch_all(&self.pool) + .await + .map_err(db_err)?; + Ok(rows + .into_iter() + .filter_map(|(id, b)| decode_vector(&b).map(|v| (id as u64, v))) + .collect()) + } + + async fn clear_embeddings(&mut self) -> Result<()> { + sqlx::query("DELETE FROM sg_embeddings WHERE repo_id = $1") + .bind(self.repo_id as i64) + .execute(&self.pool) + .await + .map_err(db_err)?; + Ok(()) + } + async fn save_next_id(&mut self, next: u64) -> Result<()> { sqlx::query( "INSERT INTO sg_next_id (repo_id, next) VALUES ($1, $2) \ @@ -660,6 +725,7 @@ impl Storage for PostgresStorage { let mut tx = self.pool.begin().await.map_err(db_err)?; for t in [ "sg_symbols", + "sg_embeddings", "sg_files", "sg_call_records", "sg_call_names", diff --git a/crates/codegraph-graph/src/storage/redis.rs b/crates/codegraph-graph/src/storage/redis.rs index 14a43b294..e01f06675 100644 --- a/crates/codegraph-graph/src/storage/redis.rs +++ b/crates/codegraph-graph/src/storage/redis.rs @@ -14,6 +14,7 @@ //! | `{prefix}:chains` | Hash | record → chain bytes | //! | `{prefix}:shortcut:{shard}:{elem}` | Set | node ids chứa elem | //! | `{prefix}:symbols` | Hash | symbol id → Symbol JSON | +//! | `{prefix}:embeddings` | Hash | symbol id → embedding BLOB (f32 little-endian) | //! | `{prefix}:nextid` | String| next symbol registry id | //! | `{prefix}:callrecords` | Hash | func id → call records | //! | `{prefix}:callnames` | Hash | call name → call sites | @@ -28,7 +29,9 @@ use tokio::sync::Mutex; use async_trait::async_trait; -use super::{FileInfo, Result, Storage, StorageError, Symbol, Tx, TxOp}; +use super::{ + FileInfo, Result, Storage, StorageError, Symbol, Tx, TxOp, decode_vector, encode_vector, +}; // ==================== KeyBuilder ==================== @@ -553,6 +556,55 @@ impl Storage for RedisStorage { Ok(out) } + async fn save_embedding(&mut self, symbol_id: u64, vector: &[f32]) -> Result<()> { + let mut conn = self.lock().await; + cmd("HSET") + .arg(self.kb.key("embeddings")) + .arg(symbol_id as i64) + .arg(encode_vector(vector)) + .query_async::<()>(&mut *conn) + .await + .map_err(|e: redis::RedisError| StorageError::Internal(e.to_string()))?; + Ok(()) + } + + async fn load_embedding(&self, symbol_id: u64) -> Result>> { + let mut conn = self.lock().await; + let data: Option> = cmd("HGET") + .arg(self.kb.key("embeddings")) + .arg(symbol_id as i64) + .query_async(&mut *conn) + .await + .map_err(|e: redis::RedisError| StorageError::Internal(e.to_string()))?; + Ok(data.and_then(|b| decode_vector(&b))) + } + + async fn load_all_embeddings(&self) -> Result>> { + let mut conn = self.lock().await; + let map: HashMap> = cmd("HGETALL") + .arg(self.kb.key("embeddings")) + .query_async(&mut *conn) + .await + .map_err(|e: redis::RedisError| StorageError::Internal(e.to_string()))?; + let mut out = HashMap::with_capacity(map.len()); + for (k, v) in map { + if let (Ok(id), Some(vec)) = (k.parse::(), decode_vector(&v)) { + out.insert(id, vec); + } + } + Ok(out) + } + + async fn clear_embeddings(&mut self) -> Result<()> { + let mut conn = self.lock().await; + cmd("DEL") + .arg(self.kb.key("embeddings")) + .query_async::<()>(&mut *conn) + .await + .map_err(|e: redis::RedisError| StorageError::Internal(e.to_string()))?; + Ok(()) + } + async fn save_next_id(&mut self, next: u64) -> Result<()> { let mut conn = self.lock().await; cmd("SET") @@ -714,6 +766,7 @@ impl Storage for RedisStorage { let mut conn = self.lock().await; cmd("DEL") .arg(self.kb.key("symbols")) + .arg(self.kb.key("embeddings")) .arg(self.kb.key("nextid")) .arg(self.kb.key("callrecords")) .arg(self.kb.key("callnames")) diff --git a/crates/codegraph-graph/src/storage/sqlite.rs b/crates/codegraph-graph/src/storage/sqlite.rs index b7d2aeeea..1cba8a77b 100644 --- a/crates/codegraph-graph/src/storage/sqlite.rs +++ b/crates/codegraph-graph/src/storage/sqlite.rs @@ -26,7 +26,15 @@ //! atomic trong một SQLite transaction tại `commit` (giống InMemory/Redis). //! Mọi query là runtime SQL (không dùng macro `query!` — tránh phụ thuộc //! `DATABASE_URL` lúc build). - +//! +//! Nếu extension sqlite-vss (`vector0`/`vss0`) có mặt (config +//! `[embedding].vss_extension`), kết nối sẽ load extension và tạo thêm virtual +//! table `sg_vss USING vss0(vec(384))` để KNN semantic chạy HNSW ANN ngay trong +//! SQLite. Thiếu extension → `sg_vss` không được tạo, KNN fallback brute-force +//! in-memory (như mọi backend khác). + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use async_trait::async_trait; @@ -34,7 +42,8 @@ use codegraph_core::{FileInfo, Symbol}; use sqlx::Row; use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePool, SqlitePoolOptions}; -use super::{EMPTY, Result, Storage, StorageError, Tx, TxOp}; +use super::{EMPTY, Result, Storage, StorageError, Tx, TxOp, decode_vector, encode_vector}; +use crate::embeddings::resolve_vss_extensions; fn db_err(e: sqlx::Error) -> StorageError { StorageError::Internal(e.to_string()) @@ -44,6 +53,10 @@ fn db_err(e: sqlx::Error) -> StorageError { pub struct SqliteStorage { pool: SqlitePool, + /// `true` nếu extension sqlite-vss (`vss0`) đã load thành công và bảng + /// `sg_vss` sẵn sàng — KNN semantic chạy qua `vss0` (HNSW ANN trong SQLite). + /// `false` → KNN fallback sang `VectorIndex` in-memory (brute-force). + vss_available: AtomicBool, } impl SqliteStorage { @@ -57,17 +70,52 @@ impl SqliteStorage { { std::fs::create_dir_all(parent).map_err(|e| StorageError::Internal(e.to_string()))?; } - let options = SqliteConnectOptions::new() + // Nếu extension sqlite-vss (`vector0`/`vss0`) có mặt → load vào kết nối + // để KNN chạy HNSW ANN ngay trong SQLite. Thiếu file → không load, KNN + // fallback brute-force (open vẫn thành công). + let vss = resolve_vss_extensions(); + let mut options = SqliteConnectOptions::new() .filename(path) .create_if_missing(true) .journal_mode(SqliteJournalMode::Wal) .busy_timeout(Duration::from_secs(5)); + let vss_requested = if let Some((v0, vss_ext)) = &vss { + options = options + .extension(v0.to_string_lossy().into_owned()) + .extension(vss_ext.to_string_lossy().into_owned()); + true + } else { + false + }; let pool = SqlitePoolOptions::new() .connect_with(options) .await .map_err(db_err)?; - let s = Self { pool }; + let s = Self { + pool, + vss_available: AtomicBool::new(false), + }; s.init().await?; + // Bật `sg_vss` (vss0 virtual table) khi extension đã được load. Nếu tạo + // bảng lỗi → tắt vss, KNN fallback brute-force (vẫn hoạt động đúng). + let available = if vss_requested { + match sqlx::query("CREATE VIRTUAL TABLE IF NOT EXISTS sg_vss USING vss0(vec(384))") + .execute(&mut *s.pool.acquire().await.map_err(db_err)?) + .await + { + Ok(_) => true, + Err(e) => { + eprintln!( + "codegraph: sqlite-vss loaded but vss0 table create failed; \ + falling back to brute-force KNN: {e}" + ); + false + } + } + } else { + false + }; + s.vss_available.store(available, Ordering::SeqCst); Ok(s) } @@ -170,6 +218,11 @@ impl SqliteStorage { id INTEGER PRIMARY KEY CHECK (id = 1), version INTEGER NOT NULL )", + // ── Embeddings (vector per symbol id) ── + "CREATE TABLE IF NOT EXISTS sg_embeddings ( + symbol_id INTEGER PRIMARY KEY, + vector BLOB NOT NULL + )", // Sentinel node id 0 + counter bắt đầu từ 1. "INSERT OR IGNORE INTO rt_nodes (id, prefix, record) VALUES (0, X'', 0)", "INSERT OR IGNORE INTO rt_counter (id, next) VALUES (1, 1)", @@ -473,6 +526,91 @@ impl Storage for SqliteStorage { Ok(next as u64) } + async fn save_embedding(&mut self, symbol_id: u64, vector: &[f32]) -> Result<()> { + let mut conn = self.pool.acquire().await.map_err(db_err)?; + sqlx::query( + "INSERT INTO sg_embeddings (symbol_id, vector) VALUES (?1, ?2) + ON CONFLICT(symbol_id) DO UPDATE SET vector = excluded.vector", + ) + .bind(symbol_id as i64) + .bind(encode_vector(vector)) + .execute(&mut *conn) + .await + .map_err(db_err)?; + // Mirror vào `vss0` (HNSW ANN) nếu extension khả dụng. + if self.vss_available.load(Ordering::SeqCst) { + sqlx::query("INSERT OR REPLACE INTO sg_vss(rowid, vec) VALUES (?1, ?2)") + .bind(symbol_id as i64) + .bind(encode_vector(vector)) + .execute(&mut *conn) + .await + .map_err(db_err)?; + } + Ok(()) + } + + async fn load_embedding(&self, symbol_id: u64) -> Result>> { + let mut conn = self.pool.acquire().await.map_err(db_err)?; + let data: Option> = + sqlx::query_scalar("SELECT vector FROM sg_embeddings WHERE symbol_id = ?1") + .bind(symbol_id as i64) + .fetch_optional(&mut *conn) + .await + .map_err(db_err)?; + Ok(data.and_then(|b| decode_vector(&b))) + } + + async fn load_all_embeddings(&self) -> Result>> { + let mut conn = self.pool.acquire().await.map_err(db_err)?; + let rows: Vec<(i64, Vec)> = + sqlx::query_as("SELECT symbol_id, vector FROM sg_embeddings ORDER BY symbol_id") + .fetch_all(&mut *conn) + .await + .map_err(db_err)?; + Ok(rows + .into_iter() + .filter_map(|(id, b)| decode_vector(&b).map(|v| (id as u64, v))) + .collect()) + } + + async fn clear_embeddings(&mut self) -> Result<()> { + let mut conn = self.pool.acquire().await.map_err(db_err)?; + sqlx::query("DELETE FROM sg_embeddings") + .execute(&mut *conn) + .await + .map_err(db_err)?; + if self.vss_available.load(Ordering::SeqCst) { + sqlx::query("DELETE FROM sg_vss") + .execute(&mut *conn) + .await + .map_err(db_err)?; + } + Ok(()) + } + + async fn knn(&self, query_vec: &[f32], k: usize) -> Result>> { + if !self.vss_available.load(Ordering::SeqCst) { + return Ok(None); + } + let mut conn = self.pool.acquire().await.map_err(db_err)?; + // `vss_search(vec, )` trả các row gần nhất + `distance` (nhỏ = gần). + // Đảo dấu distance → `sim` (lớn = gần) đồng nhất với `VectorIndex::knn`. + let rows: Vec<(i64, f64)> = sqlx::query_as( + "SELECT rowid, distance FROM sg_vss + WHERE vss_search(vec, ?) ORDER BY distance LIMIT ?", + ) + .bind(encode_vector(query_vec)) + .bind(k as i64) + .fetch_all(&mut *conn) + .await + .map_err(db_err)?; + Ok(Some( + rows.into_iter() + .map(|(id, dist)| (id as u64, -dist as f32)) + .collect(), + )) + } + async fn all_chains(&self) -> Result)>> { let mut conn = self.pool.acquire().await.map_err(db_err)?; let rows: Vec<(i64, Vec)> = @@ -614,6 +752,7 @@ impl Storage for SqliteStorage { "DELETE FROM sg_call_records", "DELETE FROM sg_call_names", "DELETE FROM sg_files", + "DELETE FROM sg_embeddings", "UPDATE sg_next_id SET next = 100 WHERE id = 1", "UPDATE sg_meta SET version = 0 WHERE id = 1", ] { @@ -1140,6 +1279,60 @@ mod tests { assert_eq!(s.get_chain(9).await.unwrap(), None); } + #[tokio::test] + async fn test_embeddings_roundtrip() { + let (_d, path) = tmp_path(); + // Save embeddings, then reload — verify BLOB persistence (dùng lại cho + // KNN/k-means mà không re-embed). + let mut s = SqliteStorage::open(&path).await.unwrap(); + let v1 = vec![0.1f32, 0.2, 0.3, -0.4]; + let v2 = vec![1.0f32, -1.0, 0.0, 0.5]; + s.save_embedding(100, &v1).await.unwrap(); + s.save_embedding(101, &v2).await.unwrap(); + // upsert overwrite cho 100. + s.save_embedding(100, &v2).await.unwrap(); + + let all = s.load_all_embeddings().await.unwrap(); + assert_eq!(all.len(), 2); + assert_eq!( + all.get(&100).unwrap(), + &v2, + "id 100 phải bị overwrite thành v2" + ); + assert_eq!(all.get(&101).unwrap(), &v2, "id 101 giữ v2"); + assert_eq!(s.load_embedding(100).await.unwrap().unwrap(), v2); + assert_eq!(s.load_embedding(101).await.unwrap().unwrap(), v2); + + // clear → rỗng + s.clear_embeddings().await.unwrap(); + assert!(s.load_all_embeddings().await.unwrap().is_empty()); + assert_eq!(s.load_embedding(101).await.unwrap(), None); + } + + /// KNN qua sqlite-vss (`vss0`) — chỉ chạy khi extension thực sự có mặt + /// (`vector0`/`vss0` trong `vss_extension` config hoặc `/vss`). + /// Thiếu extension → skip (KNN lúc đó fallback brute-force in-memory). + #[tokio::test] + async fn test_vss_knn_when_extension_present() { + if crate::embeddings::resolve_vss_extensions().is_none() { + return; + } + let (_d, path) = tmp_path(); + let mut s = SqliteStorage::open(&path).await.unwrap(); + // Hai vector 384-dim: `a` cùng chiều với query, `b` ngược chiều. + let a: Vec = vec![1.0; 384]; + let b: Vec = vec![-1.0; 384]; + let q: Vec = vec![1.0; 384]; + s.save_embedding(1, &a).await.unwrap(); + s.save_embedding(2, &b).await.unwrap(); + let hits = s.knn(&q, 2).await.unwrap(); + let hits = hits.expect("vss phải khả dụng khi extension có mặt"); + assert_eq!(hits.len(), 2); + // Gần nhất với query (1,1,...) phải là `a` (id 1), không phải `b`. + assert_eq!(hits[0].0, 1, "vss KNN phải trả symbol gần nhất trước"); + assert!(hits[0].1 > hits[1].1, "similarity phải giảm dần"); + } + #[tokio::test] async fn test_persists_across_reopen() { let (_d, path) = tmp_path(); diff --git a/crates/codegraph-graph/src/vector_index.rs b/crates/codegraph-graph/src/vector_index.rs new file mode 100644 index 000000000..375cb9205 --- /dev/null +++ b/crates/codegraph-graph/src/vector_index.rs @@ -0,0 +1,265 @@ +//! Vector index cho semantic search: lưu embedding mỗi symbol (id → vector f32 +//! đã normalize) và hỗ trợ KNN (cosine) + k-means clustering. +//! +//! MVP: **brute-force** — tính cosine với mọi vector (O(n)). Đủ cho index tới +//! vài chục ngàn symbol (latency < vài ms trên CPU). Khi cần scale → thay +//! bằng HNSW/IVF sau (cùng interface `knn`). Vector index là **derived state** +//! (hàm pure của symbols) nên rebuild từ entity store mỗi lần `ingest`/`open`, +//! không persist riêng. + +use std::collections::HashMap; + +/// Kết quả KNN: (symbol id, cosine similarity ∈ [-1, 1]). +pub type KnnHit = (u64, f32); + +/// Kết quả k-means: centroids + assignment (symbol id → cluster index). +#[derive(Debug, Clone)] +pub struct KMeansResult { + pub centroids: Vec>, + pub assignments: HashMap, +} + +/// Index vector in-memory. +pub struct VectorIndex { + dim: usize, + vectors: HashMap>, +} + +impl VectorIndex { + pub fn new(dim: usize) -> Self { + Self { + dim: dim.max(1), + vectors: HashMap::new(), + } + } + + pub fn dim(&self) -> usize { + self.dim + } + + pub fn is_empty(&self) -> bool { + self.vectors.is_empty() + } + + pub fn len(&self) -> usize { + self.vectors.len() + } + + /// Thay thế toàn bộ index. + pub fn set_all(&mut self, vectors: HashMap>) { + self.vectors = vectors; + } + + /// Thêm / cập nhật embedding của một symbol. + pub fn insert(&mut self, id: u64, vec: Vec) { + self.vectors.insert(id, vec); + } + + /// Xoá embedding của một symbol. + pub fn delete(&mut self, id: u64) { + self.vectors.remove(&id); + } + + /// Xoá toàn bộ. + pub fn clear(&mut self) { + self.vectors.clear(); + } + + /// Lấy vector của một symbol (nếu có). + pub fn get(&self, id: u64) -> Option<&Vec> { + self.vectors.get(&id) + } + + /// Cosine similarity của hai vector (giả định đã normalize → = dot). + fn cosine(a: &[f32], b: &[f32]) -> f32 { + debug_assert_eq!(a.len(), b.len()); + a.iter().zip(b).map(|(x, y)| x * y).sum() + } + + /// KNN: top-`k` symbol gần nhất với `query_vec` (cosine giảm dần). + /// `k = 0` → trả toàn bộ (sort theo similarity). Rỗng nếu index trống. + pub fn knn(&self, query_vec: &[f32], k: usize) -> Vec { + if self.vectors.is_empty() { + return Vec::new(); + } + let mut scored: Vec = self + .vectors + .iter() + .map(|(&id, v)| (id, Self::cosine(query_vec, v))) + .collect(); + scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); + if k > 0 && scored.len() > k { + scored.truncate(k); + } + scored + } + + /// K-means (Lloyd) clustering trên các vector đã lưu. Khởi tạo bằng + /// k-means++ (deterministic: seed cố định) để ổn định. Trả về centroids + + /// assignment. `k` clamp về `[1, n]`; `max_iters` giới hạn vòng lặp. + /// + /// Dùng cho việc gom nhóm symbol liên quan (VD "tất cả hàm xử lý auth"). + pub fn kmeans(&self, k: usize, max_iters: usize) -> KMeansResult { + let points: Vec<(u64, Vec)> = self + .vectors + .iter() + .map(|(&id, v)| (id, v.clone())) + .collect(); + let n = points.len(); + if n == 0 { + return KMeansResult { + centroids: Vec::new(), + assignments: HashMap::new(), + }; + } + let k = k.clamp(1, n); + + // ── k-means++ init (deterministic xorshift seed) ── + let mut rng = XorShift::new(0x9E37_79B9_7F4A_7C15 ^ (n as u64)); + let mut centroids: Vec> = Vec::with_capacity(k); + centroids.push(points[rng.next() as usize % n].1.clone()); + while centroids.len() < k { + // Chọn điểm có D² (khoảng cách tới centroid gần nhất) lớn nhất, + // dùng rng để bốc (xấp xỉ k-means++ mà không sort toàn bộ mỗi bước). + let mut best = 0usize; + let mut best_d = -1.0f32; + for (i, (_, v)) in points.iter().enumerate() { + let d = centroids + .iter() + .map(|c| Self::cosine(c, v)) + .fold(f32::MAX, |acc, s| acc.min(1.0 - s)); + if d > best_d { + best_d = d; + best = i; + } + } + centroids.push(points[best].1.clone()); + } + + // ── Lloyd iterations ── + let mut assignments: HashMap = HashMap::with_capacity(n); + for _ in 0..max_iters.max(1) { + let mut changed = false; + // Assign. + for (id, v) in &points { + let mut best = 0usize; + let mut best_s = f32::NEG_INFINITY; + for (ci, c) in centroids.iter().enumerate() { + let s = Self::cosine(c, v); + if s > best_s { + best_s = s; + best = ci; + } + } + if assignments.get(id) != Some(&best) { + assignments.insert(*id, best); + changed = true; + } + } + // Update centroids (mean rồi normalize). + let mut sums: Vec> = vec![vec![0.0f32; self.dim]; k]; + let mut counts = vec![0usize; k]; + for (id, v) in &points { + let c = assignments[id]; + counts[c] += 1; + for (j, x) in v.iter().enumerate() { + sums[c][j] += x; + } + } + for (ci, sum) in sums.iter_mut().enumerate() { + if counts[ci] > 0 { + for x in sum.iter_mut() { + *x /= counts[ci] as f32; + } + } + normalize(sum); + centroids[ci] = std::mem::take(sum); + } + if !changed { + break; + } + } + + KMeansResult { + centroids, + assignments, + } + } +} + +/// L2-normalize một vector tại chỗ. +fn normalize(v: &mut [f32]) { + let norm = v.iter().map(|x| x * x).sum::().sqrt(); + if norm > 0.0 { + for x in v.iter_mut() { + *x /= norm; + } + } +} + +/// PRNG nhỏ, deterministic (không phụ thuộc `rand`). +struct XorShift { + state: u64, +} + +impl XorShift { + fn new(seed: u64) -> Self { + Self { state: seed | 1 } + } + fn next(&mut self) -> u64 { + let mut x = self.state; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.state = x; + x + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v(xs: &[f32]) -> Vec { + xs.to_vec() + } + + #[test] + fn knn_basic() { + let mut idx = VectorIndex::new(3); + // Vector đã L2-normalized (đúng contract của cosine = dot product). + idx.insert(1, v(&[1.0, 0.0, 0.0])); + idx.insert(2, v(&[0.0, 1.0, 0.0])); + idx.insert(3, v(&[0.995, 0.0995, 0.0])); + let hits = idx.knn(&[1.0, 0.0, 0.0], 2); + assert_eq!(hits.len(), 2); + // id 1 (chính nó) và id 3 (gần nhất) đứng đầu. + assert_eq!(hits[0].0, 1); + assert_eq!(hits[1].0, 3); + assert!(hits[0].1 > hits[1].1); + } + + #[test] + fn knn_empty() { + let idx = VectorIndex::new(4); + assert!(idx.knn(&[0.0; 4], 5).is_empty()); + } + + #[test] + fn kmeans_groups_similar() { + let mut idx = VectorIndex::new(2); + // Cluster A: quanh (1,0). + idx.insert(1, v(&[1.0, 0.0])); + idx.insert(2, v(&[0.9, 0.1])); + // Cluster B: quanh (0,1). + idx.insert(3, v(&[0.0, 1.0])); + idx.insert(4, v(&[0.1, 0.9])); + let res = idx.kmeans(2, 20); + assert_eq!(res.centroids.len(), 2); + // Hai symbol trong cluster A phải cùng nhãn. + assert_eq!(res.assignments[&1], res.assignments[&2]); + assert_eq!(res.assignments[&3], res.assignments[&4]); + // Hai cluster khác nhãn. + assert_ne!(res.assignments[&1], res.assignments[&3]); + } +} diff --git a/crates/codegraph-graph/tests/lmdb.rs b/crates/codegraph-graph/tests/lmdb.rs index b89223395..d25bc27f2 100644 --- a/crates/codegraph-graph/tests/lmdb.rs +++ b/crates/codegraph-graph/tests/lmdb.rs @@ -6,10 +6,10 @@ #![cfg(feature = "lmdb")] -use codegraph_core::{CallRecord, EffectType, SYMBOL_BASE, Symbol, SymbolKind}; -use codegraph_graph::GraphIndex; +use codegraph_core::{CallRecord, EffectType, SYMBOL_BASE, Symbol, SymbolKind, SymbolMatch}; use codegraph_graph::ParseResult; use codegraph_graph::SharedGraphIndex; +use codegraph_graph::{GraphIndex, Pagination}; use std::collections::HashMap; use std::sync::Arc; @@ -214,9 +214,20 @@ async fn ingest_same_function_name_across_files_stays_distinct() { ); let hits = idx - .search_symbol("process", Some(SymbolKind::Function), 10) + .search_symbol_paged_resumable( + "process", + Some(SymbolKind::Function), + SymbolMatch::Contains, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) .await - .unwrap(); + .unwrap() + .page; assert_eq!(hits.len(), 2); let mut files: Vec<&str> = hits.iter().map(|s| s.file.as_str()).collect(); files.sort_unstable(); diff --git a/crates/codegraph-graph/tests/sqlite.rs b/crates/codegraph-graph/tests/sqlite.rs index 2c61f8f5a..ccd428ae7 100644 --- a/crates/codegraph-graph/tests/sqlite.rs +++ b/crates/codegraph-graph/tests/sqlite.rs @@ -7,8 +7,8 @@ #![cfg(feature = "sqlite")] -use codegraph_core::{CallRecord, EffectType, SYMBOL_BASE, Symbol, SymbolKind}; -use codegraph_graph::{GraphIndex, ParseResult, SharedGraphIndex}; +use codegraph_core::{CallRecord, EffectType, SYMBOL_BASE, Symbol, SymbolKind, SymbolMatch}; +use codegraph_graph::{GraphIndex, Pagination, ParseResult, SharedGraphIndex}; use std::collections::HashMap; use std::sync::Arc; @@ -234,9 +234,20 @@ async fn ingest_same_function_name_across_files_stays_distinct() { // Search tên trả đủ 2 kết quả (không hoà trộn thành 1). let hits = idx - .search_symbol("process", Some(SymbolKind::Function), 10) + .search_symbol_paged_resumable( + "process", + Some(SymbolKind::Function), + SymbolMatch::Contains, + Pagination { + limit: 10, + offset: 0, + }, + None, + None, + ) .await - .unwrap(); + .unwrap() + .page; assert_eq!(hits.len(), 2); let mut files: Vec<&str> = hits.iter().map(|s| s.file.as_str()).collect(); files.sort_unstable(); @@ -269,16 +280,42 @@ async fn sandbox_search_kinds_finds_java_method() { // Trước fix: lọc Function-only → bỏ Method → empty (sandbox fail). let only_func = idx - .search_symbol("getProfile", Some(SymbolKind::Function), 1) + .search_symbol_paged_resumable( + "getProfile", + Some(SymbolKind::Function), + SymbolMatch::Contains, + Pagination { + limit: 1, + offset: 0, + }, + None, + None, + ) .await - .unwrap(); + .unwrap() + .page; assert!(only_func.is_empty()); // Fix: sandbox chấp nhận Function | Method. let hits = idx - .search_symbol_kinds("getProfile", &[SymbolKind::Function, SymbolKind::Method], 1) + .search_symbol_paged_resumable( + "getProfile", + None, + SymbolMatch::Contains, + Pagination { + limit: 1, + offset: 0, + }, + None, + None, + ) .await - .unwrap(); + .unwrap() + .page + .into_iter() + .find(|s| matches!(s.kind, SymbolKind::Function | SymbolKind::Method)) + .into_iter() + .collect::>(); assert_eq!(hits.len(), 1); assert_eq!(hits[0].name, "getProfile"); assert_eq!(hits[0].kind, SymbolKind::Method); diff --git a/crates/codegraph-mcp/src/tools.rs b/crates/codegraph-mcp/src/tools.rs index 65bebe0ec..dbea9c5a8 100644 --- a/crates/codegraph-mcp/src/tools.rs +++ b/crates/codegraph-mcp/src/tools.rs @@ -38,18 +38,6 @@ pub fn is_known_tool(name: &str) -> bool { fn tool_defs() -> Vec { vec![ - tool( - "codegraph_search", - "Search symbols by name (substring, case-insensitive). On large indexes this can take a while — pass timeout_ms (default 20000) and, if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue the search from where it stopped.", - json!({ "type": "object", "properties": { - "query": { "type": "string" }, - "limit": { "type": "integer", "default": 10 }, - "resume": { "type": "string", "description": "Resume id from a previous timeout — retry the same call with this to continue where it stopped." }, - "timeout_ms": { "type": "integer", "default": 20000, "description": "Soft time budget in ms; 0 = no limit. On timeout the tool errors with a resume id." }, - "detail": { "type": "string", "enum": ["minimal", "medium", "verbose"], "description": "Symbol detail for this call (overrides session default): minimal = id/name/kind/file/line, medium = + signature, verbose = full Symbol." }, - "format": { "type": "string", "enum": ["minimize", "medium"], "description": "Output format for this call (overrides session default): minimize = symbol items as fixed-order positional arrays (default), medium = objects with default-valued fields omitted." } - }, "required": ["query"] }), - ), tool( "codegraph_symbol", "Look up a symbol by id or exact name. Duplicate names → ambiguous with the full match list; retry with symbol_id.", @@ -163,11 +151,11 @@ fn tool_defs() -> Vec { // ── Enhanced symbol search (semgraph_search_symbol) ── tool( "codegraph_search_symbol", - "Search symbols by name with optional kind filter, match mode, and pagination. match: 'contains' (substring anywhere, default), 'prefix' (name starts with), 'suffix' (name ENDS with — e.g. query=\"Service\" finds every *Service class), 'exact' (exact name, case-insensitive). Use 'total' with 'offset' to fetch further pages until offset >= total. On large indexes pass timeout_ms (default 20000); if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue. When more results remain, the response includes a resume id you can pass to page further without re-scanning.", + "Search symbols by name with optional kind filter, match mode, and pagination. match: 'contains' (substring anywhere, default), 'prefix' (name starts with), 'suffix' (name ENDS with — e.g. query=\"Service\" finds every *Service class), 'exact' (exact name, case-insensitive), 'semantic' (vector KNN over symbol embeddings — find symbols by similar/approximate names when you don't remember the exact spelling), 'hybrid' (merge 'contains' + 'semantic' via Reciprocal Rank Fusion). Use 'total' with 'offset' to fetch further pages until offset >= total. On large indexes pass timeout_ms (default 20000); if the call returns a timeout error containing \"resume\": \"\", retry the SAME call with that resume id to continue. When more results remain, the response includes a resume id you can pass to page further without re-scanning.", json!({ "type": "object", "properties": { "query": { "type": "string" }, "kind": { "type": "string", "enum": ["function", "method", "class", "interface", "enum", "variable", "constant", "parameter", "field", "module", "file"] }, - "match": { "type": "string", "enum": ["contains", "prefix", "suffix", "exact"], "default": "contains" }, + "match": { "type": "string", "enum": ["contains", "prefix", "suffix", "exact", "semantic", "hybrid"], "default": "contains" }, "limit": { "type": "integer", "default": 20 }, "offset": { "type": "integer", "default": 0 }, "resume": { "type": "string", "description": "Resume id from a previous timeout (or from a previous response with more pages) — retry the same call with this to continue where it stopped." }, @@ -326,39 +314,6 @@ pub async fn dispatch_with_api( args: Value, ) -> Result { match name { - "codegraph_search" => { - let q = arg_str(&args, "query")?; - let limit = args.get("limit").and_then(|v| v.as_u64()).unwrap_or(10) as u32; - let resume = args - .get("resume") - .and_then(|v| v.as_str()) - .map(str::to_string); - let timeout_ms = args - .get("timeout_ms") - .and_then(|v| v.as_u64()) - .unwrap_or(20000); - let out = api.search_resumable(q, limit, resume, timeout_ms).await?; - if out.timed_out { - // Không trả kết quả nửa chừng — báo lỗi kèm resume id để LLM retry - // cùng args + resume → search tiếp tục đúng vị trí dừng. - return Err(Error::Other(format!( - "codegraph_search timed out after {}ms (collected {} symbols so far). \ - Retry the same call with the same arguments plus \"resume\": \"{}\" \ - to continue the search from where it stopped.", - timeout_ms, - out.progress, - out.resume.as_deref().unwrap_or("") - ))); - } - let detail = detail_from_args(&args, session_detail); - let format = format_from_args(&args, session_format); - let out: Vec = out - .page - .iter() - .map(|s| symbol_json(root.as_str(), s, detail, format)) - .collect(); - emit_value(root.as_str(), Value::Array(out)) - } "codegraph_symbol" => { let detail = detail_from_args(&args, session_detail); let format = format_from_args(&args, session_format); @@ -1248,9 +1203,21 @@ pub async fn dispatch_sandbox( } else { let q = arg_str(&args, "name")?; let hits = idx - .search_symbol_kinds(q, &[SymbolKind::Function, SymbolKind::Method], 1) - .await?; - hits.first() + .search_symbol_paged_resumable( + q, + None, + SymbolMatch::Contains, + codegraph_graph::Pagination { + limit: 20, + offset: 0, + }, + None, + None, + ) + .await? + .page; + hits.into_iter() + .find(|s| matches!(s.kind, SymbolKind::Function | SymbolKind::Method)) .map(|s| s.id) .ok_or_else(|| Error::Invalid(format!("no function matching `{q}`")))? }; @@ -1323,10 +1290,21 @@ async fn run_sim( mocks: &[(String, String)], ) -> Result { let Some(sym) = idx - .search_symbol_kinds(entry_name, &[SymbolKind::Function, SymbolKind::Method], 1) + .search_symbol_paged_resumable( + entry_name, + None, + SymbolMatch::Contains, + codegraph_graph::Pagination { + limit: 20, + offset: 0, + }, + None, + None, + ) .await? + .page .into_iter() - .next() + .find(|s| matches!(s.kind, SymbolKind::Function | SymbolKind::Method)) else { return Ok(json!({ "present": false })); }; diff --git a/crates/codegraph/Cargo.toml b/crates/codegraph/Cargo.toml index ba58cc99d..60917e8fc 100644 --- a/crates/codegraph/Cargo.toml +++ b/crates/codegraph/Cargo.toml @@ -27,6 +27,16 @@ indicatif = "0.18.6" [features] # Mặc định bật RDBMS (Postgres/MySQL) để CLI + MCP server có thể serve backend -# multi-tenant. Tắt để build nhẹ: `cargo build --no-default-features`. +# multi-tenant. Embedding fastembed LÀ OPT-IN — chỉ bật khi cần semantic search: +# `cargo build --features fastembed` (hoặc set `[embedding] backend = "fastembed"` +# trong config khi chạy với bản đã compile sẵn fastembed). Tắt để build +# nhẹ: `cargo build --no-default-features`. default = ["rdbms"] rdbms = ["codegraph-graph/postgres", "codegraph-graph/mysql", "codegraph-mcp/rdbms"] +# Embedding backend fastembed (semantic search) — OPT-IN, không bật mặc định. +fastembed = ["codegraph-graph/fastembed"] +# macOS-only: bật CoreML EP cho ONNX Runtime (embed trên Apple GPU/ANE). Chỉ có +# nghĩa khi build trên macOS + `--features fastembed,apple-accel`. Non-macOS bật +# feature này sẽ lỗi (ort coreml chỉ compile trên macOS). Metal EP chưa được +# expose bởi bản ort hiện tại → "metal" config cũng map sang CoreML. +apple-accel = ["codegraph-graph/apple-accel"] diff --git a/crates/codegraph/src/main.rs b/crates/codegraph/src/main.rs index b9e87ed4c..2ab1b8b54 100644 --- a/crates/codegraph/src/main.rs +++ b/crates/codegraph/src/main.rs @@ -5,6 +5,9 @@ use codegraph_extract::{ExtractStats, Orchestrator}; use codegraph_graph::GraphIndex; use codegraph_mcp::CodegraphServer; +#[cfg(feature = "fastembed")] +use codegraph_graph::embeddings::warm_model_cache; + mod watcher; /// CLI tối giản: chỉ còn lifecycle (`init`/`deinit`) + MCP server (`serve --mcp`). @@ -46,6 +49,18 @@ enum Cmd { }, /// Remove the .codegraph/ directory. Deinit, + /// Pre-download an embedding model into the global cache (so semantic search + /// works offline). Model is cached under `[embedding].cache_dir` (default + /// `~/.cache/codegraph/embeddings`). Requires the `fastembed` feature. + #[cfg(feature = "fastembed")] + Embed { + /// Model name/alias to download, e.g. "bge-small-en-v1.5" (default). + #[arg(long, default_value = "bge-small-en-v1.5")] + model: String, + /// Cache directory (global). Default: ~/.cache/codegraph/embeddings. + #[arg(long)] + cache_dir: Option, + }, /// Run as MCP server (stdio qua `--mcp`, hoặc Streamable HTTP qua `--http`). Serve { #[arg(long)] @@ -126,6 +141,8 @@ async fn main() -> Result<()> { match cmd { Cmd::Init { no_index, progress } => cmd_init(&root, !no_index, progress).await, Cmd::Deinit => cmd_deinit(&root), + #[cfg(feature = "fastembed")] + Cmd::Embed { model, cache_dir } => cmd_embed(&model, cache_dir.as_deref()).await, Cmd::Serve { mcp, http, @@ -236,6 +253,15 @@ fn cmd_deinit(root: &Utf8Path) -> Result<()> { Ok(()) } +/// `codegraph embed --model `: pre-download model vào global cache để +/// semantic search chạy offline. +#[cfg(feature = "fastembed")] +async fn cmd_embed(model: &str, cache_dir: Option<&str>) -> Result<()> { + let dir = cache_dir.map(std::path::Path::new); + warm_model_cache(model, dir).map_err(|e| anyhow!("failed to cache embedding model: {e}"))?; + Ok(()) +} + /// `codegraph serve --mcp`: chạy MCP server trên stdio. /// `codegraph serve --http`: chạy MCP server trên Streamable HTTP. #[allow(clippy::too_many_arguments)]