[pre-commit.ci] pre-commit autoupdate

.pre-commit-config.yaml

@@ -2,7 +2,7 @@
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
   # Ruff version.
-  rev: v0.5.6
+  rev: v0.16.0
   hooks:
     # Run the linter.
     - id: ruff

archive/json-yaml.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python
 
-from pathlib import Path
+import json
 import os
 import sys
-import json
+from pathlib import Path
+
 import yaml
 
 

src/vdf_io/export_vdf/astradb_export.py

@@ -2,21 +2,21 @@
 import json
 import os
 import sys
-from typing import Dict, List
+
 from astrapy.db import AstraDB
-from tqdm import tqdm
-from cassandra.cluster import Cluster
 from cassandra.auth import PlainTextAuthProvider
+from cassandra.cluster import Cluster
 from cassandra.query import SimpleStatement
+from tqdm import tqdm
 
 from vdf_io.constants import DISK_SPACE_LIMIT
+from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 from vdf_io.meta_types import NamespaceMeta
 from vdf_io.names import DBNames
 from vdf_io.util import (
     set_arg_from_input,
     set_arg_from_password,
 )
-from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 
 
 class ExportAstraDB(ExportVDB):
@@ -166,7 +166,7 @@ def get_data_from_cql(self):
             if self.args.get("collections") is None
             else self.args.get("collections").split(",")
         )
-        index_metas: Dict[str, List[NamespaceMeta]] = {}
+        index_metas: dict[str, list[NamespaceMeta]] = {}
         self.paging_state = None
         for index_name in tqdm(index_names, desc="Fetching indexes"):
             # count rows using execute()
@@ -255,7 +255,7 @@ def execute_select_all_once(self, index_name):
 
     def get_data(self):
         index_names = self.get_index_names()
-        index_metas: Dict[str, List[NamespaceMeta]] = {}
+        index_metas: dict[str, list[NamespaceMeta]] = {}
         self.total_imported_count = 0
         for index_name in index_names:
             tqdm.write(f"Exporting collection: {index_name}")

src/vdf_io/export_vdf/chroma_export.py

@@ -1,14 +1,14 @@
 import json
 import os
 import sys
-from tqdm import tqdm
 
 import chromadb
+from tqdm import tqdm
 
 from vdf_io.constants import DEFAULT_BATCH_SIZE, DISK_SPACE_LIMIT
+from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 from vdf_io.names import DBNames
 from vdf_io.util import expand_shorthand_path, set_arg_from_input
-from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 
 
 class ExportChroma(ExportVDB):

src/vdf_io/export_vdf/kdbai_export.py

@@ -1,23 +1,21 @@
-import os
-import json
-from typing import Dict, List
-from dotenv import load_dotenv
 import datetime
-from tqdm import tqdm
+import json
+import os
 
 import kdbai_client as kdbai
+from dotenv import load_dotenv
+from tqdm import tqdm
 
 from vdf_io.export_vdf.vdb_export_cls import ExportVDB
-from vdf_io.names import DBNames
 from vdf_io.meta_types import NamespaceMeta, VDFMeta
+from vdf_io.names import DBNames
 from vdf_io.util import (
     get_author_name,
     set_arg_from_input,
     set_arg_from_password,
     standardize_metric,
 )
 
-
 load_dotenv()
 
 
@@ -87,7 +85,7 @@ def get_data(self):
             table_names = self.get_all_index_names()
         else:
             table_names = self.args["tables"].split(",")
-        index_metas: Dict[str, List[NamespaceMeta]] = {}
+        index_metas: dict[str, list[NamespaceMeta]] = {}
         for table_name in tqdm(table_names, desc="Fetching indexes"):
             index_metas[table_name] = self.export_table(table_name)
         internal_metadata = VDFMeta(

src/vdf_io/export_vdf/lancedb_export.py

@@ -1,15 +1,15 @@
 import json
 import os
-from typing import Dict, List
+
 import lancedb
 import pandas as pd
 import pyarrow
 from tqdm import tqdm
-from vdf_io.meta_types import NamespaceMeta
 
+from vdf_io.export_vdf.vdb_export_cls import ExportVDB
+from vdf_io.meta_types import NamespaceMeta
 from vdf_io.names import DBNames
 from vdf_io.util import set_arg_from_input, set_arg_from_password
-from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 
 
 class ExportLanceDB(ExportVDB):
@@ -83,7 +83,7 @@ def get_data(self):
         index_names = self.get_index_names()
         BATCH_SIZE = self.args["batch_size"]
         total = 0
-        index_metas: Dict[str, List[NamespaceMeta]] = {}
+        index_metas: dict[str, list[NamespaceMeta]] = {}
         for index_name in index_names:
             namespace_metas = []
             vectors_directory = self.create_vec_dir(index_name)

src/vdf_io/export_vdf/milvus_export.py

@@ -1,21 +1,19 @@
-from typing import Dict, List
-import os
-import json
 import datetime
-from tqdm import tqdm
-from pymilvus import connections, utility, Collection
+import json
+import os
 
+from pymilvus import Collection, connections, utility
+from tqdm import tqdm
 
 from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 from vdf_io.meta_types import NamespaceMeta, VDFMeta
+from vdf_io.names import DBNames
 from vdf_io.util import (
     get_author_name,
     set_arg_from_input,
     set_arg_from_password,
     standardize_metric,
 )
-from vdf_io.names import DBNames
-
 
 MAX_FETCH_SIZE = 1_000
 
@@ -67,7 +65,7 @@ def export_vdb(cls, args):
         milvus_export.get_data()
         return milvus_export
 
-    def __init__(self, args: Dict):
+    def __init__(self, args: dict):
         """
         Initialize the class.
 
@@ -91,7 +89,7 @@ def get_data(self) -> bool:
         else:
             collection_names = self.args.get("collections").split(",")
 
-        index_metas: Dict[str, List[NamespaceMeta]] = {}
+        index_metas: dict[str, list[NamespaceMeta]] = {}
         for collection_name in tqdm(collection_names, desc="Fetching indexes"):
             index_meta = self.get_data_for_collection(collection_name)
             index_metas[collection_name] = index_meta
@@ -111,10 +109,10 @@ def get_data(self) -> bool:
         print(json.dumps(internal_metadata.model_dump(), indent=4))
         return True
 
-    def get_all_collection_names(self) -> List[str]:
+    def get_all_collection_names(self) -> list[str]:
         return utility.list_collections()
 
-    def get_data_for_collection(self, collection_name: str) -> List[NamespaceMeta]:
+    def get_data_for_collection(self, collection_name: str) -> list[NamespaceMeta]:
         vectors_directory = self.create_vec_dir(collection_name)
 
         try:

src/vdf_io/export_vdf/pinecone_export.py

@@ -1,24 +1,24 @@
 import argparse
 import datetime
-import os
 import json
+import os
+
 import numpy as np
-from tqdm import tqdm
 from halo import Halo
-
-from pinecone.grpc import PineconeGRPC as Pinecone
 from pinecone import Vector
+from pinecone.grpc import PineconeGRPC as Pinecone
+from tqdm import tqdm
 
 from vdf_io.constants import ID_COLUMN
-from vdf_io.names import DBNames
+from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 from vdf_io.meta_types import NamespaceMeta, VDFMeta
+from vdf_io.names import DBNames
 from vdf_io.util import (
     get_author_name,
     set_arg_from_input,
     set_arg_from_password,
     standardize_metric,
 )
-from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 
 PINECONE_MAX_K = 10_000
 MAX_TRIES_OVERALL = 150
@@ -281,7 +281,7 @@ def get_all_ids_from_index(
             ]
         if self.args["id_list_file"]:
             with open(self.args["id_list_file"]) as f:
-                return [line.strip() for line in f.readlines()]
+                return [line.strip() for line in f]
 
         if self.args.get("use_list_points", use_list_points_default):
             try:
@@ -334,9 +334,7 @@ def get_all_ids_from_index(
                     range_min = min(all_ids) - fetch_size
                     range_max = max(all_ids) + 10 * fetch_size
                     range_obj = range(range_min, range_max)
-                    tqdm.write(
-                        "Checking ids in range {} to {}".format(range_min, range_max)
-                    )
+                    tqdm.write(f"Checking ids in range {range_min} to {range_max}")
                     ids_to_fetch = [
                         x
                         for x in list(range_obj)

src/vdf_io/export_vdf/qdrant_export.py

@@ -1,14 +1,14 @@
 import argparse
 import json
-from typing import Dict, List
-from qdrant_client import QdrantClient
 import os
-from tqdm import tqdm
+
 from dotenv import load_dotenv
+from qdrant_client import QdrantClient
+from tqdm import tqdm
 
 from vdf_io.export_vdf.vdb_export_cls import ExportVDB
-from vdf_io.names import DBNames
 from vdf_io.meta_types import NamespaceMeta
+from vdf_io.names import DBNames
 from vdf_io.util import set_arg_from_input, set_arg_from_password
 
 load_dotenv()
@@ -83,15 +83,15 @@ def __init__(self, args):
             prefer_grpc=self.args.get("prefer_grpc", True),
         )
 
-    def get_all_index_names(self) -> List[str]:
+    def get_all_index_names(self) -> list[str]:
         """
         Get all collection names from Qdrant
         """
         collections = self.client.get_collections().collections
         collection_names = [collection.name for collection in collections]
         return collection_names
 
-    def get_index_names(self) -> List[str]:
+    def get_index_names(self) -> list[str]:
         """
         Get collection names from args or all collection names
         """
@@ -101,7 +101,7 @@ def get_index_names(self) -> List[str]:
 
     def get_data(self):
         collection_names = self.get_index_names()
-        index_metas: Dict[str, List[NamespaceMeta]] = {}
+        index_metas: dict[str, list[NamespaceMeta]] = {}
         for collection_name in tqdm(collection_names, desc="Fetching indexes"):
             index_meta = self.get_data_for_collection(collection_name)
             index_metas[collection_name] = index_meta
@@ -136,7 +136,7 @@ def try_scroll(self, fetch_size, collection_name, next_offset):
             )
             return self.try_scroll((fetch_size * 2) // 3, collection_name, next_offset)
 
-    def get_data_for_collection(self, collection_name) -> List[NamespaceMeta]:
+    def get_data_for_collection(self, collection_name) -> list[NamespaceMeta]:
         vectors_directory = self.create_vec_dir(collection_name)
 
         total = self.client.get_collection(collection_name).vectors_count

src/vdf_io/export_vdf/turbopuffer_export.py

@@ -1,14 +1,15 @@
 import json
 import os
 import sys
-from typing import Dict, List
-from tqdm import tqdm
+
 import turbopuffer as tpuf
+from tqdm import tqdm
+
 from vdf_io.constants import DISK_SPACE_LIMIT
+from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 from vdf_io.meta_types import NamespaceMeta
 from vdf_io.names import DBNames
 from vdf_io.util import set_arg_from_input, set_arg_from_password
-from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 
 
 class ExportTurbopuffer(ExportVDB):
@@ -66,7 +67,7 @@ def get_data(self):
         ids = []
         vectors = {}
         metadata = {}
-        index_metas: Dict[str, List[NamespaceMeta]] = {}
+        index_metas: dict[str, list[NamespaceMeta]] = {}
         self.total_imported_count = 0
         for ns_name in tqdm(
             namespace_names, desc="Exporting turbopuffer namespaces (indexes)"

src/vdf_io/export_vdf/txtai_export.py

@@ -1,6 +1,6 @@
+from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 from vdf_io.names import DBNames
 from vdf_io.util import set_arg_from_input
-from vdf_io.export_vdf.vdb_export_cls import ExportVDB
 
 
 class ExportTxtai(ExportVDB):