fixing osti and o11 and adding tests

.github/workflows/test_oceans11.yml

@@ -0,0 +1,36 @@
+name: oceans11.py test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.11']
+
+    steps:
+    - uses: actions/checkout@v5
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install . -r requirements.extras.txt
+
+    - name: Test OCEANS11 Backend
+      run: |
+        pip install pytest
+        pytest dsi/backends/tests/test_oceans11.py -v --tb=short -m "not integration"
+      env:
+        PYTHONUNBUFFERED: 1

.github/workflows/test_osti.yml

@@ -0,0 +1,36 @@
+name: osti.py test
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.11']
+
+    steps:
+    - uses: actions/checkout@v5
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install . -r requirements.extras.txt
+
+    - name: Test OSTI Backend
+      run: |
+        pip install pytest
+        pytest dsi/backends/tests/test_osti.py -v --tb=short -m "not integration"
+      env:
+        PYTHONUNBUFFERED: 1

.github/workflows/test_sqlite.yml

@@ -25,10 +25,8 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.txt
-        pip install -r requirements.extras.txt
-        pip install .
+        pip install . -r requirements.extras.txt
     - name: Test SQLite
       run: |
         pip install pytest
-        pytest dsi/backends/tests/test_sqlite.py
+        pytest dsi/backends/tests/test_sqlite.py -vv -s --maxfail=1 --full-trace --log-cli-level=INFO

dsi/backends/oceans11.py

@@ -38,12 +38,12 @@ class ValueObject:
     type : str
         {'table', 'column', 'cell'}
     """
-    def __init__(self):
-        self.t_name = ""
-        self.c_name = []
-        self.row_num = None
-        self.value = None
-        self.type = ""
+    def __init__(self, t_name, c_name, row_num, value, type):
+        self.t_name = t_name # ""
+        self.c_name = c_name #[]
+        self.row_num = row_num # None
+        self.value = value # None
+        self.type = type # ""
 
 # ----------------------------------------------------------------------
 # Oceans11 Backend (Webserver - Read only)
@@ -163,6 +163,7 @@ def validate_connection(self, **kwargs):
                     path=self.base_url,
                     abs_path_workspace_folder=self.workspace,
                     username="",
+                    download_limit=1024**5
                 )
 
             if info is None:
@@ -402,7 +403,8 @@ def _download_t2_db(self, t2db_url):
             location=full_url,
             path=full_url,
             abs_path_workspace_folder=self.workspace,
-            host_username=""
+            username="",
+            download_limit=1024**5
         )
 
         if info is None or not info.get("local_path"):
@@ -813,45 +815,13 @@ def find(self, query_object, **kwargs):
         if not self._loaded:
             return []
 
-        results = []
-
-        for table_name, table in self._cache.items():
-
-            columns = list(table.keys())
-
-            if not columns:
-                continue
-
-            num_rows = len(table[columns[0]])
-
-            for row_idx in range(num_rows):
-
-                row_values = []
-
-                matched = False
-
-                for col in columns:
+        query_str = str(query_object).lower()
 
-                    value = table[col][row_idx]
-                    row_values.append(value)
-
-                    if query_object is None:
-                        continue
-
-                    if value is not None and str(query_object).lower() in str(value).lower():
-                        matched = True
-
-                if matched:
-                    results.append(
-                        ValueObject(
-                            t_name=table_name,
-                            c_name=columns,
-                            row_num=row_idx + 1,
-                            value=row_values,
-                        )
-                    )
-
-        return results
+        return (
+            self.find_table(query_str) +
+            self.find_column(query_str) +
+            self.find_cell(query_object)
+        )
 
     def find_table(self, query_object, **kwargs):
         """
@@ -1290,4 +1260,4 @@ def ingest_artifacts(self, artifacts, **kwargs) -> None:
         **Not supported - Oceans11 backend is read-only**
         """
         raise NotImplementedError("Oceans11 backend is read-only")
-
+

dsi/backends/osti.py

@@ -6,6 +6,7 @@
 """
 
 import requests
+import operator
 import pandas as pd
 from urllib.parse import urlparse
 from collections import OrderedDict
@@ -180,8 +181,12 @@ def validate_connection(self):
 
             return True
 
-        except Exception:  # noqa: E722
+        except requests.RequestException as exc: # noqa: E722
             # Silent failure to allow external workflows to continue
+            print(
+                f"Unable to connect to OSTI API at {test_url}: "
+                f"{type(exc).__name__}: {exc}"
+            )
             return False
 
     # ---------------------------------------------------
@@ -578,9 +583,11 @@ def get_table_names(self, query):
             Query string to parse
 
         Return : list
-            List of dataset names/IDs found in query
+            OSTI is a single-table backend, so the only table is 'records'.
         """
-        raise NotImplementedError("OSTI backend has not implemented get_table_names")
+        if not self._loaded or "records" not in self._cache:
+            return []
+        return ["records"]
 
     # ---------------------------------------------------
     # Query Interface (in-memory)
@@ -894,10 +901,69 @@ def find_cell(self, query_object, **kwargs):
 
 
     def find_relation(self, column_name, relation, **kwargs):
+        """Find rows in the cached OSTI records table satisfying a column relation.
         """
-        Relation finding is not supported for the OSTI backend.
-        """
-        raise NotImplementedError("OSTI Backend does not support find_relation")
+        if not self._loaded:
+            return []
+
+        if "records" not in self._cache:
+            return []
+
+        ops = {
+            "=": operator.eq,
+            "==": operator.eq,
+            "!=": operator.ne,
+            ">": operator.gt,
+            "<": operator.lt,
+            ">=": operator.ge,
+            "<=": operator.le,
+        }
+
+        relation = relation.strip()
+
+        matched_op = None
+        matched_value = None
+
+        for op in sorted(ops.keys(), key=len, reverse=True):
+            if relation.startswith(op):
+                matched_op = op
+                matched_value = relation[len(op):].strip().strip("'\"")
+                break
+
+        if matched_op is None:
+            raise ValueError(f"Unsupported relation: {relation}")
+
+        table_name = "records"
+        table = self._cache[table_name]
+
+        if column_name not in table:
+            return []
+
+        compare = ops[matched_op]
+        columns = list(table.keys())
+        results = []
+
+        for row_idx, value in enumerate(table[column_name]):
+            try:
+                lhs = float(value)
+                rhs = float(matched_value)
+            except Exception:
+                lhs = str(value)
+                rhs = str(matched_value)
+
+            try:
+                if compare(lhs, rhs):
+                    val = ValueObject()
+                    val.t_name = table_name
+                    val.c_name = columns
+                    val.row_num = row_idx
+                    val.value = [table[c][row_idx] for c in columns]
+                    val.type = "row"
+                    results.append(val)
+            except Exception:
+                continue
+
+        return results
 
     # ----------------------------------------------------------------------
     # Utility / Display