From 45595875440845fc63cc981995748c975090ceab Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 15 Jun 2026 19:13:52 +0100 Subject: [PATCH 001/104] feat: OIDC device flow --- .claude/skills/review-pr/SKILL.md | 377 ++++++++++ CHANGELOG.rst | 43 ++ docs/api.rst | 43 ++ docs/auth.rst | 233 ++++++ docs/index.rst | 1 + docs/installation.rst | 8 +- examples/oidc_device_auth.py | 66 ++ setup.py | 2 +- src/questdb/auth/__init__.py | 92 +++ src/questdb/auth/_cache.py | 277 +++++++ src/questdb/auth/_device.py | 614 +++++++++++++++ src/questdb/auth/_discovery.py | 335 +++++++++ src/questdb/auth/_errors.py | 91 +++ src/questdb/auth/_http.py | 234 ++++++ src/questdb/auth/_questdb.py | 305 ++++++++ src/questdb/auth/_render.py | 336 +++++++++ test/test.py | 18 + test/test_auth.py | 1167 +++++++++++++++++++++++++++++ 18 files changed, 4240 insertions(+), 2 deletions(-) create mode 100644 .claude/skills/review-pr/SKILL.md create mode 100644 docs/auth.rst create mode 100644 examples/oidc_device_auth.py create mode 100644 src/questdb/auth/__init__.py create mode 100644 src/questdb/auth/_cache.py create mode 100644 src/questdb/auth/_device.py create mode 100644 src/questdb/auth/_discovery.py create mode 100644 src/questdb/auth/_errors.py create mode 100644 src/questdb/auth/_http.py create mode 100644 src/questdb/auth/_questdb.py create mode 100644 src/questdb/auth/_render.py create mode 100644 test/test_auth.py diff --git a/.claude/skills/review-pr/SKILL.md b/.claude/skills/review-pr/SKILL.md new file mode 100644 index 00000000..6b408b4a --- /dev/null +++ b/.claude/skills/review-pr/SKILL.md @@ -0,0 +1,377 @@ +--- +name: review-pr +description: Review a GitHub pull request against py-questdb-client (Cython + C-ABI) coding standards +argument-hint: [PR number or URL] [--level=0..3] +allowed-tools: Bash(gh *), Bash(git *), Read, Grep, Glob, Agent +--- + +Review the pull request `$ARGUMENTS`. + +## Review mindset + +You are a senior QuestDB engineer performing a blocking code review. `py-questdb-client` is mission-critical software: a **Cython** extension that wraps the **`c-questdb-client` (Rust) library** through its **C ABI**, and is used to ingest production data from customer Python applications. A bug here causes data loss, silent data corruption, segfaults that take down the host Python interpreter, reference-count leaks, or native memory leaks. There is zero tolerance for correctness issues, memory unsafety, refcount imbalance, GIL violations, or an FFI binding that disagrees with the C header it calls. Be critical, thorough, and opinionated. Your job is to catch problems before they ship, not to be nice. + +- **Assume nothing is correct until you've verified it.** Read surrounding code to understand context — don't just look at the diff in isolation. +- **The diff is a hint, not the boundary of the review.** The highest-value bugs almost always live at callsites outside the diff that depend on contracts the diff quietly changed (a `cdef` helper's error-return convention, a buffer's ownership, a `qdb_pystr_buf` arena's lifetime). Treat the diff as the entry point, not the scope. +- **Flag every issue you find**, no matter how small. Do not soften language or hedge. Say "this is wrong" not "this might be an issue". +- **Do not praise the code.** Skip "looks good", "nice work", "clever approach". Focus entirely on problems and risks. +- **Think adversarially.** For each change, work through: + - Inputs: which values break this? Empty buffers, zero-length strings, `None`, NaN/inf floats, boundary integers (`INT64_MAX`/`INT64_MIN`), max-length symbols, non-UTF-8 `str`, `bytes` with embedded NULs, huge `int` that overflows `int64_t`. + - Encoding: how does the code behave when a Python `str` contains lone surrogates, astral codepoints, or characters that fail UTF-8 encoding? + - Memory: every `malloc`/`calloc`/`realloc` — is it freed on the error path, the exception path, and the early-return path? Every `Py_INCREF` — is there a matching `Py_DECREF`? Every `PyObject_GetBuffer` — a matching `PyBuffer_Release`? + - GIL: does a `with nogil` block touch a Python object or call a CPython API function? Does a `cdef ... nogil` function need the GIL it doesn't hold? + - Failure modes: connection dropping mid-flush, partial write, TLS handshake failure, auth rejection, server rejection — does the buffer/sender end in a usable state, and does native memory get released? + - C-ABI callers: what happens when a C function returns `NULL`, returns an error via its out-param, or hands back a pointer the Cython side must free exactly once? +- **Check what's missing**, not just what's there. Missing tests, missing error handling, missing edge cases, missing `ingress.pyi` stub updates for public API changes, `.pxd` declarations out of sync with the C header. +- **Verify every claim.** If the PR title says "fix", verify the bug actually existed and the fix is correct. If it says "improve performance", look for benchmarks or reason about the change against the per-row hot path. If it says "simplify", verify the new code is actually simpler and doesn't drop behavior (e.g. a dropped `free` on an error branch). Treat the PR description as an unverified hypothesis. +- **Read the full context of changed files** when the diff alone is ambiguous. Use Read/Grep/Glob to inspect surrounding code, callers, and related tests. +- **Assess reachability before reporting.** For every potential bug, trace the actual callers and inputs. If a problem requires physically impossible conditions (a length larger than `SIZE_MAX`, a NUL injected through an API that already rejects it, a panic behind a validation guard), it is not a real finding — drop it. Focus on bugs that real workloads can trigger, not theoretical edge cases. +- **Never review generated or build artifacts.** `src/questdb/ingress.c`, `*.html` (Cython annotation), and `*.so` are build outputs. The source of truth is `*.pyx`, `*.pxi`, `*.pxd`, and `*.pyi`. If the diff contains a regenerated `ingress.c`, review the `.pyx`/`.pxi` change that produced it, not the generated C. + +## Review level + +Parse `$ARGUMENTS` for a level token: `--level=N`, `-lN`, or a bare single digit `0`-`3`. **If no level is given, default to 0.** Strip the level token before feeding the remainder (PR number or URL) to `gh` commands. + +The level controls how much of the review below actually runs. Lower levels keep the same review *spirit* — adversarial, blocking, no praise — but cut the breadth of the analysis. Higher levels have significantly higher token cost; reserve level 3 for high-stakes PRs (C-ABI `.pxd` changes, a `c-questdb-client` submodule bump, the dataframe/Arrow ingestion path, `nogil` sections, manual `malloc`/refcount code, ILP wire format, or auth/TLS configuration). + +| Level | What runs | +|-------|-----------| +| **0 (default)** | Steps 1, 2, 4. Skip Step 2.5. Skip Step 3 — no agent spawn; review the diff inline in the main loop, using Read/Grep on demand to resolve ambiguities. Skip Step 3b — verify each finding inline as you write it. Single-pass review covering correctness, Cython memory/refcount/GIL safety, C-ABI binding correctness, tests, and coding standards on the diff itself. | +| **1** | Adds Step 2.5a (semantic delta only — skip 2.5b/2.5c/2.5d). In Step 3, launch only Agent 1 (correctness), Agent 2 (Cython memory & refcount safety), and Agent 7 (tests) in parallel. Skip all other agents. Skip Step 3b — verify findings inline as you draft the report. | +| **2** | Full Step 2.5, but in 2.5b restrict the callsite inventory to public Python symbols (exported in `__all__` / `ingress.pyi`) plus every `cdef`/`cpdef` function and every C-ABI symbol declared in the `.pxd` files. In Step 3, launch Agents 1-8. Skip Agent 9 (cross-context) and Agent 10 (adversarial fresh-context). Step 3b uses a single batched verification agent for all findings instead of one per finding. | +| **3** | Every step below as written, all 10 agents, per-finding verification. The full mission-critical pass. | + +State the chosen level in one line at the start of the review so the user knows what they're getting (e.g., "Reviewing PR #141 at level 2"). If the level was defaulted, mention that level 3 exists for full review. + +## Step 1: Gather PR context + +Capture the PR identifier in `$PR` (the part of `$ARGUMENTS` left after stripping the level token), then fetch metadata, diff, and review comments in a single bash call so `$PR` is in scope for all three `gh` invocations: + +```bash +PR='' +gh pr view "$PR" --json number,title,body,labels,state +gh pr diff "$PR" +gh pr view "$PR" --comments +``` + +If the diff modifies `c-questdb-client` (the git submodule pointer) or any `.pxd` file, note it now — a submodule bump or binding change is the highest-risk class of change in this repo and forces level-3 scrutiny of the C-ABI surface regardless of the requested level. + +## Step 2: PR title and description + +Check: +- Title is clear and describes the change +- Description speaks to end-user impact, not implementation internals +- If fixing an issue, `Fixes #NNN` or a link to the issue is present +- Tone is level-headed and analytical +- For public API changes (anything in `__all__`, a new/changed method on `Sender`/`Buffer`/`Client`, a new keyword argument, or a changed default), the description calls out the API change explicitly, and `CHANGELOG.rst` is updated +- For a `c-questdb-client` submodule bump, the description states which upstream change is being pulled in and why + +## Step 2.5: Map the change surface + +Before launching review agents, produce a structured change surface map. This step is mandatory and must use Grep/Glob — do not reason about callsites from memory. The output of this step is required input for every agent in Step 3. + +### 2.5a Semantic delta per changed symbol + +For every modified or added function (`def`, `cdef`, `cpdef`), method, class, `cdef class` attribute, module-level constant, enum member, or C-ABI declaration in a `.pxd`, write: + +- **Symbol:** fully-qualified name (e.g., `questdb.ingress.Buffer.column`, `_dataframe`, `c_err_to_py`, `line_sender_buffer_column_f64`) +- **Before:** signature, return type, **Cython exception convention** (`except -1` / `except *` / `except? -1` / `except +` / none / `noexcept`), what it raises and on which inputs, `nogil`-ness, whether it touches Python objects, allocation behavior (`malloc`/`calloc`/`realloc`), refcount effect (does it steal/borrow/own a reference?), C-ABI ownership semantics (who frees returned pointers), thread-safety +- **After:** same fields +- **Delta:** one line stating what semantically changed + +"Refactored", "cleaned up", "improved", "simplified" are not acceptable deltas. State the actual behavioral difference. If nothing semantically changed, write "no behavioral change" — but only after checking, not as a default. + +### 2.5b Callsite inventory + +For every changed symbol that is public (in `__all__` / `ingress.pyi`), `cdef`/`cpdef`, declared in a `.pxd`, or a C-ABI function, run Grep across the repository to find every callsite, override, or reference outside the diff. + +Produce a list grouped by file. Search at minimum: + +- **Cython implementation & includes:** `grep -rn 'symbol_name' src/questdb/*.pyx src/questdb/*.pxi` +- **Cython C-ABI / helper declarations:** `grep -rn 'symbol_name' src/questdb/*.pxd` +- **Type stubs:** `grep -rn 'symbol_name' src/questdb/ingress.pyi` +- **C-ABI header (source of truth):** `grep -rn 'symbol_name' c-questdb-client/include/questdb/ingress/` +- **Rust helper crate:** `grep -rn 'symbol_name' rpyutils/src/ rpyutils/include/` +- **Unit & mock-server tests:** `grep -rn 'symbol_name' test/test.py test/mock_server.py test/test_tools.py` +- **System / integration tests:** `grep -rn 'symbol_name' test/system_test.py` +- **DataFrame tests, fuzz tests, leak tests:** `grep -rn 'symbol_name' test/test_dataframe.py test/test_client_dataframe_fuzz.py test/test_dataframe_fuzz.py test/test_dataframe_leaks.py test/test_client_capsule_path.py` +- **Examples:** `grep -rn 'symbol_name' examples/` +- **Docs:** `grep -rn 'symbol_name' docs/` + +A changed public / `cdef` / `.pxd` symbol with zero recorded Grep calls in the trace is a skill violation. The model is not allowed to assert "this is only used here" without showing the search. + +### 2.5c Implicit contract list + +For each changed symbol, walk this checklist and write one line per item, stating before vs after: + +- **Cython exception convention:** does the function return a C type with the right `except` clause? A `cdef` function returning `int`/`void`/a pointer with **no** `except` clause (or `noexcept`, the Cython 3 default for `nogil` functions) **silently swallows any Python exception raised inside it.** Did the convention change, and do all callers still propagate errors correctly? +- **Raises which exceptions on which inputs** (`IngressError`, `ValueError`, `TypeError`, `IngressServerRejectionError`, `UnsupportedDataFrameShapeError`) and which callers catch vs propagate them +- **Native memory:** does the symbol allocate (`malloc`/`calloc`/`realloc`) and who frees it? Does it free on every path including the exception path? +- **Reference counting:** does it `Py_INCREF`/`Py_DECREF`, store a borrowed `PyObject*`, hold a weakref/capsule, or return a borrowed vs owned reference? +- **Buffer protocol:** does it call `PyObject_GetBuffer` (and the matching `PyBuffer_Release`)? Does it keep the exporter alive while the raw pointer is in use? +- **GIL:** does it run under `nogil`? Does it release the GIL around a blocking C call (flush/connect)? Does it reacquire to raise? +- **C-ABI ownership:** does it pass a `line_sender_buffer`/`line_sender_utf8`/`qdb_pystr_buf` pointer into Rust, and who owns it afterward? Is a returned `line_sender_error*` freed exactly once (`line_sender_error_free`)? +- **`qdb_pystr_buf` arena lifetime:** are UTF-8 pointers obtained from the arena still valid after a subsequent `clear`/append (which may reallocate and invalidate earlier pointers)? +- **Buffer/sender state on error:** does a failed call leave the `Buffer` half-written, or the `Sender` in an unusable state requiring reconstruction? +- **`.pxd` ↔ C header agreement:** parameter types, `const`-ness, struct layout, enum discriminant order, return type — does the Cython declaration still match `c-questdb-client/include/questdb/ingress/*.h`? +- **`.pyi` ↔ implementation agreement:** does the stub still match the real signature, defaults, and return type? +- **Wire format:** any change to the ILP bytes produced (protocol v1 / v2), timestamp units, or column encoding. + +### 2.5d Cross-context exposure list + +End this step with an explicit list of "places this change is visible from but the diff does not touch". This is the highest-priority input for the bug-hunting agents in Step 3. + +Group the callsites from 2.5b by execution context. Typical contexts in this codebase: + +- **C-ABI binding surface:** every C-ABI function declared in `src/questdb/line_sender.pxd` / `conf_str.pxd` / `arrow_c_data_interface.pxd` / `mpdecimal_compat.pxd` / `rpyutils.pxd` that the changed code calls (transitively) +- **Buffer build hot path:** `Buffer.column`, `Buffer.symbol`, `Buffer.row`, `Buffer.at*`, and their `cdef` helpers +- **DataFrame / Arrow ingestion path:** everything in `dataframe.pxi`, the pandas/numpy/pyarrow/polars code paths, Arrow C Data Interface (`ArrowArray`/`ArrowSchema`/`ArrowArrayStream`) consumption and release callbacks, PyCapsule handling +- **Egress / query path:** `egress.pxi`, `QueryResult` +- **Flush path:** `Sender.flush`, `Buffer` → transport, the `with nogil` blocking sections +- **Auto-flush logic:** any callsite that triggers flush implicitly (row count / byte threshold / interval) +- **Configuration parsing:** `Sender.from_conf` / `from_env`, the `conf_str` parser, keyword-argument handling +- **Authentication / TLS:** auth token / basic-auth / TLS-CA configuration paths +- **`nogil` / threading surface:** the `active_senders` registry (`rpyutils/src/active_senders.rs`), any code reachable from multiple threads +- **`qdb_pystr_buf` arena users:** every function that obtains UTF-8 pointers from the per-`Buffer` string arena +- **Python type stubs:** `ingress.pyi` +- **Tests:** `test/test.py`, `test/system_test.py`, `test/test_dataframe.py`, fuzz and leak tests +- **Examples & docs:** `examples/*.py`, `docs/` + +Every entry on this list must be reviewed in Step 3. + +### 2.5e Build & binding profile facts + +**This sub-step runs at every level, including levels 0 and 1 where the rest of Step 2.5 is skipped.** A single Cython directive or a submodule bump can flip the safety story for the entire extension; agents must reason from the actual profile, not from defaults. + +Record, with file:line citations: + +- **Cython compiler directives** at the top of `ingress.pyx` and in `setup.py` (`language_level`, `binding`, and — if set — `boundscheck`, `wraparound`, `cdivision`, `initializedcheck`, `nonecheck`). If `boundscheck=False` / `wraparound=False`, **out-of-range or negative C-array/typed-memoryview indexing is undefined behavior, not an `IndexError`** — agents must treat indexing as a crash surface, not a guarded operation. +- **Cython exception-default fact:** in Cython 3, a `cdef`/`cpdef` function declared `nogil` (or any `cdef` returning a non-object type without an explicit `except` clause) defaults to `noexcept` — it **swallows Python exceptions silently**. Agents 1, 2, and 3 must check the actual `except` clause on every changed `cdef` and not assume exceptions propagate. +- **`c-questdb-client` submodule commit** (`git submodule status`) — if the diff moves it, the pinned commit's headers under `c-questdb-client/include/questdb/ingress/` are the *new* source of truth that every `.pxd` must match. Re-verify the `.pxd` ↔ `.h` agreement against the new commit. +- **`rpyutils` Rust crate:** if `rpyutils/src/**` or `rpyutils/Cargo.toml` changed, note its panic/profile behavior — a panic in `rpyutils` reached across the C ABI aborts the Python process. Its headers (`rpyutils/include/`, generated via `cbindgen.toml`) must match `rpyutils.pxd`. +- **Minimum numpy / Python versions** (`pyproject.toml`: `requires-python`, `numpy>=1.21.0`). Code that uses a newer numpy C-API or Python C-API symbol than the floor breaks the oldest supported build. State the floor. +- **`abort()` is imported** (`from libc.stdlib cimport ... abort`). Any reachable `abort()` call, or any Rust panic that crosses the C ABI, terminates the host interpreter with no traceback. Flag the path. + +A review without this section is incomplete. State the relevant facts (directives, exception default, submodule commit) in one line at the top of every Step 3 agent prompt so the agent reasons from the right premise. + +## Step 3: Parallel review + +Every agent receives: +1. The PR diff +2. The full change surface map from Step 2.5 (semantic deltas, callsite inventory, implicit contracts, cross-context exposure list, build & binding profile facts) + +### Anti-anchoring directive (applies to all agents) + +- **Bugs at callsites outside the diff outrank bugs inside the diff.** A confirmed bug in a file the PR did not touch but that calls a changed symbol is a P0 finding. +- **"Looks correct in isolation" is not a valid conclusion.** Before clearing a changed symbol, the agent must walk the callsite inventory from 2.5b and explicitly state, per callsite, whether the new behavior is still correct there. +- **The diff is the entry point, not the scope.** If the change surface map shows the symbol is reachable from N other files, the review covers N+1 files. +- **Project-wide settings affect untouched code.** A change to a Cython directive in `ingress.pyx` or `setup.py` (e.g. flipping `boundscheck` off), a `c-questdb-client` submodule bump, or a `.pxd` declaration change retroactively changes the safety/ABI story for **every** function that compiles under that directive or calls that binding — not just the diff. When directives, `setup.py`, `pyproject.toml`, or `.pxd`/submodule pointers appear in the diff, the review covers the affected surface of the whole extension, not just the touched lines. +- A single finding of the form "in `dataframe.pxi` the new behavior of `Buffer.column` leaks `b.validity` on the exception path" is worth more than five findings inside the diff. + +### Agents + +Launch the following agents in parallel. + +**Agent 1 — Correctness & bugs:** `None`/NULL handling, edge cases, logic errors, off-by-one, operator precedence, error paths. Integer correctness across the Python↔C boundary: Python `int` → `int64_t`/`size_t` conversion and overflow, `` / `` / `` casts that truncate or wrap, signed/unsigned mismatches, negative-length math. NaN/inf float handling. Timestamp unit conversions (micros vs nanos). Correct ILP wire format (v1 / v2). Cross-reference every changed symbol against its callsite inventory and verify the new behavior is correct at each callsite. + +**Agent 2 — Cython memory, refcount & crash surface:** In a Cython extension, anything that corrupts memory or aborts the native side takes down the host Python interpreter with no traceback. Flag every reachable instance of: + +- **Native memory leaks / double-free / use-after-free:** every `malloc`/`calloc`/`realloc` must be `free`d on **all** paths — success, early `return`, and the exception/`except` path (prefer `try/finally`). A `realloc` whose return value is assigned back to the same pointer leaks the original on failure (it returns `NULL` without freeing). Freeing a pointer twice, or using it after `free`, corrupts the heap. +- **Reference-count errors:** every `Py_INCREF` needs a matching `Py_DECREF` on all paths; a missing `DECREF` leaks, an extra `DECREF` causes a later use-after-free crash. Borrowed references (`PyWeakref_GetObject`, dict/list borrows, `PyObject*` stored without incref) must not outlive their owner. Verify `PyCapsule` and weakref handling. +- **Buffer-protocol imbalance:** every `PyObject_GetBuffer` must have a matching `PyBuffer_Release` on all paths, and the raw pointer must not be used after the exporting object can be collected. +- **Indexing under `boundscheck=False`:** per 2.5e, C-array and typed-memoryview indexing is unchecked — an out-of-range or negative index is UB, not an exception. Verify bounds are established before every index on the hot path. +- **Silent exception swallowing:** a `cdef` function returning a C type without the correct `except` clause (or `noexcept`) drops Python exceptions on the floor, turning an error into wrong data. Verify the `except` convention against what the body raises. +- **Direct aborts:** any reachable `abort()` (it is imported), and any **Rust panic crossing the C ABI** (from `c-questdb-client` or `rpyutils`) — both terminate the interpreter. The only defense is that the native side returns an error code/`line_sender_error*`, never panics. +- **Uninitialized memory:** a struct field or `malloc`'d region read before it is written (use `calloc` or explicit init), especially partially-built `pyobj_built_t`-style structs on an error path that then get freed. + +State the relevant build facts (directives, exception default, submodule commit) from 2.5e in the agent's first sentence, and evaluate every finding under the actual settings, not the textbook defaults. + +**Agent 3 — C-ABI boundary safety:** Check every call into the `c-questdb-client` / `rpyutils` C ABI. Verify: +- **`.pxd` matches the C header.** For every changed or called C-ABI symbol, read the actual declaration in `c-questdb-client/include/questdb/ingress/*.h` (or `rpyutils/include/`) and confirm the `.pxd` declaration matches it exactly: parameter types, pointer/`const`-ness, return type, struct field order and types, enum discriminant order. A mismatch is silent memory corruption / ABI breakage. If the submodule pointer moved, verify against the **new** pinned commit. +- **NULL handling:** every pointer returned from a C function checked before dereference; every pointer argument that could be `NULL` handled. +- **Error object lifecycle:** every `line_sender_error*` obtained via an out-param is converted (`c_err_to_py`) and freed exactly once (`line_sender_error_free`) — never leaked, never double-freed, never freed then read. +- **Ownership transfer:** `line_sender_buffer`, `line_sender_utf8`, `qdb_pystr_buf`, `line_sender` handles — who allocates, who frees, and is the lifetime correct relative to the owning `cdef class` (`__cinit__`/`__dealloc__`)? +- **`qdb_pystr_buf` arena invalidation:** UTF-8 pointers handed to Rust must remain valid until the buffer write completes and must not be invalidated by an intervening arena `clear`/append. +- **String encoding:** Python `str` → UTF-8 (`line_sender_utf8`), correct length passed, no lone surrogates, embedded-NUL handling, `bytes` vs `str` distinction. + +**Agent 4 — GIL & concurrency:** Verify: +- **`nogil` correctness:** no `with nogil` block (or `cdef ... nogil` function) touches a Python object, calls the CPython C-API, raises a Python exception, or `INCREF`/`DECREF`s — doing so without the GIL is a crash/corruption. Errors discovered under `nogil` must be deferred and raised after reacquiring the GIL. +- **GIL release around blocking calls:** the flush/connect/network C calls should release the GIL (`with nogil`) so other threads run; verify the released region doesn't reference Python state. +- **Thread-safety:** `Sender`, `Buffer`, and the `active_senders` registry (`rpyutils/src/active_senders.rs`) — verify documented thread-safety matches the implementation, and that shared mutable state reachable from multiple threads is synchronized. Cross-reference every callsite from 2.5b for violations of the concurrency contract. +- **Free-threaded build:** if the change assumes the GIL serializes access, note whether it holds under a free-threaded (no-GIL) CPython build (the CI matrix includes `*t` free-threaded targets). + +**Agent 5 — Resource management & lifecycle:** Leaks on all code paths (especially errors). Check `__cinit__`/`__dealloc__` pairing on every `cdef class` (does `__dealloc__` free everything `__cinit__` and methods allocated, and is it safe when `__cinit__` failed partway?). Native handle lifecycle (`line_sender`, `line_sender_buffer`, `qdb_pystr_buf`). Socket/connection/TLS teardown on error (handled by Rust, but verify the Cython side calls close/free). **Arrow C Data Interface:** `ArrowArray`/`ArrowSchema`/`ArrowArrayStream` `release` callbacks invoked exactly once; PyCapsule consumption semantics correct; no double-release. Walk every callsite from 2.5b that constructs, owns, or transfers ownership of a native handle and verify cleanup on all paths (success, exception, early return). + +**Agent 6 — Performance & allocations:** Unnecessary work on hot paths — the per-row buffer build (`Buffer.column`/`symbol`/`row`) and the per-column DataFrame loop (`dataframe.pxi`). Flag: Python-level operations (attribute lookups, `dict` access, object boxing, `str` re-encoding) inside the inner per-row/per-cell loop that should be hoisted or done at C level; allocations per row/cell that should be amortized; excessive copying of data that could be zero-copy via the buffer protocol / Arrow; O(n²) patterns over rows or columns. Analyze scaling at realistic volume: millions of rows per flush, hundreds of columns. Setup-path costs (sender construction, config parsing, schema inspection done once per DataFrame) are acceptable; per-row/per-cell costs are not. + +**Agent 7 — Test review & coverage:** Coverage gaps, error-path tests, `None`/edge-case tests, boundary conditions, regression tests, test quality. Check: +- Unit / mock-server tests in `test/test.py` (uses `test/mock_server.py`) +- System / integration tests against a real QuestDB in `test/system_test.py` +- DataFrame tests in `test/test_dataframe.py`, fuzz tests in `test/test_client_dataframe_fuzz.py` / `test/test_dataframe_fuzz.py`, and **leak tests** in `test/test_dataframe_leaks.py` (new native-memory or refcount handling should have a leak test) +- Capsule / Arrow path tests in `test/test_client_capsule_path.py` +- Examples in `examples/` still run (and `examples.manifest.yaml` is consistent) + +Cross-reference 2.5d: every cross-context exposure should have a test that exercises the changed symbol from that context. Missing tests for cross-context callsites — especially a new native-memory path without a leak test, or a new C-ABI binding without a system test — is a high-priority finding. + +**Agent 8 — Code quality & API design:** Public API ergonomics and consistency. **`ingress.pyi` stub must match the implementation** (signatures, defaults, return types, new symbols added to `__all__`). Docstrings on public classes/methods. `CHANGELOG.rst` updated for user-visible changes. Backward compatibility of the Python API (renamed/removed kwargs, changed defaults, changed exception types) — breaking changes must be intentional and called out in the PR body. Naming consistent with the codebase. No dead code, no unused `cimport`/`import`. Docs under `docs/` updated for API changes. + +**Agent 9 — Cross-context caller impact:** Walk the callsite inventory from 2.5b. For every callsite, fetch the surrounding code (the calling function plus its callers up two levels) and answer: + +- Does this caller pass inputs the new behavior handles incorrectly? +- Does this caller depend on a contract from the implicit contract list (2.5c) that the change broke — e.g. relying on the old `except` convention, the old ownership of a buffer, the old `qdb_pystr_buf` lifetime, the old refcount behavior? +- Is this caller in a context (a `with nogil` block, the per-row hot loop, an auto-flush trigger, an Arrow release callback, a `__dealloc__`, an exception/error path) where the new behavior misbehaves even if the inputs are valid? +- For a changed `cdef`/`cpdef` exception convention: do all callers still detect and propagate the error? +- For a changed C-ABI declaration: does the `.pxd` still match the C header, and do all Cython callers pass the right types/ownership? +- For a changed buffer/sender state machine: do all callers respect the new state transitions (buffer cleared after error before reuse; flush only when flushable)? + +This agent's output is structured per callsite, not per failure mode. Each callsite gets a verdict: SAFE / BROKEN / NEEDS VERIFICATION. Every BROKEN entry is a P0 finding regardless of whether the file is in the diff. + +This agent is not optional even when the diff is small. Small diffs to widely-used symbols (`Buffer.column`, `Sender.flush`, the dataframe entry point, a C-ABI binding) have the largest blast radius. + +**Agent 10 — Fresh-context adversarial:** Dispatched separately from agents 1-9 to escape checklist anchoring. This agent operates under different rules from the rest: + +- It receives ONLY the PR diff and the names of the changed files. It does NOT receive the change surface map from Step 2.5, the implicit contract list, the cross-context exposure list, or any of the review checklists below. +- Its sole instruction: "find ways this code is wrong". No category list, no failure-mode taxonomy, no project-specific style guide. +- It is free to use Read, Grep, and Glob to explore the repository however it wants. +- Findings are not pre-classified by category. Each finding states: what's wrong, why it's wrong, and the code path that demonstrates it. + +The point of this agent is to surface bugs the structured agents cannot see because they are reasoning inside the same frame. A finding here that none of agents 1-9 produced is high signal — it means the structured review missed it. A finding here that overlaps with agents 1-9 is corroboration. + +Run this agent in parallel with agents 1-9. It is mandatory regardless of diff size. + +Combine all agent findings into a single deduplicated **draft** report. Do NOT present this draft to the user yet — it goes straight into verification. + +## Step 3b: Verify every finding against source code + +The parallel review agents work from the diff plus the change surface map and frequently produce false positives — especially around native memory ownership, refcounting, GIL boundaries, Cython exception conventions, and C-ABI lifecycle. Every finding MUST be verified before it is reported. + +For each finding in the draft report: + +1. **Read the actual source code** at the exact lines cited (in the `.pyx`/`.pxi`/`.pxd`/`.pyi`, never the generated `ingress.c`). Do not rely on the agent's description alone. +2. **Trace the full code path:** follow callers and `cdef` helpers. Remember Cython's `include` model — `dataframe.pxi` and `egress.pxi` are textually included into `ingress.pyx`, so symbols are shared across them. +3. **Check both sides of the C ABI:** if a finding involves Cython↔Rust interaction, read both the Cython call and the C header in `c-questdb-client/include/questdb/ingress/` (or `rpyutils/include/`). Verify ownership transfer, error propagation, and freeing on both sides. +4. **For native-memory-leak claims:** trace every `malloc`/`calloc`/`realloc` to its `free` on ALL paths (success, early return, `except`/exception unwind). Confirm the intervening code can actually raise before claiming the exception path leaks. +5. **For refcount claims:** count `Py_INCREF`/`Py_DECREF` on every path; confirm borrowed-vs-owned reasoning against the CPython C-API contract of each function used. +6. **For exception-swallowing claims:** check the actual `except` clause on the `cdef` and whether the body can raise. Under Cython 3 a `nogil` `cdef` defaults to `noexcept` — confirm whether that's the real declaration. +7. **For GIL claims:** verify the cited code is actually inside a `nogil` region and actually touches a Python object / C-API; a `cdef` function called from `nogil` may itself acquire the GIL. +8. **For C-ABI / `.pxd` mismatch claims:** read the exact declaration in the pinned header and compare field-by-field. A claimed mismatch that actually matches is a false positive. +9. **For numeric overflow/truncation claims:** check reachability at realistic scale — ILP buffers up to a few hundred MB, millions of rows per flush, columns in the tens to low hundreds. Drop overflows that require values beyond that scale. +10. **For performance claims:** confirm the cost is on the per-row/per-cell hot path and measurable relative to surrounding I/O. Downgrade negligible savings to a nit. Exception: a per-row or per-cell allocation / Python-object operation on the buffer-build path is always worth flagging. +11. **For cross-context findings (Agent 9):** re-read the callsite in full, including callers up two levels, and confirm the broken behavior is reachable from production or test paths users will exercise. + +**Classify each finding** as: +- **CONFIRMED in-diff** — the bug is real and inside the diff +- **CONFIRMED at out-of-diff callsite** — the bug is in an unchanged file because the changed symbol is used there in a way that's now broken (cite the file and the contract from 2.5c that was violated) +- **FALSE POSITIVE** — the code is actually correct (explain why) +- **CONFIRMED with nuance** — the issue exists but is less severe than stated (explain) + +**Move false positives to a separate "Downgraded" section** at the end of the report. For each, give a one-line explanation of why it was dismissed. This lets the PR author verify the reasoning and catch verification mistakes. + +Launch verification agents in parallel where findings are independent. Each verification agent should read surrounding source files, not just the diff. + +## Review checklists + +Review the diff for: + +### Correctness & bugs +- `None`/NULL handling at API boundaries +- Edge cases and error paths +- Logic errors, off-by-one, incorrect bounds, wrong operator precedence +- Integer overflow/truncation across the Python↔C boundary (`int` → `int64_t`/`size_t`, ``/`` casts, signed/unsigned) +- Float edge cases (NaN, inf), timestamp unit conversions (micros vs nanos) +- Correct ILP wire format (v1 / v2) +- **Reachability expansion:** for each changed symbol, list the new contexts it can appear in (DataFrame path, `nogil` section, auto-flush, Arrow callback, error path) and verify it works in each. + +### Cython memory & refcount safety +- Every `malloc`/`calloc`/`realloc` freed on success, early-return, and exception paths (prefer `try/finally`); no double-free, no use-after-free; `realloc`-failure path doesn't leak the original +- Every `Py_INCREF` matched by `Py_DECREF`; borrowed references not outliving their owner; weakref/capsule handling correct +- Every `PyObject_GetBuffer` matched by `PyBuffer_Release`; exporter kept alive while the pointer is used +- Correct Cython `except` convention on every `cdef`/`cpdef` returning a C type (no silent exception swallowing; `noexcept` is the Cython-3 default for `nogil` `cdef`) +- No reachable `abort()`, and no Rust panic crossing the C ABI (both kill the interpreter) +- Indexing safe under the active `boundscheck`/`wraparound` directives +- No uninitialized struct/heap memory read (use `calloc` or init before use, especially on partially-built error paths) + +### C-ABI boundary +- `.pxd` declarations match `c-questdb-client/include/questdb/ingress/*.h` (and `rpyutils/include/`) exactly — types, `const`, struct layout, enum order, return type — against the **pinned** submodule commit +- All pointers returned from C checked for NULL before dereference +- Every `line_sender_error*` freed exactly once (`line_sender_error_free`), never double-freed or leaked +- Ownership semantics clear and correct (who allocates the handle, who frees it, lifetime vs the owning `cdef class`) +- `qdb_pystr_buf` arena pointers stay valid until consumed; not invalidated by an intervening `clear`/append +- String handling: `str` → UTF-8 with correct length, lone-surrogate rejection, embedded-NUL handling, `bytes`/`str` distinction +- ABI stability: a submodule bump that reorders a struct or renumbers an enum requires matching `.pxd` updates + +### GIL & concurrency +- No Python object access / C-API call / refcount op / raise inside a `with nogil` block or `cdef ... nogil` function +- GIL released around blocking network/flush C calls; released region references no Python state; errors deferred and raised after reacquiring +- `Sender`/`Buffer`/`active_senders` thread-safety matches documentation; shared mutable state synchronized +- Assumptions that the GIL serializes access re-checked for the free-threaded CPython build + +### Performance +- No per-row/per-cell Python-level operations (attribute/dict lookups, boxing, `str` re-encoding) in the buffer-build or DataFrame inner loops that belong at C level or hoisted to setup +- No per-row/per-cell allocations that should be amortized +- Zero-copy where possible (buffer protocol, Arrow) instead of copying +- No O(n²) over rows or columns at realistic scale (millions of rows, hundreds of columns) + +### Resource management +- `__cinit__`/`__dealloc__` pair frees everything allocated, and `__dealloc__` is safe after a partially-failed `__cinit__` +- Native handles (`line_sender`, `line_sender_buffer`, `qdb_pystr_buf`) released on all paths +- Socket/connection/TLS cleanup on error (Cython side invokes the Rust close/free) +- Arrow `release` callbacks invoked exactly once; PyCapsule consumed correctly; no double-release +- No leak through the C-ABI boundary (ownership documented and consistent) + +### Code quality +- `ingress.pyi` stub matches the implementation (signatures, defaults, return types, `__all__`) +- Public API consistent and ergonomic; backward-compatible (or breaking changes called out in the PR body) +- `CHANGELOG.rst` updated for user-visible changes; `docs/` updated for API changes +- Docstrings on public classes/methods +- Naming consistent with the codebase; no dead code or unused `import`/`cimport` + +### Test review +- **Coverage gaps:** every new/changed code path has a corresponding test; flag missing ones explicitly as "missing test for X" +- **Cross-context coverage:** every entry in the cross-context exposure list (2.5d) has a test exercising the changed symbol from that context +- **Leak coverage:** new native-memory or refcount-handling code has a test in `test/test_dataframe_leaks.py` (or equivalent) +- **Error-path coverage:** failure cases, partial writes, connection drops, TLS/auth failures, server rejections, and edge conditions tested — not just the happy path +- **Edge-case tests:** `None`, empty buffers, zero-length strings, max-length symbols, boundary integers, NaN/inf, non-UTF-8 strings +- **C-ABI / binding changes** covered by a system test in `test/system_test.py` +- **DataFrame / Arrow changes** covered in `test/test_dataframe.py` and the fuzz/capsule tests +- **Test quality:** tests assert the right thing; watch for trivially-passing tests +- **Regression tests:** a bug fix has a test that reproduces the original bug and fails without the fix + +### Unresolved TODOs and FIXMEs +- Scan the diff for `TODO`, `FIXME`, `HACK`, `XXX`, `WORKAROUND`. For each: + - Pre-existing (just moved/reformatted) or newly introduced in this PR? + - If new: unfinished work that should block merge, or an acceptable known limitation? Flag deferred bugs or incomplete implementations. + - If it references a ticket/issue, verify the reference exists. + +### Commit messages +- Plain English titles, under 50 chars +- Active voice, naming the acting subject + +## Step 4: Output + +Present ONLY verified findings (false positives are excluded from Critical/Moderate/Minor). Structure as: + +### Critical +Issues that must be fixed before merge. Each must include: +- Exact file path and line numbers (including out-of-diff files) +- Whether the finding is **in-diff** or **out-of-diff** +- Code path trace showing why the bug is real +- For out-of-diff findings: the contract from 2.5c that was violated and the callsite that triggers it +- Suggested fix + +### Moderate +Issues worth addressing but not blocking. + +### Minor +Style nits and suggestions. + +### Downgraded (false positives) +Findings from the initial review that were dismissed after source code verification. For each, state: +- The original claim (one line) +- Why it was dismissed (one line, citing the specific code that disproves it) + +### Summary +- One-line verdict: approve, request changes, or needs discussion +- Highlight any regressions or tradeoffs +- State how many draft findings were verified vs dropped as false positives (e.g., "8 findings verified, 4 false positives removed") +- State the in-diff vs out-of-diff split (e.g., "5 findings in-diff, 3 findings out-of-diff"). If the diff is non-trivial and out-of-diff is zero, the cross-context pass likely underran — re-invoke Agent 9 with a wider grep before finalizing. \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c3deae50..557b131c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,49 @@ Changelog ========= +Unreleased +---------- + +Features +~~~~~~~~ + +OIDC Authentication (:mod:`questdb.auth`) +************************************************ + +New :mod:`questdb.auth` module to sign in interactively to OIDC-secured +QuestDB Enterprise from Python — including from **remote** kernels +(JupyterHub, SageMaker, Colab, VS Code-remote) that have no local browser. + +It runs the OAuth 2.0 Device Authorization Grant (RFC 8628) client-side: you +authorize in any browser (laptop or phone), and the token is presented to +QuestDB over the auth paths it already supports (HTTP ``Bearer`` / PG-wire +``_sso``). No server change is required. + +.. code-block:: python + + from questdb.auth import OidcDeviceAuth, connect + + # Just the token (use it with PG-wire, HTTP, or any client): + auth = OidcDeviceAuth.from_questdb("https://questdb.example.com:9000") + token = auth.token() + + # Or the integrated session (query to a DataFrame, feed adapters): + qdb = connect("https://questdb.example.com:9000") + df = qdb.sql("SELECT * FROM trades LIMIT 10") + +Highlights: + +* Auto-discovery of OIDC config from the QuestDB ``/settings`` endpoint, with a + fallback to the IdP ``.well-known`` document. +* In-process token cache with silent refresh; optional on-disk cache. +* Adapters for pandas (REST ``/exec``), SQLAlchemy, psycopg and the ingestion + ``Sender``. +* ``token()`` / ``headers()`` require no dependencies beyond the standard + library; ``pandas`` / ``sqlalchemy`` / ``psycopg`` / ``qrcode`` / ``IPython`` + are imported lazily. + +See the :ref:`OIDC authentication guide ` for details. + 4.1.0 (2025-11-28) ------------------ diff --git a/docs/api.rst b/docs/api.rst index b3e1f11e..6b428050 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -67,3 +67,46 @@ questdb.ingress :members: :undoc-members: :show-inheritance: + +questdb.auth +============ + +See the :ref:`oidc_auth` guide for an overview. + +.. autofunction:: questdb.auth.connect + +.. autoclass:: questdb.auth.QuestDB + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: questdb.auth.OidcDeviceAuth + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: questdb.auth.OidcConfig + :members: + :undoc-members: + :show-inheritance: + +.. autoexception:: questdb.auth.OidcError + :show-inheritance: + +.. autoexception:: questdb.auth.OidcConfigError + :show-inheritance: + +.. autoexception:: questdb.auth.OidcInteractionRequired + :show-inheritance: + +.. autoexception:: questdb.auth.OidcDeviceFlowError + :show-inheritance: + +.. autoexception:: questdb.auth.OidcTimeoutError + :show-inheritance: + +.. autoexception:: questdb.auth.OidcAuthError + :show-inheritance: + +.. autoexception:: questdb.auth.OidcNetworkError + :show-inheritance: diff --git a/docs/auth.rst b/docs/auth.rst new file mode 100644 index 00000000..7e40ada5 --- /dev/null +++ b/docs/auth.rst @@ -0,0 +1,233 @@ +.. _oidc_auth: + +=================== +OIDC Authentication +=================== + +QuestDB Enterprise can be secured with `OpenID Connect (OIDC) +`_. The :mod:`questdb.auth` module +lets you sign in interactively from Python — including from a **remote** kernel +(JupyterHub, SageMaker, Colab, VS Code-remote, containers) where there is no +local browser. + +It runs the `OAuth 2.0 Device Authorization Grant (RFC 8628) +`_ entirely client-side: you +authorize in **any** browser (your laptop or your phone), while the kernel only +makes outbound calls to your identity provider (IdP). The resulting token is +then presented to QuestDB over the auth paths it already supports — HTTP +``Authorization: Bearer`` or PG-wire ``_sso`` — so **no server change is +required**. + +.. note:: + + This feature targets **QuestDB Enterprise with OIDC enabled**. The IdP + client referenced by ``acl.oidc.client.id`` must have the device grant + (``urn:ietf:params:oauth:grant-type:device_code``) enabled and be a public + client. See :ref:`oidc_idp_requirements`. + +Two ways to use it +================== + +You can let the helper drive everything, or you can just take the token and use +it with your own tooling. + +Just the token (PG-wire / HTTP / anything) +------------------------------------------ + +If you connect to QuestDB yourself — over PG-wire, raw HTTP, or any other +client — you only need a valid token. This path has **no extra dependencies**. + +.. code-block:: python + + from questdb.auth import OidcDeviceAuth + + # Discover the OIDC configuration from the QuestDB server: + auth = OidcDeviceAuth.from_questdb("https://questdb.example.com:9000") + + token = auth.token() # runs the device flow on first use, else cached + headers = auth.headers() # {"Authorization": "Bearer "} + + # Use the token however you like, e.g. PG-wire via psycopg: + import psycopg + conn = psycopg.connect( + host="questdb.example.com", port=8812, dbname="qdb", + user="_sso", password=token) + +The integrated session +---------------------- + +The high-level :func:`questdb.auth.connect` returns a :class:`~questdb.auth.QuestDB` +session that signs you in and adapts the token into the common Python access +paths. + +.. code-block:: python + + from questdb.auth import connect + + qdb = connect("https://questdb.example.com:9000") # interactive sign-in + df = qdb.sql("SELECT * FROM trades WHERE ts > dateadd('h', -1, now())") + + # Bring-your-own client, same auto-refreshed token: + engine = qdb.sqlalchemy_engine() # PG-wire, token as _sso + with qdb.psycopg() as conn: # raw psycopg + ... + with qdb.sender() as sender: # ingestion (ILP/HTTP) + sender.row("trades", columns={"price": 101.5}, + at=TimestampNanos.now()) + +On first use you will see a sign-in prompt (rendered as a clickable link in +Jupyter, plain text on a terminal):: + + 🔐 Sign in to QuestDB + Open https://idp.example.com/device and enter code: WDJB-MJHT + (or open directly: https://idp.example.com/device?user_code=WDJB-MJHT) + ⏳ waiting for authorization… (4:51 left) + ✅ Signed in as alice@example.com — token cached, expires in 60 min + +Re-running any cell is silent — the token is cached and refreshed silently on +the next use once it nears expiry. + +How it works +============ + +Configuration discovery +------------------------ + +:meth:`OidcDeviceAuth.from_questdb ` +(and :func:`~questdb.auth.connect`) resolve the OIDC configuration in this +order: + +1. ``GET {url}/settings`` (public, no auth) for the QuestDB-authoritative + values: ``acl.oidc.client.id``, ``acl.oidc.scope``, ``acl.oidc.token.endpoint``, + ``acl.oidc.groups.encoded.in.token`` and (on newer servers) + ``acl.oidc.device.authorization.endpoint``. +2. If the device-authorization endpoint is not advertised, the helper falls + back to the IdP discovery document + (``{issuer}/.well-known/openid-configuration``). The issuer is taken from an + explicit ``issuer=`` / ``discovery_url=`` argument, or derived from the token + endpoint's origin. + +Anything you pass explicitly overrides discovery. You can also skip discovery +entirely: + +.. code-block:: python + + auth = OidcDeviceAuth( + client_id="questdb", + device_authorization_endpoint="https://idp/.../device", + token_endpoint="https://idp/.../token", + scope="openid groups", + groups_in_token=True, # send id_token (True) vs access_token (False) + audience="questdb", # optional; some IdPs need it to set `aud` + cache="memory") + +Which token is sent +------------------- + +The helper mirrors QuestDB's own selection logic +(``groupsEncodedInToken ? idToken : accessToken``): + +============================================ ================= +``acl.oidc.groups.encoded.in.token`` Helper sends +============================================ ================= +``true`` ``id_token`` +``false`` ``access_token`` +============================================ ================= + +When sending the ``id_token`` the ``openid`` scope is requested automatically. + +Token lifecycle (cache + refresh) +--------------------------------- + +``token()`` returns the cached token while it is valid (with a small clock-skew +margin). When it nears expiry the helper silently refreshes it using the +``refresh_token`` if one was issued. If the refresh token is missing or rejected +(expired/revoked), it re-runs the interactive sign-in; a transient network error +is raised instead, so you can retry without being needlessly re-prompted. A lock +serializes refresh so parallel cells/threads don't double-prompt. + +Cache backends (``cache=`` argument): + +* ``"memory"`` *(default)* — process-global, nothing written to disk. + Re-running cells is silent; a kernel restart re-prompts once. +* ``"file"`` — ``~/.questdb/oidc-cache.json`` (mode ``600``). Survives kernel + restarts and is shared across kernels on the same host. **Security + trade-off:** the refresh token is stored at rest. +* ``None`` — never persist; prompt every time. + +Non-interactive contexts +------------------------- + +Scheduled / non-interactive notebooks (papermill, cron, CI) have no human to +authorize the device. The helper detects this and raises +:class:`~questdb.auth.OidcInteractionRequired` instead of hanging. Use a QuestDB +**service-account REST token** or the **client-credentials** grant there. + +Connection adapters +=================== + +* :meth:`QuestDB.sql ` — query over REST ``/exec`` to a + pandas DataFrame using ``Authorization: Bearer``. Recommended: there is no + token-length limit (a groups-encoded JWT can be several KB). +* :meth:`QuestDB.sqlalchemy_engine ` — + PG-wire engine that injects a fresh token as the ``_sso`` password for every + new connection. Requires ``acl.oidc.pg.token.as.password.enabled=true``. +* :meth:`QuestDB.psycopg ` — a raw psycopg / + psycopg2 connection. +* :meth:`QuestDB.sender ` — a + :class:`~questdb.ingress.Sender` for ingestion (ILP over HTTP). + +.. note:: + + QuestDB validates the token at **authentication** time, not per query. An + already-open PG connection survives token expiry; only **new** connections + need a fresh token — which is why the PG-wire adapter supplies the token + per-connect. + +.. _oidc_idp_requirements: + +IdP requirements +=============== + +The OIDC client referenced by ``acl.oidc.client.id`` must: + +* have the **Device Authorization grant** enabled; +* be a **public client** (no secret in a notebook); +* optionally issue **refresh tokens** for the device grant (for silent refresh); +* issue tokens whose ``aud`` matches ``acl.oidc.audience`` (some IdPs need an + ``audience``/``resource`` request parameter); +* include the **groups** claim in the token (``groups.encoded.in.token=true``) + or expose it via the **userinfo** endpoint (``false``), matching the server. + +Security notes +============= + +* No IdP passwords are ever entered in the notebook; MFA/SSO happen at the IdP. +* ``https`` is required. Plaintext ``http`` to a **loopback** address + (``localhost`` / ``127.0.0.1`` / ``::1``) is always allowed — it never leaves + the host. ``insecure=True`` additionally permits plaintext to a non-loopback + **QuestDB** host (local development only); it does **not** downgrade the + **IdP**, so the device code and refresh token are never sent in cleartext + over the network. Certificate verification is never disabled. +* **Endpoint trust.** The device code and the long-lived refresh token are sent + to the device-authorization and token endpoints, which are discovered from + QuestDB ``/settings``. The helper requires both endpoints to share a single + origin and rejects the configuration otherwise. Because ``/settings`` is + authoritative-by-QuestDB, a compromised server could in principle point them + elsewhere; pass ``issuer=`` (or ``discovery_url=``) to **pin** the IdP so the + endpoints are verified to belong to it and credentials can't be redirected to + another host. +* Adapters avoid logging the token / PG DSN. Avoid logging them yourself. +* Standard proxy / CA settings (``HTTPS_PROXY``, ``REQUESTS_CA_BUNDLE``, + ``SSL_CERT_FILE``) are honoured; you can also pass ``ca_bundle=``. + +Dependencies +=========== + +``token()`` / ``headers()`` need nothing beyond the standard library. The +following are imported lazily, only when used: + +* ``pandas`` — for :meth:`QuestDB.sql`; +* ``sqlalchemy`` and ``psycopg`` / ``psycopg2`` — for the PG-wire adapters; +* ``qrcode`` — to render a QR code for phone-based authorization (``qr=True``); +* ``IPython`` — for the rich Jupyter prompt (falls back to plain text). diff --git a/docs/index.rst b/docs/index.rst index 4540c9b5..6babf211 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,6 +14,7 @@ Contents installation sender conf + auth examples api troubleshooting diff --git a/docs/installation.rst b/docs/installation.rst index dc2f0405..fccd7599 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,7 +6,7 @@ Dependency ========== The Python QuestDB client does not have any additional run-time dependencies and -will run on any version of Python >= 3.9 on most platforms and architectures. +will run on any version of Python >= 3.10 on most platforms and architectures. From version 3.0.0, this library depends on ``numpy>=1.21.0``. @@ -23,6 +23,12 @@ These are bundled as the ``dataframe`` extra. Without this option, you may still ingest data row-by-row. +The :ref:`OIDC authentication helper ` (:mod:`questdb.auth`) needs +no extra dependencies for ``token()`` / ``headers()``. Some of its conveniences +import the following lazily, only when used: ``pandas`` (for ``sql()``), +``sqlalchemy`` and ``psycopg`` / ``psycopg2`` (PG-wire adapters), ``qrcode`` +(QR-code prompt) and ``IPython`` (rich Jupyter prompt). + PIP --- diff --git a/examples/oidc_device_auth.py b/examples/oidc_device_auth.py new file mode 100644 index 00000000..2d7d4535 --- /dev/null +++ b/examples/oidc_device_auth.py @@ -0,0 +1,66 @@ +""" +Interactive OIDC sign-in to QuestDB Enterprise from Python (e.g. a notebook). + +Runs the OAuth 2.0 Device Authorization Grant (RFC 8628) client-side: you +authorize in any browser (laptop or phone), while the code runs on a possibly +remote kernel that only makes outbound calls to your identity provider. + +This requires QuestDB Enterprise with OIDC enabled and an IdP client that has +the device grant enabled. It cannot run unattended (there is a human in the +loop), so it is not part of the automated example suite. +""" + +import sys + +from questdb.auth import connect, OidcDeviceAuth, OidcError +from questdb.ingress import TimestampNanos + + +QUESTDB_URL = 'https://questdb.example.com:9000' + + +def integrated(url: str = QUESTDB_URL): + """The high-level path: sign in, then query / ingest with one object.""" + # First call triggers the interactive device-flow sign-in; the token is + # cached, so re-running this is silent until it expires. + qdb = connect(url) + + # Query straight to a pandas DataFrame over REST (Authorization: Bearer). + df = qdb.sql("SELECT * FROM trades WHERE ts > dateadd('h', -1, now())") + print(df) + + # Feed the same auto-refreshed token into your existing tooling: + # engine = qdb.sqlalchemy_engine() # PG-wire, token as _sso password + # with qdb.psycopg() as conn: ... # raw psycopg + with qdb.sender() as sender: # ingestion (ILP over HTTP) + sender.row( + 'trades', + symbols={'symbol': 'ETH-USD', 'side': 'sell'}, + columns={'price': 2615.54, 'amount': 0.00044}, + at=TimestampNanos.now()) + + +def bring_your_own_client(url: str = QUESTDB_URL): + """The low-level path: you just want the token (PG-wire / HTTP / anything).""" + auth = OidcDeviceAuth.from_questdb(url) + + token = auth.token() # valid, auto-refreshed id/access token + headers = auth.headers() # {"Authorization": "Bearer "} + print('Authorization header ready:', 'Authorization' in headers) + + # e.g. hand the token to psycopg yourself over PG-wire: + # import psycopg + # conn = psycopg.connect(host='questdb.example.com', port=8812, + # dbname='qdb', user='_sso', password=token) + return token + + +def main(): + try: + integrated() + except OidcError as e: + sys.stderr.write(f'OIDC sign-in failed: {e}\n') + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 74438319..2f6ab44a 100755 --- a/setup.py +++ b/setup.py @@ -175,7 +175,7 @@ def readme(): name='questdb', version='4.1.0', platforms=['any'], - python_requires='>=3.8', + python_requires='>=3.10', install_requires=[], ext_modules = cythonize([ingress_extension()], annotate=True), cmdclass={'build_ext': questdb_build_ext}, diff --git a/src/questdb/auth/__init__.py b/src/questdb/auth/__init__.py new file mode 100644 index 00000000..e3768bca --- /dev/null +++ b/src/questdb/auth/__init__.py @@ -0,0 +1,92 @@ +################################################################################ +## ___ _ ____ ____ +## / _ \ _ _ ___ ___| |_| _ \| __ ) +## | | | | | | |/ _ \/ __| __| | | | _ \ +## | |_| | |_| | __/\__ \ |_| |_| | |_) | +## \__\_\\__,_|\___||___/\__|____/|____/ +## +## Copyright (c) 2014-2019 Appsicle +## Copyright (c) 2019-2024 QuestDB +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +""" +OIDC authentication helper for QuestDB (Jupyter-first). + +Runs the OAuth 2.0 Device Authorization Grant (RFC 8628) entirely client-side, +obtains a token, and presents it to QuestDB over the auth paths it already +supports (HTTP ``Bearer`` / PG-wire ``_sso``). Designed for data scientists on +local **and remote** kernels (JupyterHub, SageMaker, Colab, VS Code-remote), +where the kernel has no browser: you authorize in any browser (laptop or +phone), the kernel only makes outbound calls to the IdP. + +Two ways to use it, depending on your needs: + +* **Just the token** — works with anything (PG-wire, HTTP, your own tooling):: + + from questdb.auth import OidcDeviceAuth + + auth = OidcDeviceAuth.from_questdb("https://questdb.example.com:9000") + token = auth.token() # device flow on first use + headers = auth.headers() # {"Authorization": "Bearer .."} + +* **The integrated session** — query to a DataFrame and feed adapters:: + + from questdb.auth import connect + + qdb = connect("https://questdb.example.com:9000") + df = qdb.sql("SELECT * FROM trades LIMIT 10") + engine = qdb.sqlalchemy_engine() # PG-wire, token as _sso password + with qdb.sender() as sender: # ingestion (ILP/HTTP) + ... + +Only ``token()`` / ``headers()`` are needed for the bring-your-own-client path, +and they require no optional dependencies. ``pandas`` (for ``sql()``), +``sqlalchemy`` / ``psycopg`` (adapters), ``qrcode`` and ``IPython`` are imported +lazily, only when used. +""" + +from ._device import OidcDeviceAuth +from ._discovery import OidcConfig +from ._cache import TokenCache, TokenSet, FileCache, MemoryCache, NullCache +from ._errors import ( + OidcError, + OidcConfigError, + OidcNetworkError, + OidcInteractionRequired, + OidcDeviceFlowError, + OidcTimeoutError, + OidcAuthError, +) +from ._questdb import QuestDB, connect + +__all__ = [ + 'connect', + 'QuestDB', + 'OidcDeviceAuth', + 'OidcConfig', + 'TokenCache', + 'TokenSet', + 'MemoryCache', + 'FileCache', + 'NullCache', + 'OidcError', + 'OidcConfigError', + 'OidcNetworkError', + 'OidcInteractionRequired', + 'OidcDeviceFlowError', + 'OidcTimeoutError', + 'OidcAuthError', +] diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py new file mode 100644 index 00000000..858e113e --- /dev/null +++ b/src/questdb/auth/_cache.py @@ -0,0 +1,277 @@ +################################################################################ +## ___ _ ____ ____ +## / _ \ _ _ ___ ___| |_| _ \| __ ) +## | | | | | | |/ _ \/ __| __| | | | _ \ +## | |_| | |_| | __/\__ \ |_| |_| | |_) | +## \__\_\\__,_|\___||___/\__|____/|____/ +## +## Copyright (c) 2014-2019 Appsicle +## Copyright (c) 2019-2024 QuestDB +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +"""Token state and cache backends for :mod:`questdb.auth`.""" + +from __future__ import annotations + +import contextlib +import json +import os +import pathlib +import tempfile +import threading +from dataclasses import asdict, dataclass, replace +from typing import Dict, Optional, Union + +from ._errors import OidcConfigError + +# Refresh a little before the real expiry to absorb clock skew / latency. +DEFAULT_SKEW_SECONDS = 30 + + +@dataclass +class TokenSet: + """A set of tokens obtained from the IdP, plus their expiry.""" + + access_token: Optional[str] = None + id_token: Optional[str] = None + refresh_token: Optional[str] = None + expires_at: float = 0.0 # epoch seconds; 0 == unknown + token_type: str = 'Bearer' + scope: Optional[str] = None + sub: Optional[str] = None + issued_at: float = 0.0 # epoch seconds; 0 == unknown + + def is_valid(self, now: float, skew: float = DEFAULT_SKEW_SECONDS) -> bool: + """True if the token is present and not within ``skew`` of expiry.""" + if self.expires_at <= 0: + return False + # Never let the early-refresh skew exceed half the token's own + # lifetime, so a short-lived (< 2*skew) token isn't reported expired + # the instant it is issued (which would refresh on every call). + if self.issued_at: + lifetime = self.expires_at - self.issued_at + if lifetime > 0: + skew = min(skew, lifetime / 2) + return now < (self.expires_at - skew) + + def to_dict(self) -> Dict[str, object]: + return asdict(self) + + @classmethod + def from_dict(cls, d: Dict[str, object]) -> 'TokenSet': + known = {f for f in cls.__dataclass_fields__} # noqa: C416 + return cls(**{k: v for k, v in d.items() if k in known}) + + +class TokenCache: + """Interface for token caches.""" + + def load(self, key: str) -> Optional[TokenSet]: # pragma: no cover + raise NotImplementedError + + def store(self, key: str, tokens: TokenSet) -> None: # pragma: no cover + raise NotImplementedError + + def clear(self, key: str) -> None: # pragma: no cover + raise NotImplementedError + + +# Module-global so that re-running a notebook cell (which constructs a fresh +# ``OidcDeviceAuth``) reuses the already-acquired token instead of re-prompting. +_MEMORY_STORE: Dict[str, TokenSet] = {} +_MEMORY_LOCK = threading.Lock() + + +class MemoryCache(TokenCache): + """ + Process-global, in-memory cache (the default). + + Safest backend: nothing is written to disk. Tokens survive for the life + of the Python process, so re-running cells is silent, but a kernel + restart re-prompts once. + """ + + def load(self, key: str) -> Optional[TokenSet]: + # Return a copy so callers can't mutate the cached entry in place + # (the live token is refreshed/rotated independently). + with _MEMORY_LOCK: + tokens = _MEMORY_STORE.get(key) + return replace(tokens) if tokens is not None else None + + def store(self, key: str, tokens: TokenSet) -> None: + with _MEMORY_LOCK: + _MEMORY_STORE[key] = replace(tokens) + + def clear(self, key: str) -> None: + with _MEMORY_LOCK: + _MEMORY_STORE.pop(key, None) + + +class NullCache(TokenCache): + """Never persists anything; prompts every time.""" + + def load(self, key: str) -> Optional[TokenSet]: + return None + + def store(self, key: str, tokens: TokenSet) -> None: + pass + + def clear(self, key: str) -> None: + pass + + +# Cross-process file locking, used to serialize read-modify-write on the +# shared cache file. fcntl.flock (POSIX) also serializes across threads/ +# instances in one process (locks are per open file description). Where no OS +# primitive is available it degrades to a best-effort no-op; the atomic +# os.replace still guarantees readers never see a torn file. +try: + import fcntl + + def _lock_fd(fd: int) -> None: + fcntl.flock(fd, fcntl.LOCK_EX) + + def _unlock_fd(fd: int) -> None: + fcntl.flock(fd, fcntl.LOCK_UN) +except ImportError: # pragma: no cover - non-POSIX (e.g. Windows) + try: + import msvcrt + + def _lock_fd(fd: int) -> None: + try: + msvcrt.locking(fd, msvcrt.LK_LOCK, 1) + except OSError: + pass + + def _unlock_fd(fd: int) -> None: + try: + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + except OSError: + pass + except ImportError: # pragma: no cover + def _lock_fd(fd: int) -> None: + pass + + def _unlock_fd(fd: int) -> None: + pass + + +@contextlib.contextmanager +def _interprocess_lock(lock_path: pathlib.Path): + """Best-effort exclusive lock via a sidecar lock file.""" + fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR, 0o600) + try: + _lock_fd(fd) + try: + yield + finally: + _unlock_fd(fd) + finally: + os.close(fd) + + +class FileCache(TokenCache): + """ + Opt-in on-disk cache at ``~/.questdb/oidc-cache.json`` (mode ``600``). + + Survives kernel restarts and is shared across kernels on the same host. + Security trade-off: a refresh token is stored at rest. The file is created + owner-only (``0600``) from the start via an atomic temp-file replace, and a + sidecar lock file serializes concurrent read-modify-writes across kernels + so entries are not corrupted or lost. + """ + + def __init__(self, path: Optional[Union[str, os.PathLike]] = None): + if path is None: + path = pathlib.Path.home() / '.questdb' / 'oidc-cache.json' + self.path = pathlib.Path(path) + self._lock_path = self.path.with_name(self.path.name + '.lock') + + def _ensure_dir(self) -> None: + parent = self.path.parent + parent.mkdir(parents=True, exist_ok=True) + try: + os.chmod(parent, 0o700) + except OSError: + pass + + def _read_all(self) -> Dict[str, dict]: + try: + with open(self.path, 'r', encoding='utf-8') as f: + data = json.load(f) + if isinstance(data, dict): + return data + except (FileNotFoundError, ValueError, OSError): + pass + return {} + + def _write_all(self, data: Dict[str, dict]) -> None: + # Atomic, owner-only replace. mkstemp creates the file mode 0600 with a + # unique name, so concurrent writers never share a temp file and the + # refresh token is never group/world-readable, even briefly. + fd, tmp = tempfile.mkstemp( + dir=str(self.path.parent), prefix='.oidc-', suffix='.tmp') + try: + with os.fdopen(fd, 'w', encoding='utf-8') as f: + json.dump(data, f) + os.replace(tmp, self.path) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(tmp) + raise + + def load(self, key: str) -> Optional[TokenSet]: + # Lock-free: the atomic replace guarantees a complete file is read. + entry = self._read_all().get(key) + if isinstance(entry, dict): + try: + return TokenSet.from_dict(entry) + except TypeError: + return None + return None + + def store(self, key: str, tokens: TokenSet) -> None: + self._ensure_dir() + with _interprocess_lock(self._lock_path): + data = self._read_all() + data[key] = tokens.to_dict() + self._write_all(data) + + def clear(self, key: str) -> None: + self._ensure_dir() + with _interprocess_lock(self._lock_path): + data = self._read_all() + if key in data: + del data[key] + self._write_all(data) + + +_CacheSpec = Union[str, None, TokenCache] + + +def make_cache(spec: _CacheSpec) -> TokenCache: + """Resolve a cache spec (``"memory"`` / ``"file"`` / ``None`` / instance).""" + if isinstance(spec, TokenCache): + return spec + if spec is None or spec == 'none': + return NullCache() + if spec == 'memory': + return MemoryCache() + if spec == 'file': + return FileCache() + raise OidcConfigError( + f'Unknown cache backend {spec!r}; ' + "expected 'memory', 'file', None, or a TokenCache instance.") diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py new file mode 100644 index 00000000..45694c64 --- /dev/null +++ b/src/questdb/auth/_device.py @@ -0,0 +1,614 @@ +################################################################################ +## ___ _ ____ ____ +## / _ \ _ _ ___ ___| |_| _ \| __ ) +## | | | | | | |/ _ \/ __| __| | | | _ \ +## | |_| | |_| | __/\__ \ |_| |_| | |_) | +## \__\_\\__,_|\___||___/\__|____/|____/ +## +## Copyright (c) 2014-2019 Appsicle +## Copyright (c) 2019-2024 QuestDB +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +"""The OAuth 2.0 device authorization grant (RFC 8628) token manager.""" + +from __future__ import annotations + +import base64 +import binascii +import json +import threading +import time +import urllib.parse +import webbrowser +from typing import Any, Dict, Optional + +from ._cache import TokenSet, make_cache +from ._discovery import OidcConfig, resolve_config, validate_endpoint_origins +from ._errors import ( + OidcConfigError, + OidcDeviceFlowError, + OidcError, + OidcInteractionRequired, + OidcNetworkError, + OidcTimeoutError, +) +from ._http import build_ssl_context, post_form +from ._render import ( + Renderer, + _safe_link_url, + detect_interactive, + in_ipython_kernel, + make_renderer, +) + +DEVICE_CODE_GRANT = 'urn:ietf:params:oauth:grant-type:device_code' +REFRESH_GRANT = 'refresh_token' + +_VALID_FLOWS = ('auto', 'device', 'loopback') + +# A non-positive expires_in is non-conformant; treat it as "unknown". +_DEFAULT_EXPIRES_IN = 3600 + + +class _SystemClock: + """Real time source; the default for :class:`OidcDeviceAuth`.""" + sleep = staticmethod(time.sleep) + monotonic = staticmethod(time.monotonic) + now = staticmethod(time.time) + + +_SYSTEM_CLOCK = _SystemClock() + + +def _decode_jwt_claims(token: Optional[str]) -> Dict[str, Any]: + """ + Best-effort decode of a JWT payload **without signature verification**. + + Used only to show a friendly identity in the sign-in message. QuestDB + performs the real validation. Returns ``{}`` for opaque/invalid tokens. + """ + if not token or token.count('.') < 2: + return {} + try: + payload = token.split('.')[1] + payload += '=' * (-len(payload) % 4) # restore base64 padding + raw = base64.urlsafe_b64decode(payload.encode('ascii')) + claims = json.loads(raw) + return claims if isinstance(claims, dict) else {} + except (ValueError, binascii.Error, UnicodeDecodeError): + return {} + + +def _identity_from_claims(claims: Dict[str, Any]) -> Optional[str]: + for key in ('email', 'preferred_username', 'upn', 'name', 'sub'): + value = claims.get(key) + if value: + return str(value) + return None + + +class OidcDeviceAuth: + """ + Acquire and refresh an OIDC token via the device authorization grant. + + The token is presented to QuestDB over the auth paths it already + supports: HTTP ``Authorization: Bearer`` or PG-wire ``_sso`` (token as + password). The flow runs entirely client-side; QuestDB is never in the + token-acquisition path. + + Most users only ever call :meth:`token` (or :meth:`headers`). The first + call runs the interactive device flow; subsequent calls return the cached + token and refresh it silently (synchronously, on the first call made after + it nears expiry — there is no background thread). Acquisition is + serialized so concurrent callers don't double-prompt, while a valid cached + token is returned without blocking on another thread's in-progress + sign-in. + + .. code-block:: python + + from questdb.auth import OidcDeviceAuth + + # Discover everything from the QuestDB server: + auth = OidcDeviceAuth.from_questdb("https://questdb.example.com:9000") + token = auth.token() # device flow on first use, else cached + + Or fully explicit (no server discovery): + + .. code-block:: python + + auth = OidcDeviceAuth( + client_id="questdb", + device_authorization_endpoint="https://idp/.../device", + token_endpoint="https://idp/.../token", + scope="openid groups", + groups_in_token=True, + audience="questdb", + cache="memory") + """ + + def __init__( + self, + client_id: str, + device_authorization_endpoint: str, + token_endpoint: str, + *, + scope: str = 'openid', + groups_in_token: bool = True, + audience: Optional[str] = None, + issuer: Optional[str] = None, + cache: Any = 'memory', + insecure: bool = False, + ca_bundle: Optional[str] = None, + open_browser: bool = False, + interactive: Optional[bool] = None, + qr: bool = False, + renderer: Optional[Renderer] = None, + default_interval: int = 5, + _clock=None): # injectable time source for testing + if not client_id: + raise OidcConfigError('client_id is required') + if not device_authorization_endpoint: + raise OidcConfigError('device_authorization_endpoint is required') + if not token_endpoint: + raise OidcConfigError('token_endpoint is required') + + # Sending the id_token requires the ``openid`` scope. + if groups_in_token and 'openid' not in scope.split(): + scope = ('openid ' + scope).strip() + + self.config = OidcConfig( + client_id=client_id, + token_endpoint=token_endpoint, + device_authorization_endpoint=device_authorization_endpoint, + scope=scope, + groups_in_token=groups_in_token, + audience=audience, + issuer=issuer) + + # Enforce the credential-endpoint co-location / issuer pin on every + # construction path (not just discovery), so the documented guarantee + # holds for the explicit constructor too. + validate_endpoint_origins( + self.config.token_endpoint, + self.config.device_authorization_endpoint, + self.config.issuer) + + # `insecure` permits plaintext http only to QuestDB (e.g. a local dev + # server). The IdP is always held to https — or loopback http — by + # _idp_post, so the device code / refresh token are never sent in + # cleartext over the network even when this is set. + self.insecure = insecure + self.open_browser = open_browser + self._interactive = interactive + self._default_interval = default_interval + self._cache = make_cache(cache) + self._ctx = build_ssl_context(ca_bundle) + self._renderer = renderer if renderer is not None else make_renderer(qr=qr) + # Serializes token *acquisition* (a silent refresh or the interactive + # sign-in) only. Concurrent callers are possible via the threaded + # SQLAlchemy/psycopg adapters: without this, several connections + # opening as the token expires would run overlapping refreshes, and + # with refresh-token rotation all but one would fail and force a + # spurious re-prompt. It is NOT held on the fast path, so a caller with + # a valid cached token never blocks behind another thread's sign-in. + self._lock = threading.Lock() + self._tokens: Optional[TokenSet] = None + clock = _clock or _SYSTEM_CLOCK + self._sleep = clock.sleep + self._monotonic = clock.monotonic + self._now = clock.now + + # -- construction ------------------------------------------------------- + + @classmethod + def from_questdb( + cls, + url: str, + *, + client_id: Optional[str] = None, + scope: Optional[str] = None, + audience: Optional[str] = None, + groups_in_token: Optional[bool] = None, + issuer: Optional[str] = None, + discovery_url: Optional[str] = None, + token_endpoint: Optional[str] = None, + device_authorization_endpoint: Optional[str] = None, + flow: str = 'auto', + cache: Any = 'memory', + insecure: bool = False, + ca_bundle: Optional[str] = None, + open_browser: bool = False, + interactive: Optional[bool] = None, + qr: bool = False, + renderer: Optional[Renderer] = None, + _clock=None) -> 'OidcDeviceAuth': # injectable time source + """ + Build an :class:`OidcDeviceAuth` by discovering config from QuestDB. + + Reads ``{url}/settings`` for the OIDC client id, scope, endpoints and + groups mode, falling back to the IdP ``.well-known`` document for the + device-authorization endpoint when QuestDB does not advertise it. + Any explicit keyword overrides discovery. + """ + _validate_flow(flow) + ctx = build_ssl_context(ca_bundle) + cfg = resolve_config( + questdb_url=url, + client_id=client_id, + scope=scope, + audience=audience, + groups_in_token=groups_in_token, + token_endpoint=token_endpoint, + device_authorization_endpoint=device_authorization_endpoint, + issuer=issuer, + discovery_url=discovery_url, + ctx=ctx, + insecure=insecure) + return cls( + client_id=cfg.client_id, + device_authorization_endpoint=cfg.device_authorization_endpoint, + token_endpoint=cfg.token_endpoint, + scope=cfg.scope, + groups_in_token=cfg.groups_in_token, + audience=cfg.audience, + issuer=cfg.issuer, + cache=cache, + insecure=insecure, + ca_bundle=ca_bundle, + open_browser=open_browser, + interactive=interactive, + qr=qr, + renderer=renderer, + _clock=_clock) + + # -- public API --------------------------------------------------------- + + def token(self) -> str: + """ + Return a valid token for QuestDB, acquiring or refreshing as needed. + + Returns the ``id_token`` when the server expects groups encoded in the + token (``acl.oidc.groups.encoded.in.token=true``), otherwise the + ``access_token`` — mirroring QuestDB's own selection logic. + """ + return self._select(self._obtain_tokens()) + + def headers(self) -> Dict[str, str]: + """Return ``{"Authorization": "Bearer "}``.""" + return {'Authorization': f'Bearer {self.token()}'} + + @property + def cache_key(self) -> str: + """ + Identifies the token's security context for caching. + + Two sessions share a cached token only when they would accept the same + one: same IdP token endpoint (**path included**, so multi-tenant realms + sharing a host don't collide), client id, scope *set* (order-insensitive), + and audience. The QuestDB URL is deliberately excluded — the same IdP + token is valid against any QuestDB that trusts it. + """ + c = self.config + scope = ' '.join(sorted(c.scope.split())) if c.scope else '' + return '\x1f'.join([ + c.issuer or '', + _normalize_url(c.token_endpoint), + c.client_id, + scope, + c.audience or '']) + + def clear(self) -> None: + """Forget the cached token (forces a fresh sign-in next time).""" + # Serialize against acquisition so a concurrent refresh/sign-in can't + # re-populate the cache right after we clear it. + with self._lock: + self._tokens = None + self._cache.clear(self.cache_key) + + # -- token lifecycle ---------------------------------------------------- + + def _select(self, tokens: TokenSet) -> str: + if self.config.groups_in_token: + if not tokens.id_token: + raise OidcConfigError( + 'Server expects groups encoded in the token but the IdP ' + 'returned no id_token. Ensure the "openid" scope is ' + 'requested (current scope: ' + f'{self.config.scope!r}).') + return tokens.id_token + if not tokens.access_token: + raise OidcConfigError('IdP returned no access_token.') + return tokens.access_token + + def _has_required_token(self, tokens: TokenSet) -> bool: + """ + True if ``tokens`` carries the kind :meth:`_select` will return — the + ``id_token`` when groups are encoded in the token, else the + ``access_token``. The cache gate and the post-refresh check share this + predicate so they can't disagree with ``_select``. + """ + if self.config.groups_in_token: + return bool(tokens.id_token) + return bool(tokens.access_token) + + def _obtain_tokens(self) -> TokenSet: + # Fast path: return a valid cached token without taking the lock, so a + # caller with a usable token never blocks behind another thread's + # in-progress refresh or interactive sign-in. + tokens = self._valid_cached() + if tokens is not None: + return tokens + # Slow path: serialize acquisition so concurrent callers don't run + # overlapping refreshes or double-prompt; the loser re-checks and + # reuses the winner's freshly acquired token. + with self._lock: + tokens = self._valid_cached() + if tokens is not None: + return tokens + return self._acquire() + + def _valid_cached(self) -> Optional[TokenSet]: + tokens = self._tokens + if tokens is None: + tokens = self._cache.load(self.cache_key) + if tokens is not None: + self._tokens = tokens + if (tokens is not None and tokens.is_valid(self._now()) + and self._has_required_token(tokens)): + return tokens + return None + + def _acquire(self) -> TokenSet: + # Called while holding self._lock. Try a silent refresh, else run the + # interactive device flow. + tokens = self._tokens + if tokens is not None and tokens.refresh_token: + try: + refreshed = self._refresh(tokens) + except OidcNetworkError: + # Transient connectivity failure: the refresh token is still + # valid, so re-authenticating won't help (the interactive flow + # needs the same network) and would needlessly re-prompt. + # Surface it — the cached token + refresh_token are kept, so a + # later call retries the refresh. + raise + except OidcError: + # The refresh token was rejected (expired/revoked) or the IdP + # returned an unusable response: fall through to a fresh + # interactive sign-in. + pass + else: + # Only accept a refresh that actually yields the token kind we + # need. Some IdPs don't re-issue the id_token on refresh; such + # a response is unusable, so fall through to the interactive + # flow rather than caching it and looping on every call. + if self._has_required_token(refreshed): + self._store(refreshed) + return refreshed + + fresh = self._run_device_flow() + self._store(fresh) + return fresh + + def _store(self, tokens: TokenSet) -> None: + self._tokens = tokens + self._cache.store(self.cache_key, tokens) + + def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: + try: + expires_in = int(body.get('expires_in', _DEFAULT_EXPIRES_IN)) + except (TypeError, ValueError): + expires_in = _DEFAULT_EXPIRES_IN + if expires_in <= 0: + # A non-positive lifetime would mark a just-issued token as already + # expired, causing refresh/re-prompt churn. Treat it as unknown. + expires_in = _DEFAULT_EXPIRES_IN + claims = (_decode_jwt_claims(body.get('id_token')) + or _decode_jwt_claims(body.get('access_token'))) + now = self._now() + return TokenSet( + access_token=body.get('access_token'), + id_token=body.get('id_token'), + refresh_token=body.get('refresh_token'), + expires_at=now + expires_in, + issued_at=now, + token_type=body.get('token_type', 'Bearer'), + scope=body.get('scope', self.config.scope), + sub=claims.get('sub')) + + def _idp_post(self, url: str, form: Dict[str, Any]): + # IdP POSTs carry the device code / refresh token, so they are always + # required to be https (loopback http is fine for local dev); the + # user's `insecure` flag — which is about the QuestDB link — never + # downgrades them. + return post_form(url, form, ctx=self._ctx, insecure=False) + + def _refresh(self, tokens: TokenSet) -> TokenSet: + status, body = self._idp_post( + self.config.token_endpoint, + { + 'grant_type': REFRESH_GRANT, + 'refresh_token': tokens.refresh_token, + 'client_id': self.config.client_id, + 'scope': self.config.scope, + }) + if status == 200: + refreshed = self._tokenset_from_response(body) + # Many IdPs do not rotate the refresh token; keep the old one. + if not refreshed.refresh_token: + refreshed.refresh_token = tokens.refresh_token + return refreshed + raise OidcDeviceFlowError( + f"Token refresh failed: {body.get('error', 'unknown error')}", + error=body.get('error'), + error_description=body.get('error_description')) + + # -- device flow (RFC 8628) --------------------------------------------- + + def _run_device_flow(self) -> TokenSet: + if not self._is_interactive(): + raise OidcInteractionRequired( + 'Interactive sign-in is required, but no interactive terminal ' + 'or notebook was detected (e.g. papermill / cron / CI). Use a ' + 'QuestDB service-account REST token or the OAuth2 ' + 'client-credentials grant for non-interactive contexts.') + + resp = self._request_device_code() + self._renderer.on_prompt(resp) + self._maybe_open_browser(resp) + tokens = self._poll_for_token(resp) + claims = (_decode_jwt_claims(tokens.id_token) + or _decode_jwt_claims(tokens.access_token)) + identity = _identity_from_claims(claims) + self._renderer.on_success( + identity, max(0.0, tokens.expires_at - self._now())) + return tokens + + def _request_device_code(self) -> Dict[str, Any]: + form = { + 'client_id': self.config.client_id, + 'scope': self.config.scope, + } + if self.config.audience: + form['audience'] = self.config.audience + status, body = self._idp_post( + self.config.device_authorization_endpoint, form) + if status == 200 and body.get('device_code') and body.get('user_code'): + return body + error = body.get('error') + if status in (400, 404, 405) or error in ( + 'invalid_client', 'unauthorized_client', + 'unsupported_grant_type'): + raise OidcDeviceFlowError( + 'The IdP rejected the device-authorization request ' + f'(HTTP {status}, error={error!r}). Ensure the OIDC client ' + f'{self.config.client_id!r} has the device grant ' + "('urn:ietf:params:oauth:grant-type:device_code') enabled and " + 'is registered as a public client.', + error=error, + error_description=body.get('error_description')) + raise OidcDeviceFlowError( + f'Device authorization request failed (HTTP {status}): ' + f'{body.get("error_description") or error or body}', + error=error, + error_description=body.get('error_description')) + + def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: + device_code = resp['device_code'] + try: + interval = max(1, int(resp.get('interval', self._default_interval))) + except (TypeError, ValueError): + interval = self._default_interval + try: + expires_in = int(resp.get('expires_in', 600)) + except (TypeError, ValueError): + expires_in = 600 + deadline = self._monotonic() + expires_in + + while True: + remaining = deadline - self._monotonic() + if remaining <= 0: + self._renderer.on_failure( + 'Code expired — run the cell again to retry.') + raise OidcTimeoutError( + 'The device code expired before authorization completed. ' + 'Run the sign-in again.', + error='expired_token') + self._renderer.on_waiting(remaining) + self._sleep(interval) + + status, body = self._idp_post( + self.config.token_endpoint, + { + 'grant_type': DEVICE_CODE_GRANT, + 'device_code': device_code, + 'client_id': self.config.client_id, + }) + + if status == 200 and body.get('access_token'): + return self._tokenset_from_response(body) + + error = body.get('error') + if error == 'authorization_pending': + continue + if error == 'slow_down': + interval += 5 + continue + if error == 'expired_token': + self._renderer.on_failure( + 'Code expired — run the cell again to retry.') + raise OidcTimeoutError( + 'The device code expired before authorization completed. ' + 'Run the sign-in again.', + error=error) + # access_denied or any other terminal error. + description = body.get('error_description') or error or 'unknown error' + self._renderer.on_failure(f'Sign-in failed: {description}') + raise OidcDeviceFlowError( + f'Device flow failed: {description}', + error=error, + error_description=body.get('error_description')) + + # -- helpers ------------------------------------------------------------ + + def _is_interactive(self) -> bool: + if self._interactive is not None: + return self._interactive + return detect_interactive() + + def _maybe_open_browser(self, resp: Dict[str, Any]) -> None: + # Never auto-open on a (possibly remote) notebook kernel; only do so + # for an explicitly opted-in local terminal session. + if not self.open_browser or in_ipython_kernel(): + return + # Only open an http(s) URL — never a javascript:/data: scheme from a + # malicious or MITM'd device response. + target = _safe_link_url( + resp.get('verification_uri_complete') + or resp.get('verification_uri') + or resp.get('verification_url')) + if target: + try: + webbrowser.open(target) + except Exception: + pass + + +def _validate_flow(flow: str) -> None: + if flow not in _VALID_FLOWS: + raise OidcConfigError( + f'Unknown flow {flow!r}; expected one of {_VALID_FLOWS}.') + if flow == 'loopback': + raise OidcConfigError( + "The 'loopback' (Authorization Code + PKCE) flow is not yet " + "implemented. Use flow='device' (works on local and remote " + 'kernels alike).') + + +def _normalize_url(url: str) -> str: + # Full URL with scheme/host lower-cased and the default port dropped, but + # the path kept (it distinguishes multi-tenant realms). Used for the cache + # key so trivial spelling differences don't cause a spurious re-prompt. + parts = urllib.parse.urlparse(url) + scheme = (parts.scheme or '').lower() + host = (parts.hostname or '').lower() + default_port = {'https': 443, 'http': 80}.get(scheme) + if parts.port and parts.port != default_port: + netloc = f'{host}:{parts.port}' + else: + netloc = host + query = f'?{parts.query}' if parts.query else '' + return f'{scheme}://{netloc}{parts.path}{query}' diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py new file mode 100644 index 00000000..a1cccbe8 --- /dev/null +++ b/src/questdb/auth/_discovery.py @@ -0,0 +1,335 @@ +################################################################################ +## ___ _ ____ ____ +## / _ \ _ _ ___ ___| |_| _ \| __ ) +## | | | | | | |/ _ \/ __| __| | | | _ \ +## | |_| | |_| | __/\__ \ |_| |_| | |_) | +## \__\_\\__,_|\___||___/\__|____/|____/ +## +## Copyright (c) 2014-2019 Appsicle +## Copyright (c) 2019-2024 QuestDB +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +""" +OIDC configuration discovery. + +Resolution order, mirroring the design doc: + +1. ``GET {questdb_url}/settings`` (public, no auth) -> the QuestDB-authoritative + ``acl.oidc.*`` values (client id, scope, endpoints, groups mode). +2. If the device-authorization endpoint is not advertised by QuestDB (today's + servers), fall back to the IdP discovery document + (``{issuer}/.well-known/openid-configuration``). +""" + +from __future__ import annotations + +import ssl +import urllib.parse +from dataclasses import dataclass +from typing import Any, Dict, Optional + +from ._errors import OidcConfigError +from ._http import get_json + +# QuestDB /settings keys (see EntPropServerConfiguration.exportConfiguration()). +_K_ENABLED = 'acl.oidc.enabled' +_K_CLIENT_ID = 'acl.oidc.client.id' +_K_SCOPE = 'acl.oidc.scope' +_K_TOKEN_ENDPOINT = 'acl.oidc.token.endpoint' +_K_AUTHORIZATION_ENDPOINT = 'acl.oidc.authorization.endpoint' +_K_DEVICE_ENDPOINT = 'acl.oidc.device.authorization.endpoint' # design §7 (new) +_K_GROUPS_IN_TOKEN = 'acl.oidc.groups.encoded.in.token' +_K_AUDIENCE = 'acl.oidc.audience' +_K_HOST = 'acl.oidc.host' +_K_PORT = 'acl.oidc.port' +_K_TLS_ENABLED = 'acl.oidc.tls.enabled' + + +@dataclass +class OidcConfig: + """Resolved OIDC parameters needed to run the device flow.""" + + client_id: str + token_endpoint: str + device_authorization_endpoint: str + scope: str = 'openid' + groups_in_token: bool = True + audience: Optional[str] = None + issuer: Optional[str] = None + authorization_endpoint: Optional[str] = None + + +def _as_bool(value: Any, default: Optional[bool] = None) -> Optional[bool]: + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return bool(value) + if isinstance(value, str): + v = value.strip().lower() + if v in ('true', '1', 'yes', 'on'): + return True + if v in ('false', '0', 'no', 'off', ''): + return False + return default + + +def settings_config(settings: Any) -> Dict[str, Any]: + """ + Return the flat config map from a ``/settings`` response. + + Modern servers nest values under a ``"config"`` object; older ones return + them at the top level. We tolerate both. + """ + if isinstance(settings, dict): + cfg = settings.get('config') + if isinstance(cfg, dict): + return cfg + return settings + return {} + + +def fetch_settings( + questdb_url: str, + *, + ctx: Optional[ssl.SSLContext] = None, + insecure: bool = False, + timeout: float = 30) -> Dict[str, Any]: + """Fetch and return the QuestDB ``/settings`` config map.""" + base = questdb_url.rstrip('/') + data = get_json(base + '/settings', ctx=ctx, insecure=insecure, + timeout=timeout) + return settings_config(data) + + +def _origin(url: str) -> Optional[str]: + parts = urllib.parse.urlparse(url) + if parts.scheme and parts.netloc: + return f'{parts.scheme}://{parts.netloc}' + return None + + +_DEFAULT_PORTS = {'https': 443, 'http': 80} + + +def _normalized_origin(url: str) -> tuple: + """(scheme, host, port) with default ports filled in, for comparison.""" + parts = urllib.parse.urlparse(url) + scheme = (parts.scheme or '').lower() + host = (parts.hostname or '').lower() + port = parts.port or _DEFAULT_PORTS.get(scheme) + return (scheme, host, port) + + +def _origin_str(url: str) -> str: + scheme, host, port = _normalized_origin(url) + return f'{scheme}://{host}:{port}' if port else f'{scheme}://{host}' + + +def validate_endpoint_origins( + token_endpoint: str, + device_authorization_endpoint: str, + issuer: Optional[str] = None) -> None: + """ + Reject an OIDC configuration that would send credentials off-origin. + + The device code and the long-lived refresh token are POSTed to the device- + authorization and token endpoints. These come from QuestDB ``/settings`` + (or the IdP ``.well-known``), which the client trusts; this check limits a + tampered or MITM'd configuration from redirecting those credentials to an + attacker-controlled host: + + * the two credential endpoints must share a single origin (they are always + co-located on the authorization server per RFC 8628); and + * when the ``issuer`` is known independently (passed explicitly or resolved + from the IdP ``.well-known``), both endpoints must belong to it. + + Pass ``issuer=`` to pin the IdP explicitly when QuestDB advertises the + endpoints directly (so a compromised server cannot redirect the token POST). + """ + if _normalized_origin(token_endpoint) != _normalized_origin( + device_authorization_endpoint): + raise OidcConfigError( + 'OIDC token and device-authorization endpoints are on different ' + f'origins ({_origin_str(token_endpoint)} vs ' + f'{_origin_str(device_authorization_endpoint)}); refusing to send ' + 'credentials. This indicates a misconfigured or tampered OIDC ' + 'configuration.') + if issuer: + issuer_origin = _normalized_origin(issuer) + for label, url in ( + ('token endpoint', token_endpoint), + ('device-authorization endpoint', + device_authorization_endpoint)): + if _normalized_origin(url) != issuer_origin: + raise OidcConfigError( + f'OIDC {label} origin ({_origin_str(url)}) does not match ' + f'the issuer origin ({_origin_str(issuer)}); refusing to ' + 'send credentials to an endpoint outside the trusted ' + 'issuer.') + + +def _resolve_endpoint(value: Optional[str], cfg: Dict[str, Any]) -> Optional[str]: + """ + Turn a possibly-relative endpoint into a full URL. + + QuestDB usually exports fully-resolved URLs, but some deployments store + only the path (e.g. ``/as/token.oauth2``) alongside ``acl.oidc.host``. + """ + if not value: + return None + if value.startswith('http://') or value.startswith('https://'): + return value + if value.startswith('/'): + host = cfg.get(_K_HOST) + if host: + tls = _as_bool(cfg.get(_K_TLS_ENABLED), default=True) + scheme = 'https' if tls else 'http' + port = cfg.get(_K_PORT) + netloc = f'{host}:{port}' if port else str(host) + return f'{scheme}://{netloc}{value}' + return value + + +def well_known_url(issuer: str) -> str: + return issuer.rstrip('/') + '/.well-known/openid-configuration' + + +def discover_device_endpoint_from_idp( + *, + issuer: Optional[str], + discovery_url: Optional[str], + token_endpoint: Optional[str], + ctx: Optional[ssl.SSLContext] = None, + insecure: bool = False, + timeout: float = 30) -> Dict[str, Any]: + """ + Fetch the IdP ``.well-known/openid-configuration`` and return it. + + The discovery URL is taken from ``discovery_url``, else built from + ``issuer``, else (best effort) from the origin of ``token_endpoint``. + """ + url = discovery_url + if not url and issuer: + url = well_known_url(issuer) + if not url and token_endpoint: + origin = _origin(token_endpoint) + if origin: + url = well_known_url(origin) + if not url: + raise OidcConfigError( + 'Cannot discover the IdP device-authorization endpoint: no ' + 'issuer / discovery_url given and none could be derived. Pass ' + 'issuer=... or device_authorization_endpoint=... explicitly.') + return get_json(url, ctx=ctx, insecure=insecure, timeout=timeout) + + +def resolve_config( + *, + questdb_url: Optional[str] = None, + client_id: Optional[str] = None, + scope: Optional[str] = None, + audience: Optional[str] = None, + groups_in_token: Optional[bool] = None, + token_endpoint: Optional[str] = None, + device_authorization_endpoint: Optional[str] = None, + authorization_endpoint: Optional[str] = None, + issuer: Optional[str] = None, + discovery_url: Optional[str] = None, + ctx: Optional[ssl.SSLContext] = None, + insecure: bool = False, + timeout: float = 30) -> OidcConfig: + """ + Resolve a complete :class:`OidcConfig`. + + Explicit keyword arguments always win; anything left ``None`` is filled in + from QuestDB ``/settings`` (if ``questdb_url`` is given) and, as a last + resort for the device endpoint, the IdP discovery document. + """ + cfg: Dict[str, Any] = {} + if questdb_url: + cfg = fetch_settings( + questdb_url, ctx=ctx, insecure=insecure, timeout=timeout) + enabled = _as_bool(cfg.get(_K_ENABLED), default=None) + if enabled is False: + raise OidcConfigError( + f'QuestDB at {questdb_url} reports OIDC is disabled ' + f'({_K_ENABLED}=false). Nothing to authenticate against.') + + client_id = client_id or cfg.get(_K_CLIENT_ID) + if not client_id: + raise OidcConfigError( + 'Missing OIDC client_id. QuestDB did not advertise ' + f'{_K_CLIENT_ID!r} via /settings; pass client_id=... explicitly.') + + if scope is None: + scope = cfg.get(_K_SCOPE) or 'openid' + if groups_in_token is None: + groups_in_token = _as_bool(cfg.get(_K_GROUPS_IN_TOKEN), default=True) + if audience is None: + audience = cfg.get(_K_AUDIENCE) or None + + token_endpoint = ( + token_endpoint or _resolve_endpoint(cfg.get(_K_TOKEN_ENDPOINT), cfg)) + authorization_endpoint = ( + authorization_endpoint + or _resolve_endpoint(cfg.get(_K_AUTHORIZATION_ENDPOINT), cfg)) + device_authorization_endpoint = ( + device_authorization_endpoint + or _resolve_endpoint(cfg.get(_K_DEVICE_ENDPOINT), cfg)) + + # Fall back to IdP discovery when QuestDB doesn't advertise the device + # endpoint (and/or the token endpoint). This contacts the IdP, so it is + # held to https/loopback (insecure=False) regardless of the QuestDB flag. + if not device_authorization_endpoint or not token_endpoint: + doc = discover_device_endpoint_from_idp( + issuer=issuer, discovery_url=discovery_url, + token_endpoint=token_endpoint, ctx=ctx, insecure=False, + timeout=timeout) + device_authorization_endpoint = ( + device_authorization_endpoint + or doc.get('device_authorization_endpoint')) + token_endpoint = token_endpoint or doc.get('token_endpoint') + authorization_endpoint = ( + authorization_endpoint or doc.get('authorization_endpoint')) + issuer = issuer or doc.get('issuer') + + if not token_endpoint: + raise OidcConfigError( + 'Could not resolve the OIDC token endpoint from QuestDB /settings ' + 'or IdP discovery. Pass token_endpoint=... explicitly.') + if not device_authorization_endpoint: + raise OidcConfigError( + 'Could not resolve the device-authorization endpoint. The IdP ' + 'discovery document did not contain ' + '"device_authorization_endpoint". Ensure the IdP supports the ' + 'device grant, or pass device_authorization_endpoint=... ' + 'explicitly.') + + # Note: the credential-endpoint origin check (validate_endpoint_origins) + # is enforced centrally in OidcDeviceAuth.__init__, which every path + # (including the explicit constructor) goes through. + + return OidcConfig( + client_id=client_id, + token_endpoint=token_endpoint, + device_authorization_endpoint=device_authorization_endpoint, + scope=scope, + groups_in_token=bool(groups_in_token), + audience=audience, + issuer=issuer, + authorization_endpoint=authorization_endpoint) diff --git a/src/questdb/auth/_errors.py b/src/questdb/auth/_errors.py new file mode 100644 index 00000000..7262f0cc --- /dev/null +++ b/src/questdb/auth/_errors.py @@ -0,0 +1,91 @@ +################################################################################ +## ___ _ ____ ____ +## / _ \ _ _ ___ ___| |_| _ \| __ ) +## | | | | | | |/ _ \/ __| __| | | | _ \ +## | |_| | |_| | __/\__ \ |_| |_| | |_) | +## \__\_\\__,_|\___||___/\__|____/|____/ +## +## Copyright (c) 2014-2019 Appsicle +## Copyright (c) 2019-2024 QuestDB +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +"""Exceptions raised by :mod:`questdb.auth`.""" + +from __future__ import annotations + +from typing import Optional + + +class OidcError(Exception): + """Base class for every error raised by :mod:`questdb.auth`.""" + + +class OidcConfigError(OidcError): + """ + The OIDC configuration could not be resolved or is inconsistent. + + Raised, for example, when QuestDB does not advertise OIDC, when the + IdP device-authorization endpoint cannot be discovered, or when a + required argument is missing. + """ + + +class OidcNetworkError(OidcError): + """A network-level failure while talking to QuestDB or the IdP.""" + + +class OidcInteractionRequired(OidcError): + """ + Interactive sign-in is required but the process is not interactive. + + This is raised instead of hanging forever when the device flow is + started from a context with no human to authorize it (e.g. a + ``papermill`` run, a cron job or CI). Use a QuestDB service-account + REST token or the OAuth2 client-credentials grant in those contexts. + """ + + +class OidcDeviceFlowError(OidcError): + """ + The OAuth 2.0 device authorization grant failed. + + The original IdP ``error``/``error_description`` are preserved on the + exception when available. + """ + + def __init__( + self, + message: str, + *, + error: Optional[str] = None, + error_description: Optional[str] = None): + super().__init__(message) + self.error = error + self.error_description = error_description + + +class OidcTimeoutError(OidcDeviceFlowError): + """The user did not authorize the device in time (the code expired).""" + + +class OidcAuthError(OidcError): + """ + QuestDB rejected the token we presented. + + Typically a ``401``/``403`` from the server. The message includes hints + about the most common causes (scope / ``groups.encoded.in.token`` / + ``audience`` mismatches). + """ diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py new file mode 100644 index 00000000..fc5b158b --- /dev/null +++ b/src/questdb/auth/_http.py @@ -0,0 +1,234 @@ +################################################################################ +## ___ _ ____ ____ +## / _ \ _ _ ___ ___| |_| _ \| __ ) +## | | | | | | |/ _ \/ __| __| | | | _ \ +## | |_| | |_| | __/\__ \ |_| |_| | |_) | +## \__\_\\__,_|\___||___/\__|____/|____/ +## +## Copyright (c) 2014-2019 Appsicle +## Copyright (c) 2019-2024 QuestDB +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +""" +A tiny HTTP helper built on the standard library. + +OIDC device flow implementation deliberately avoids a hard dependency on ``requests``/``httpx`` +so that ``OidcDeviceAuth.token()`` / ``headers()`` work out of the box with no +extra installs. Only the device flow, discovery and the REST adapter use this +module; the heavier adapters (SQLAlchemy / psycopg / ingestion ``Sender``) bring +their own transports. + +Standard proxy environment variables (``HTTPS_PROXY`` / ``HTTP_PROXY`` / +``NO_PROXY``) are honoured automatically by ``urllib``. A custom CA bundle can be +supplied explicitly or via ``REQUESTS_CA_BUNDLE`` / ``SSL_CERT_FILE``. +""" + +from __future__ import annotations + +import ipaddress +import json +import os +import ssl +import urllib.error +import urllib.parse +import urllib.request +from typing import Any, Dict, Mapping, Optional + +from ._errors import OidcConfigError, OidcNetworkError, OidcError + +_DEFAULT_TIMEOUT = 30 +_USER_AGENT = 'questdb-python-client (oidc-auth)' + + +def build_ssl_context(ca_bundle: Optional[str] = None) -> ssl.SSLContext: + """ + Build an SSL context, honouring an explicit CA bundle or the + ``REQUESTS_CA_BUNDLE`` / ``SSL_CERT_FILE`` environment variables + (useful behind a corporate TLS-intercepting proxy). + """ + ca = ( + ca_bundle + or os.environ.get('REQUESTS_CA_BUNDLE') + or os.environ.get('SSL_CERT_FILE')) + if ca: + if os.path.isdir(ca): + return ssl.create_default_context(capath=ca) + return ssl.create_default_context(cafile=ca) + return ssl.create_default_context() + + +class HttpResponse: + """A minimal response wrapper (status + raw body + headers).""" + + __slots__ = ('status', 'body', 'headers') + + def __init__(self, status: int, body: bytes, headers: Mapping[str, str]): + self.status = status + self.body = body + self.headers = dict(headers) + + def text(self) -> str: + return self.body.decode('utf-8', errors='replace') + + def json(self) -> Any: + return json.loads(self.body.decode('utf-8')) + + @property + def ok(self) -> bool: + return 200 <= self.status < 300 + + +def _is_loopback(host: Optional[str]) -> bool: + # Traffic to a loopback address never leaves the host, so plaintext http + # carries no network interception risk and is always permitted. + if not host: + return False + if host.lower() == 'localhost': + return True + try: + return ipaddress.ip_address(host).is_loopback + except ValueError: + return False + + +def _require_secure(url: str, insecure: bool) -> None: + parts = urllib.parse.urlparse(url) + scheme = parts.scheme.lower() + if scheme == 'https': + return + if scheme == 'http': + if _is_loopback(parts.hostname): + return + if insecure: + return + raise OidcConfigError( + f'Refusing to use insecure URL {url!r} (scheme {scheme!r}). Use https ' + '(loopback http is always allowed for local development); pass ' + 'insecure=True only to permit plaintext to a non-loopback host.') + + +def _opener(ctx: Optional[ssl.SSLContext]) -> urllib.request.OpenerDirector: + # build_opener keeps the default ProxyHandler (which reads *_PROXY env + # vars), while letting us pin our own TLS context. + if ctx is None: + return urllib.request.build_opener() + return urllib.request.build_opener(urllib.request.HTTPSHandler(context=ctx)) + + +def request( + method: str, + url: str, + *, + form: Optional[Mapping[str, Any]] = None, + data: Optional[bytes] = None, + headers: Optional[Mapping[str, str]] = None, + timeout: float = _DEFAULT_TIMEOUT, + ctx: Optional[ssl.SSLContext] = None, + insecure: bool = False) -> HttpResponse: + """ + Perform a single HTTP request. + + ``form`` is form-url-encoded into the body (``application/x-www-form- + urlencoded``). HTTP error statuses (``4xx``/``5xx``) are returned as an + :class:`HttpResponse` rather than raised, so callers can inspect OAuth + error bodies (e.g. ``authorization_pending``). Only genuine network + failures raise (:class:`OidcNetworkError`). + """ + _require_secure(url, insecure) + body: Optional[bytes] = data + req_headers = {'User-Agent': _USER_AGENT, 'Accept': 'application/json'} + if form is not None: + body = urllib.parse.urlencode( + {k: v for k, v in form.items() if v is not None}).encode('utf-8') + req_headers['Content-Type'] = 'application/x-www-form-urlencoded' + if headers: + req_headers.update(headers) + + req = urllib.request.Request( + url, data=body, headers=req_headers, method=method.upper()) + try: + with _opener(ctx).open(req, timeout=timeout) as resp: + return HttpResponse( + getattr(resp, 'status', resp.getcode()), + resp.read(), + resp.headers) + except urllib.error.HTTPError as e: + # 4xx/5xx still carry a (possibly JSON) body we want to inspect. + # Map a mid-body read failure to a network error (rather than letting a + # bare OSError escape) and close the error response so its socket isn't + # leaked (the poll loop drives many 400s during a long sign-in). + try: + body = e.read() + except (TimeoutError, OSError) as read_err: + raise OidcNetworkError( + f'Failed to read response from {url}: {read_err}') from read_err + finally: + e.close() + return HttpResponse(e.code, body, e.headers or {}) + except urllib.error.URLError as e: + raise OidcNetworkError(f'Failed to reach {url}: {e.reason}') from e + except (TimeoutError, OSError) as e: + raise OidcNetworkError(f'Failed to reach {url}: {e}') from e + + +def get_json( + url: str, + *, + headers: Optional[Mapping[str, str]] = None, + timeout: float = _DEFAULT_TIMEOUT, + ctx: Optional[ssl.SSLContext] = None, + insecure: bool = False) -> Any: + """GET a URL and parse a JSON response, raising on non-2xx.""" + resp = request( + 'GET', url, headers=headers, timeout=timeout, ctx=ctx, + insecure=insecure) + if not resp.ok: + raise OidcError( + f'HTTP {resp.status} from {url}: {resp.text()[:200]}') + try: + return resp.json() + except (ValueError, UnicodeDecodeError) as e: + raise OidcError(f'Invalid JSON from {url}: {e}') from e + + +def post_form( + url: str, + form: Mapping[str, Any], + *, + headers: Optional[Mapping[str, str]] = None, + timeout: float = _DEFAULT_TIMEOUT, + ctx: Optional[ssl.SSLContext] = None, + insecure: bool = False) -> tuple[int, Dict[str, Any]]: + """ + POST a form-url-encoded body and parse the JSON response. + + Returns ``(status, parsed_json)``. Used for the device-authorization and + token endpoints, which return JSON bodies on both success and error. + """ + resp = request( + 'POST', url, form=form, headers=headers, timeout=timeout, ctx=ctx, + insecure=insecure) + try: + parsed = resp.json() + except (ValueError, UnicodeDecodeError): + if resp.ok: + raise OidcError( + f'Expected JSON from {url}, got: {resp.text()[:200]}') + # Non-JSON error body: surface the status + text. + raise OidcError(f'HTTP {resp.status} from {url}: {resp.text()[:200]}') + if not isinstance(parsed, dict): + raise OidcError(f'Unexpected JSON shape from {url}: {parsed!r}') + return resp.status, parsed diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py new file mode 100644 index 00000000..2b95d75b --- /dev/null +++ b/src/questdb/auth/_questdb.py @@ -0,0 +1,305 @@ +################################################################################ +## ___ _ ____ ____ +## / _ \ _ _ ___ ___| |_| _ \| __ ) +## | | | | | | |/ _ \/ __| __| | | | _ \ +## | |_| | |_| | __/\__ \ |_| |_| | |_) | +## \__\_\\__,_|\___||___/\__|____/|____/ +## +## Copyright (c) 2014-2019 Appsicle +## Copyright (c) 2019-2024 QuestDB +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +"""High-level QuestDB session: token, REST queries, and connection adapters.""" + +from __future__ import annotations + +import urllib.parse +from typing import Any, Dict, Optional + +from ._device import OidcDeviceAuth +from ._errors import OidcAuthError, OidcError +from ._http import request + +_DEFAULT_PG_PORT = 8812 +_DEFAULT_DATABASE = 'qdb' + +_AUTH_HINT = ( + 'QuestDB rejected the token (HTTP {status}). Common causes:\n' + " * scope / 'acl.oidc.groups.encoded.in.token' mismatch — the server may " + 'expect the id_token (groups in token) while an access_token was sent, or ' + 'vice-versa;\n' + " * the 'groups'/'sub' claim is missing — check the requested scope;\n" + " * 'aud' mismatch — the token's audience does not match " + "'acl.oidc.audience' (try passing audience=...).") + + +def _import_pandas(): + try: + import pandas # type: ignore + return pandas + except ImportError as e: + raise ImportError( + 'Missing optional dependency `pandas`, required for ' + 'QuestDB.sql(). Install it with `pip install questdb[dataframe]`. ' + 'See https://py-questdb-client.readthedocs.io/en/latest/' + 'installation.html') from e + + +def _exec_json_to_df(data: Dict[str, Any], pandas): + columns = data.get('columns') or [] + names = [c.get('name') for c in columns] + dataset = data.get('dataset') + if dataset is None: + dataset = data.get('data') or [] + try: + df = pandas.DataFrame(dataset, columns=names or None) + except ValueError as e: + raise OidcError( + f'Unexpected shape in QuestDB /exec response: {e}') from e + for col in columns: + name = col.get('name') + if col.get('type') in ('TIMESTAMP', 'DATE') and name in df.columns: + try: + df[name] = pandas.to_datetime(df[name], errors='coerce') + except Exception: + pass + return df + + +def _pg_module(): + try: + import psycopg # type: ignore # psycopg v3 + return psycopg + except ImportError: + pass + try: + import psycopg2 # type: ignore + return psycopg2 + except ImportError: + raise ImportError( + 'A PostgreSQL driver is required: install `psycopg` (v3) or ' + '`psycopg2-binary`.') + + +class QuestDB: + """ + A thin, authenticated QuestDB session built on an :class:`OidcDeviceAuth`. + + Provides a one-call DataFrame query over REST plus adapters that feed the + same auto-refreshed token into your existing tools (SQLAlchemy / psycopg / + the ingestion ``Sender``). You can also just take :meth:`token` / + :meth:`headers` and wire them up yourself. + """ + + def __init__( + self, + url: str, + auth: OidcDeviceAuth, + *, + insecure: bool = False): + self.url = url.rstrip('/') + self.auth = auth + self._insecure = insecure + self._ctx = auth._ctx + self._parts = urllib.parse.urlparse(self.url) + + # -- token access ------------------------------------------------------- + + def token(self) -> str: + """Return a valid, auto-refreshed token (see :meth:`OidcDeviceAuth.token`).""" + return self.auth.token() + + def headers(self) -> Dict[str, str]: + """Return ``{"Authorization": "Bearer "}``.""" + return self.auth.headers() + + # -- REST query --------------------------------------------------------- + + def sql(self, query: str, *, limit: Optional[str] = None, + timeout: float = 60) -> 'pandas.DataFrame': + """ + Run a SQL query over QuestDB's REST ``/exec`` endpoint and return a + :class:`pandas.DataFrame`. + + Uses ``Authorization: Bearer`` (no token-length limit, unlike PG-wire), + which makes it the recommended path for large groups-encoded JWTs. + + :param query: The SQL query to run. + :param limit: Optional QuestDB ``limit`` (e.g. ``"1,1000"``). + :param timeout: Request timeout in seconds. + """ + pandas = _import_pandas() + params = {'query': query} + if limit is not None: + params['limit'] = limit + url = f'{self.url}/exec?' + urllib.parse.urlencode(params) + resp = request( + 'GET', url, headers=self.headers(), ctx=self._ctx, + insecure=self._insecure, timeout=timeout) + if resp.status in (401, 403): + raise OidcAuthError(_AUTH_HINT.format(status=resp.status)) + if not resp.ok: + detail = resp.text()[:300] + try: + detail = resp.json().get('error', detail) + except Exception: + pass + raise OidcError( + f'QuestDB query failed (HTTP {resp.status}): {detail}') + return _exec_json_to_df(resp.json(), pandas) + + # -- connection adapters ------------------------------------------------ + + def _host(self) -> Optional[str]: + return self._parts.hostname + + def sqlalchemy_engine( + self, + *, + host: Optional[str] = None, + pg_port: int = _DEFAULT_PG_PORT, + database: str = _DEFAULT_DATABASE, + drivername: Optional[str] = None, + **engine_kwargs) -> 'sqlalchemy.engine.Engine': + """ + Build a SQLAlchemy ``Engine`` for QuestDB's PG-wire endpoint. + + Connects as user ``_sso`` and injects a **fresh** token as the password + for every new connection (via a ``do_connect`` listener), so pooled + connections always authenticate with a valid token. Requires + ``acl.oidc.pg.token.as.password.enabled=true`` on the server. + """ + try: + from sqlalchemy import create_engine, event + from sqlalchemy.engine import URL + except ImportError as e: + raise ImportError( + 'SQLAlchemy is required for QuestDB.sqlalchemy_engine(); ' + 'install it with `pip install sqlalchemy`.') from e + + if drivername is None: + mod = _pg_module() + drivername = ( + 'postgresql+psycopg' + if mod.__name__ == 'psycopg' + else 'postgresql+psycopg2') + + url = URL.create( + drivername=drivername, + username='_sso', + host=host or self._host(), + port=pg_port, + database=database) + engine = create_engine(url, **engine_kwargs) + + auth = self.auth + + @event.listens_for(engine, 'do_connect') + def _provide_token(dialect, conn_rec, cargs, cparams): # noqa: ANN001 + cparams['password'] = auth.token() + + return engine + + def psycopg( + self, + *, + host: Optional[str] = None, + pg_port: int = _DEFAULT_PG_PORT, + database: str = _DEFAULT_DATABASE, + **connect_kwargs) -> 'Any': + """ + Open a raw psycopg (v3) or psycopg2 connection to QuestDB's PG-wire + endpoint, authenticating as ``_sso`` with the current token. + + The token is captured at connect time; open a new connection to pick up + a refreshed token. + """ + mod = _pg_module() + return mod.connect( + host=host or self._host(), + port=pg_port, + dbname=database, + user='_sso', + password=self.auth.token(), + **connect_kwargs) + + def sender(self, *, port: Optional[int] = None, + **sender_kwargs) -> 'questdb.ingress.Sender': + """ + Build a :class:`questdb.ingress.Sender` (ILP-over-HTTP) configured with + the current bearer token, for ingestion. + + The token is captured at creation time; create a new sender to pick up + a refreshed token. + """ + try: + from questdb.ingress import Sender + except ImportError as e: + raise ImportError( + 'The compiled `questdb.ingress` module is required for ' + 'QuestDB.sender(). Install the full client wheel ' + '(`pip install questdb`).') from e + + scheme = 'https' if self._parts.scheme == 'https' else 'http' + resolved_port = port or self._parts.port or ( + 443 if scheme == 'https' else 9000) + conf = f'{scheme}::addr={self._host()}:{resolved_port};' + return Sender.from_conf(conf, token=self.auth.token(), **sender_kwargs) + + +def connect( + url: str, + *, + flow: str = 'auto', + cache: Any = 'memory', + insecure: bool = False, + eager: bool = True, + **opts) -> QuestDB: + """ + High-level entry point: authenticate to QuestDB and return a + :class:`QuestDB` session. + + .. code-block:: python + + from questdb.auth import connect + + qdb = connect("https://questdb.example.com:9000") # signs in + df = qdb.sql("SELECT * FROM trades LIMIT 10") + + Configuration (OIDC client id, scope, endpoints, groups mode) is discovered + from ``{url}/settings`` and, as needed, the IdP ``.well-known`` document. + Re-running the same call reuses the cached token (no re-prompt). + + :param url: The QuestDB HTTP(S) base URL, e.g. + ``"https://questdb.example.com:9000"``. + :param flow: ``"auto"`` (default), ``"device"`` or ``"loopback"``. Today + ``"auto"`` always resolves to the device flow (works on local and + remote kernels); ``"loopback"`` is reserved for a future release. + :param cache: Token cache backend: ``"memory"`` (default), ``"file"`` or + ``None``. + :param insecure: Allow plaintext ``http://`` URLs (development only). + :param eager: If ``True`` (default), sign in immediately; otherwise defer + until the first call that needs a token. + :param opts: Forwarded to :meth:`OidcDeviceAuth.from_questdb` (e.g. + ``client_id``, ``scope``, ``audience``, ``issuer``, ``open_browser``, + ``qr``, ``ca_bundle``). + """ + auth = OidcDeviceAuth.from_questdb( + url, flow=flow, cache=cache, insecure=insecure, **opts) + qdb = QuestDB(url, auth, insecure=insecure) + if eager: + auth.token() + return qdb diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py new file mode 100644 index 00000000..8a0fa9fb --- /dev/null +++ b/src/questdb/auth/_render.py @@ -0,0 +1,336 @@ +################################################################################ +## ___ _ ____ ____ +## / _ \ _ _ ___ ___| |_| _ \| __ ) +## | | | | | | |/ _ \/ __| __| | | | _ \ +## | |_| | |_| | __/\__ \ |_| |_| | |_) | +## \__\_\\__,_|\___||___/\__|____/|____/ +## +## Copyright (c) 2014-2019 Appsicle +## Copyright (c) 2019-2024 QuestDB +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +""" +Presentation of the device-flow prompt. + +Renders a clickable link + user code in Jupyter (via ``IPython.display``) and +falls back to plain text on a terminal. Nothing here is required for +``token()`` / ``headers()`` to work; ``IPython`` and ``qrcode`` are imported +lazily and only when actually used. +""" + +from __future__ import annotations + +import html +import sys +import urllib.parse +from typing import Any, Dict, Optional, TextIO + + +def in_ipython_kernel() -> bool: + """True when running inside an interactive Jupyter/ZMQ kernel.""" + try: + from IPython import get_ipython # type: ignore + except Exception: + return False + ip = get_ipython() + if ip is None: + return False + # ZMQInteractiveShell == notebook / qtconsole / lab; TerminalInteractive + # Shell == ipython in a terminal (still interactive). + return ip.__class__.__name__ in ( + 'ZMQInteractiveShell', 'TerminalInteractiveShell') + + +def detect_interactive() -> bool: + """ + Best-effort detection of whether a human can complete the sign-in. + + Interactive when attached to a TTY or running in an interactive IPython + shell. This guards against hanging forever in a non-interactive context + (papermill / cron / CI). + """ + if in_ipython_kernel(): + return True + try: + return bool(sys.stdin and sys.stdin.isatty() + and sys.stdout and sys.stdout.isatty()) + except Exception: + return False + + +def _verification_uri(resp: Dict[str, Any]) -> str: + # RFC 8628 uses ``verification_uri``; some IdPs (older Google) use + # ``verification_url``. + return resp.get('verification_uri') or resp.get('verification_url') or '' + + +def _verification_uri_complete(resp: Dict[str, Any]) -> Optional[str]: + return (resp.get('verification_uri_complete') + or resp.get('verification_url_complete')) + + +def _safe_link_url(url: Optional[str]) -> Optional[str]: + """ + Return ``url`` only if it uses an ``http(s)`` scheme, else ``None``. + + The verification URL comes from the IdP's device-authorization response, + which is untrusted input. Embedding it in an HTML ``href`` without a scheme + allowlist would let a malicious/MITM'd response inject a ``javascript:`` or + ``data:`` URL that executes in the notebook DOM when clicked + (``html.escape`` guards markup, not the URL scheme). + """ + if not url: + return None + try: + scheme = urllib.parse.urlparse(url).scheme.lower() + except (ValueError, TypeError): + return None + return url if scheme in ('http', 'https') else None + + +def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: + """ + Render ``url`` as a clickable link, or as inert escaped text if its scheme + is not ``http(s)``. + + The visible label defaults to the URL itself. When the URL is rejected, the + (escaped) URL is shown as plain text so the user can still see/copy it, but + it is never turned into a clickable/executable link. + """ + safe = _safe_link_url(url) + label = html.escape(text if text is not None else (url or '')) + if safe is None: + return label + return (f'{label}') + + +def format_prompt(resp: Dict[str, Any]) -> str: + """Plain-text sign-in prompt (also used as the notebook fallback).""" + uri = _verification_uri(resp) + code = resp.get('user_code', '') + complete = _verification_uri_complete(resp) + lines = [ + '🔐 Sign in to QuestDB', + f' Open {uri} and enter code: {code}', + ] + if complete: + lines.append(f' (or open directly: {complete})') + return '\n'.join(lines) + + +def _fmt_mmss(seconds: float) -> str: + seconds = max(0, int(seconds)) + return f'{seconds // 60}:{seconds % 60:02d}' + + +class Renderer: + """No-op renderer interface; subclasses present the prompt to the user.""" + + def on_prompt(self, resp: Dict[str, Any]) -> None: + pass + + def on_waiting(self, seconds_left: float) -> None: + pass + + def on_success(self, identity: Optional[str], expires_in: float) -> None: + pass + + def on_failure(self, message: str) -> None: + pass + + +class TerminalRenderer(Renderer): + """Plain-text rendering for terminals (writes to ``stderr`` by default).""" + + def __init__(self, stream: Optional[TextIO] = None, qr: bool = False): + self._stream = stream if stream is not None else sys.stderr + self._qr = qr + self._countdown_active = False + + def _write(self, text: str) -> None: + try: + self._stream.write(text) + self._stream.flush() + except Exception: + pass + + def on_prompt(self, resp: Dict[str, Any]) -> None: + self._write(format_prompt(resp) + '\n') + if self._qr: + target = _verification_uri_complete(resp) or _verification_uri(resp) + art = _qr_ascii(target) + if art: + self._write(art + '\n') + + def on_waiting(self, seconds_left: float) -> None: + self._countdown_active = True + self._write(f'\r ⏳ waiting for authorization… ({_fmt_mmss(seconds_left)} left) ') + + def on_success(self, identity: Optional[str], expires_in: float) -> None: + if self._countdown_active: + self._write('\n') + self._countdown_active = False + who = f' as {identity}' if identity else '' + mins = max(1, int(round(expires_in / 60))) + self._write(f'✅ Signed in{who} — token cached, expires in {mins} min\n') + + def on_failure(self, message: str) -> None: + if self._countdown_active: + self._write('\n') + self._countdown_active = False + self._write(f'❌ {message}\n') + + +class JupyterRenderer(Renderer): + """Rich rendering for Jupyter using an updatable display handle.""" + + def __init__(self, qr: bool = False): + self._qr = qr + self._handle = None + self._resp: Dict[str, Any] = {} + + def _display(self, html_str: str): + from IPython.display import HTML, display # type: ignore + if self._handle is None: + self._handle = display(HTML(html_str), display_id=True) + else: + self._handle.update(HTML(html_str)) + + def _panel(self, body: str) -> str: + return ( + '
' + + body + '
') + + def on_prompt(self, resp: Dict[str, Any]) -> None: + self._resp = resp + uri = _verification_uri(resp) + code = html.escape(str(resp.get('user_code', ''))) + complete = _verification_uri_complete(resp) + body = [ + '
' + '🔐 Sign in to QuestDB
', + f'
Open {_render_link(uri)} and enter code:
', + f'
{code}
', + ] + if _safe_link_url(complete): + body.append( + '
' + _render_link( + complete, text='Click here to authorize directly →') + + '
') + if self._qr: + qr_target = _safe_link_url(complete) or _safe_link_url(uri) + data_uri = _qr_data_uri(qr_target) if qr_target else None + if data_uri: + body.append( + f'QR code') + body.append( + '
' + '⏳ waiting for authorization…
') + self._display(self._panel(''.join(body))) + + def on_waiting(self, seconds_left: float) -> None: + # Re-render the whole panel (cheap) with an updated countdown. + if not self._resp: + return + self._resp = dict(self._resp) + self._render_with_status( + f'⏳ waiting for authorization… ({_fmt_mmss(seconds_left)} left)', + color='#888') + + def on_success(self, identity: Optional[str], expires_in: float) -> None: + who = html.escape(identity) if identity else '' + mins = max(1, int(round(expires_in / 60))) + suffix = f' as {who}' if who else '' + self._render_with_status( + f'✅ Signed in{suffix} — token cached, expires in {mins} min', + color='#2e7d32') + + def on_failure(self, message: str) -> None: + self._render_with_status('❌ ' + html.escape(message), color='#c62828') + + def _render_with_status(self, status_html: str, color: str) -> None: + resp = self._resp + uri = _verification_uri(resp) + code = html.escape(str(resp.get('user_code', ''))) + complete = _verification_uri_complete(resp) + body = [ + '
' + '🔐 Sign in to QuestDB
', + f'
Open {_render_link(uri)} and enter code:
', + f'
{code}
', + ] + if _safe_link_url(complete): + body.append( + '
' + _render_link( + complete, text='Click here to authorize directly →') + + '
') + body.append( + f'
{status_html}
') + self._display(self._panel(''.join(body))) + + +def make_renderer(qr: bool = False) -> Renderer: + """Pick a renderer appropriate for the current environment.""" + if in_ipython_kernel(): + try: + import IPython.display # noqa: F401 # type: ignore + return JupyterRenderer(qr=qr) + except Exception: + pass + return TerminalRenderer(qr=qr) + + +def _qr_ascii(data: str) -> Optional[str]: + if not data: + return None + try: + import qrcode # type: ignore + except Exception: + return None + try: + qr = qrcode.QRCode(border=1) + qr.add_data(data) + qr.make(fit=True) + import io + buf = io.StringIO() + qr.print_ascii(out=buf, invert=True) + return buf.getvalue() + except Exception: + return None + + +def _qr_data_uri(data: str) -> Optional[str]: + if not data: + return None + try: + import qrcode # type: ignore + except Exception: + return None + try: + import base64 + import io + img = qrcode.make(data) + buf = io.BytesIO() + img.save(buf, format='PNG') + b64 = base64.b64encode(buf.getvalue()).decode('ascii') + return f'data:image/png;base64,{b64}' + except Exception: + return None diff --git a/test/test.py b/test/test.py index 18f4461a..6fdd418f 100755 --- a/test/test.py +++ b/test/test.py @@ -33,6 +33,24 @@ from fixture import _parse_version +# OIDC auth tests (pure-Python; no compiled extension required). +# Imported here so they are picked up by ``unittest.main()`` in CI. +from test_auth import ( + TestDeviceFlow, + TestNonInteractive, + TestRefresh, + TestFileCache, + TestDiscovery, + TestRestAdapter, + TestAdapters, + TestConcurrency, + TestConfigHelpers, + TestEndpointValidation, + TestCacheKey, + TestTransportSecurity, + TestRendererSecurity, +) + NUMPY_VERSION = _parse_version(np.__version__) try: diff --git a/test/test_auth.py b/test/test_auth.py new file mode 100644 index 00000000..427ccb57 --- /dev/null +++ b/test/test_auth.py @@ -0,0 +1,1167 @@ +#!/usr/bin/env python3 +################################################################################ +## ___ _ ____ ____ +## / _ \ _ _ ___ ___| |_| _ \| __ ) +## | | | | | | |/ _ \/ __| __| | | | _ \ +## | |_| | |_| | __/\__ \ |_| |_| | |_) | +## \__\_\\__,_|\___||___/\__|____/|____/ +## +## Copyright (c) 2014-2019 Appsicle +## Copyright (c) 2019-2024 QuestDB +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +""" +Standalone unit tests for ``questdb.auth``. + +These do not require the compiled ``questdb.ingress`` extension; they exercise +the device flow, discovery, caching, refresh and the REST adapter against an +in-process mock IdP + mock QuestDB server. + +Run directly:: + + python3 test/test_auth.py -v +""" + +import base64 +import importlib.util +import json +import os +import sys +import tempfile +import threading +import types +import unittest +import http.server +import urllib.parse +from unittest import mock + +sys.dont_write_bytecode = True +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + +from questdb.auth import ( # noqa: E402 + OidcDeviceAuth, + QuestDB, + connect, + OidcError, + OidcConfigError, + OidcDeviceFlowError, + OidcTimeoutError, + OidcInteractionRequired, + OidcAuthError, + OidcNetworkError, + TokenSet, +) +from questdb.auth._cache import FileCache, MemoryCache, _MEMORY_STORE # noqa: E402 +from questdb.auth._render import Renderer # noqa: E402 + +try: + import pandas as pd +except ImportError: + pd = None + +try: + import fcntl as _fcntl # noqa: F401 + _HAS_FCNTL = True +except ImportError: + _HAS_FCNTL = False + +_HAS_PG_DRIVER = ( + importlib.util.find_spec('psycopg') is not None + or importlib.util.find_spec('psycopg2') is not None) + + +class _FakeAuth: + """A stand-in OidcDeviceAuth for adapter tests (no network).""" + + _ctx = None + + def __init__(self, token='TKN'): + self._token = token + self.calls = 0 + + def token(self): + self.calls += 1 + return self._token + + def headers(self): + return {'Authorization': f'Bearer {self._token}'} + + +def _jwt(claims): + """Build an unsigned JWT-shaped string with the given payload claims.""" + def b64(obj): + raw = json.dumps(obj).encode() + return base64.urlsafe_b64encode(raw).rstrip(b'=').decode() + return f'{b64({"alg": "none"})}.{b64(claims)}.sig' + + +ID_TOKEN = _jwt({'sub': 'user-1', 'email': 'alice@example.com', + 'groups': ['analysts']}) +ACCESS_TOKEN = _jwt({'sub': 'user-1', 'scope': 'openid'}) + + +class FakeClock: + """Deterministic clock: ``sleep`` advances both monotonic and wall time.""" + + def __init__(self): + self.mono = 0.0 + self.wall = 1_000_000.0 + self.sleeps = [] + + def sleep(self, dt): + self.sleeps.append(dt) + self.mono += dt + self.wall += dt + + def monotonic(self): + return self.mono + + def now(self): + return self.wall + + +class MockState: + """Scriptable behaviour shared with the request handler.""" + + def __init__(self): + self.settings = {} + self.well_known = None + # FIFO of (status, body) returned for device_code grant polls. + # When exhausted, the last entry repeats. + self.token_script = [(200, None)] # None => default success body + self.refresh_response = None # (status, body) or None + self.device_response = None # override device-auth response body + self.device_status = 200 + self.expected_bearer = None # for /exec auth check + self.exec_response = None + self.exec_status = 200 + # Recording. + self.device_requests = 0 + self.token_requests = [] + self.refresh_requests = 0 + self.exec_requests = [] + + +class _Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + @property + def state(self): + return self.server.state + + def _send_json(self, status, obj): + data = json.dumps(obj).encode() + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _read_form(self): + length = int(self.headers.get('Content-Length', 0)) + body = self.rfile.read(length).decode() + return {k: v[0] for k, v in urllib.parse.parse_qs(body).items()} + + def do_GET(self): + path = urllib.parse.urlparse(self.path).path + if path == '/settings': + self._send_json(200, self.state.settings) + elif path == '/.well-known/openid-configuration': + if self.state.well_known is None: + self._send_json(404, {'error': 'not found'}) + else: + self._send_json(200, self.state.well_known) + elif path == '/exec': + auth = self.headers.get('Authorization') + if self.state.expected_bearer and auth != ( + 'Bearer ' + self.state.expected_bearer): + self._send_json(401, {'error': 'unauthorized'}) + return + self.state.exec_requests.append(self.path) + self._send_json(self.state.exec_status, self.state.exec_response or { + 'columns': [ + {'name': 'ts', 'type': 'TIMESTAMP'}, + {'name': 'price', 'type': 'DOUBLE'}, + ], + 'dataset': [ + ['2021-01-01T00:00:00.000000Z', 1.5], + ['2021-01-02T00:00:00.000000Z', 2.5], + ], + 'count': 2, + }) + else: + self._send_json(404, {'error': 'not found'}) + + def do_POST(self): + path = urllib.parse.urlparse(self.path).path + form = self._read_form() + if path == '/device': + self.state.device_requests += 1 + if self.state.device_status != 200: + self._send_json(self.state.device_status, + self.state.device_response or + {'error': 'invalid_client'}) + return + body = self.state.device_response or { + 'device_code': 'DEV-CODE', + 'user_code': 'WDJB-MJHT', + 'verification_uri': 'https://idp.example.com/device', + 'verification_uri_complete': + 'https://idp.example.com/device?user_code=WDJB-MJHT', + 'expires_in': 600, + 'interval': 5, + } + self._send_json(200, body) + elif path == '/token': + grant = form.get('grant_type') + if grant == 'refresh_token': + self.state.refresh_requests += 1 + status, body = self.state.refresh_response or ( + 200, self._default_token_body()) + self._send_json(status, body) + return + self.state.token_requests.append(form) + idx = min(len(self.state.token_requests) - 1, + len(self.state.token_script) - 1) + status, body = self.state.token_script[idx] + if body is None: + body = self._default_token_body() + self._send_json(status, body) + else: + self._send_json(404, {'error': 'not found'}) + + @staticmethod + def _default_token_body(): + return { + 'access_token': ACCESS_TOKEN, + 'id_token': ID_TOKEN, + 'refresh_token': 'REFRESH-1', + 'token_type': 'Bearer', + 'expires_in': 3600, + 'scope': 'openid groups', + } + + +class _MockServer(http.server.HTTPServer): + def __init__(self): + super().__init__(('127.0.0.1', 0), _Handler) + self.state = MockState() + + +class AuthTestBase(unittest.TestCase): + def setUp(self): + _MEMORY_STORE.clear() + self.server = _MockServer() + self.state = self.server.state + self.thread = threading.Thread( + target=lambda: self.server.serve_forever(poll_interval=0.02), + daemon=True) + self.thread.start() + self.base = f'http://127.0.0.1:{self.server.server_port}' + + def tearDown(self): + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) + + def make_auth(self, *, clock=None, groups_in_token=True, cache='memory', + interactive=True, **kw): + clock = clock or FakeClock() + self._clock = clock + return OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint=self.base + '/device', + token_endpoint=self.base + '/token', + scope='openid groups', + groups_in_token=groups_in_token, + cache=cache, + insecure=True, + interactive=interactive, + renderer=Renderer(), + _clock=clock, + **kw) + + +class TestDeviceFlow(AuthTestBase): + def test_happy_path_returns_id_token(self): + self.state.token_script = [ + (400, {'error': 'authorization_pending'}), + (400, {'error': 'authorization_pending'}), + (200, None), + ] + auth = self.make_auth() + token = auth.token() + self.assertEqual(token, ID_TOKEN) + # 3 token polls, slept 'interval' (5s) before each. + self.assertEqual(len(self.state.token_requests), 3) + self.assertEqual(self._clock.sleeps, [5, 5, 5]) + + def test_access_token_when_groups_not_in_token(self): + auth = self.make_auth(groups_in_token=False) + self.assertEqual(auth.token(), ACCESS_TOKEN) + + def test_headers(self): + auth = self.make_auth() + self.assertEqual(auth.headers(), + {'Authorization': 'Bearer ' + ID_TOKEN}) + + def test_slow_down_backs_off(self): + self.state.token_script = [ + (400, {'error': 'slow_down'}), + (200, None), + ] + auth = self.make_auth() + auth.token() + # interval starts at 5, +5 after slow_down. + self.assertEqual(self._clock.sleeps, [5, 10]) + + def test_timeout_when_never_authorized(self): + self.state.device_response = { + 'device_code': 'DEV-CODE', 'user_code': 'X', + 'verification_uri': 'https://idp/device', + 'expires_in': 10, 'interval': 5, + } + self.state.token_script = [(400, {'error': 'authorization_pending'})] + auth = self.make_auth() + with self.assertRaises(OidcTimeoutError): + auth.token() + + def test_access_denied_is_surfaced(self): + self.state.token_script = [ + (400, {'error': 'access_denied', + 'error_description': 'user said no'}), + ] + auth = self.make_auth() + with self.assertRaises(OidcDeviceFlowError) as cm: + auth.token() + self.assertEqual(cm.exception.error, 'access_denied') + self.assertIn('user said no', str(cm.exception)) + + def test_device_endpoint_rejects_grant(self): + self.state.device_status = 400 + self.state.device_response = {'error': 'invalid_client'} + auth = self.make_auth() + with self.assertRaises(OidcDeviceFlowError) as cm: + auth.token() + self.assertIn('device grant', str(cm.exception)) + + def test_token_caches_in_memory_across_instances(self): + self.make_auth().token() + self.assertEqual(self.state.device_requests, 1) + # A brand-new instance with the same config reuses the cached token. + self.make_auth().token() + self.assertEqual(self.state.device_requests, 1) + + def test_missing_id_token_raises_config_error(self): + self.state.token_script = [(200, { + 'access_token': ACCESS_TOKEN, 'token_type': 'Bearer', + 'expires_in': 3600})] # no id_token + auth = self.make_auth(groups_in_token=True) + with self.assertRaises(OidcConfigError): + auth.token() + + def test_200_without_access_token_is_not_success(self): + # A 200 with no access_token must not be treated as a token. + self.state.token_script = [(200, {'token_type': 'Bearer'})] + auth = self.make_auth() + with self.assertRaises(OidcDeviceFlowError): + auth.token() + + def test_access_token_headers(self): + auth = self.make_auth(groups_in_token=False) + self.assertEqual(auth.headers(), + {'Authorization': 'Bearer ' + ACCESS_TOKEN}) + + def test_clear_forces_resignin(self): + auth = self.make_auth() + auth.token() + self.assertEqual(self.state.device_requests, 1) + auth.clear() + auth.token() + self.assertEqual(self.state.device_requests, 2) # prompted again + + def test_openid_scope_auto_added_for_groups_in_token(self): + # groups-in-token requires an id_token, which needs the openid scope. + auth = OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint=self.base + '/device', + token_endpoint=self.base + '/token', + scope='groups', groups_in_token=True, # no 'openid' + cache='memory', insecure=True, renderer=Renderer()) + self.assertIn('openid', auth.config.scope.split()) + + def test_zero_expires_in_is_treated_as_unknown(self): + # A non-positive expires_in must not mark the just-issued token expired. + self.state.token_script = [(200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'token_type': 'Bearer', 'expires_in': 0})] + auth = self.make_auth() + auth.token() + self.assertTrue(auth._tokens.is_valid(self._clock.now())) + + def test_short_lived_token_valid_at_issue(self): + # A small positive expires_in (< 2*skew) must not read as expired the + # instant it is issued (adaptive skew = min(skew, lifetime/2)). + self.state.token_script = [(200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'token_type': 'Bearer', 'expires_in': 20})] + auth = self.make_auth() + auth.token() + t = auth._tokens + self.assertEqual(round(t.expires_at - t.issued_at), 20) + self.assertTrue(t.is_valid(t.issued_at)) # usable right after issue + self.assertFalse(t.is_valid(t.expires_at)) # but still does expire + + def test_open_browser_rejects_dangerous_scheme(self): + auth = self.make_auth(open_browser=True) + with mock.patch('webbrowser.open') as opener: + auth._maybe_open_browser({'verification_uri': 'javascript:alert(1)'}) + opener.assert_not_called() + auth._maybe_open_browser( + {'verification_uri': 'https://idp.example.com/device'}) + opener.assert_called_once_with('https://idp.example.com/device') + + def test_memory_cache_returns_independent_copy(self): + cache = MemoryCache() + cache.store('k', TokenSet(access_token='a', refresh_token='r', + expires_at=1.0)) + loaded = cache.load('k') + loaded.refresh_token = 'MUTATED' + self.assertEqual(cache.load('k').refresh_token, 'r') + + +class TestNonInteractive(AuthTestBase): + def test_non_interactive_raises_without_polling(self): + auth = self.make_auth(interactive=False) + with self.assertRaises(OidcInteractionRequired): + auth.token() + self.assertEqual(self.state.device_requests, 0) + + +class TestRefresh(AuthTestBase): + def _seed_expired(self, auth, refresh_token='REFRESH-1'): + expired = TokenSet( + access_token='old-access', id_token='old-id', + refresh_token=refresh_token, + expires_at=self._clock.now() - 10) + auth._cache.store(auth.cache_key, expired) + + def test_silent_refresh(self): + auth = self.make_auth() + self._seed_expired(auth) + token = auth.token() + self.assertEqual(token, ID_TOKEN) + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual(self.state.device_requests, 0) # no re-prompt + + def test_refresh_failure_falls_back_to_device_flow(self): + auth = self.make_auth() + self._seed_expired(auth) + self.state.refresh_response = (400, {'error': 'invalid_grant'}) + token = auth.token() + self.assertEqual(token, ID_TOKEN) + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual(self.state.device_requests, 1) # re-prompted + + def test_refresh_token_preserved_when_not_rotated(self): + auth = self.make_auth() + self._seed_expired(auth) + self.state.refresh_response = (200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'token_type': 'Bearer', 'expires_in': 3600}) # no new refresh + auth.token() + self.assertEqual(auth._tokens.refresh_token, 'REFRESH-1') + + def test_refresh_without_id_token_falls_back_to_device_flow(self): + # groups_in_token=True but the IdP's refresh omits the id_token: the + # refresh is unusable, so fall back to the interactive flow rather than + # caching it and looping (the device flow yields a complete token). + auth = self.make_auth(groups_in_token=True) + self._seed_expired(auth) + self.state.refresh_response = (200, { + 'access_token': ACCESS_TOKEN, 'token_type': 'Bearer', + 'expires_in': 3600}) # no id_token + token = auth.token() + self.assertEqual(token, ID_TOKEN) # from the device flow + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual(self.state.device_requests, 1) # fell back + + def test_refresh_without_id_token_non_interactive_does_not_loop(self): + # Same situation but non-interactive: surface a clear error rather than + # repeatedly re-running a refresh that can never satisfy _select. + auth = self.make_auth(groups_in_token=True, interactive=False) + self._seed_expired(auth) + self.state.refresh_response = (200, { + 'access_token': ACCESS_TOKEN, 'token_type': 'Bearer', + 'expires_in': 3600}) # no id_token + with self.assertRaises(OidcInteractionRequired): + auth.token() + self.assertEqual(self.state.device_requests, 0) + + def test_cached_token_missing_required_kind_is_refreshed(self): + # A cached, non-expired token that lacks the required kind (here: + # access_token in non-groups mode) must not pass the cache gate and + # then hard-fail in _select; it should trigger a refresh instead. + auth = self.make_auth(groups_in_token=False) + auth._cache.store(auth.cache_key, TokenSet( + access_token=None, id_token='id', refresh_token='REFRESH-1', + expires_at=self._clock.now() + 3600)) + token = auth.token() + self.assertEqual(token, ACCESS_TOKEN) + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual(self.state.device_requests, 0) + + def test_refresh_network_error_propagates_without_reprompt(self): + # Both endpoints point at a closed port (same origin, so the co-location + # check passes), so the refresh POST fails at the transport layer. The + # error must propagate from the *token* endpoint (the refresh), proving + # the flow did NOT fall back to the device flow on a transient blip. + clock = FakeClock() + auth = OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint='http://127.0.0.1:1/device', + token_endpoint='http://127.0.0.1:1/token', # connection refused + scope='openid groups', groups_in_token=True, cache='memory', + insecure=True, interactive=True, renderer=Renderer(), + _clock=clock) + expired = TokenSet( + access_token='old', id_token='old-id', refresh_token='REFRESH-1', + expires_at=clock.now() - 10) + auth._cache.store(auth.cache_key, expired) + + with self.assertRaises(OidcNetworkError) as cm: + auth.token() + # The error is from the refresh (token endpoint), not a device-flow + # fallback (device endpoint), and the refresh token is kept for a retry. + self.assertIn('/token', str(cm.exception)) + self.assertEqual(auth._tokens.refresh_token, 'REFRESH-1') + + +class TestFileCache(AuthTestBase): + def test_file_cache_works_without_os_lock(self): + # Exercise the no-fcntl/no-msvcrt fallback: with the lock primitives + # no-op'd, the atomic temp-file replace must still keep every entry. + import questdb.auth._cache as cache_mod + tmp = tempfile.mkdtemp() + path = os.path.join(tmp, 'cache.json') + with mock.patch.object(cache_mod, '_lock_fd', lambda fd: None), \ + mock.patch.object(cache_mod, '_unlock_fd', lambda fd: None): + cache = FileCache(path) + cache.store('k1', TokenSet(access_token='a1', expires_at=1.0)) + cache.store('k2', TokenSet(access_token='a2', expires_at=1.0)) + self.assertEqual(cache.load('k1').access_token, 'a1') + self.assertEqual(cache.load('k2').access_token, 'a2') + + def test_file_cache_survives_new_instance(self): + tmp = tempfile.mkdtemp() + path = os.path.join(tmp, 'cache.json') + cache1 = FileCache(path) + self.make_auth(cache=cache1).token() + self.assertEqual(self.state.device_requests, 1) + # New process simulation: fresh memory, load from file. + _MEMORY_STORE.clear() + cache2 = FileCache(path) + token = self.make_auth(cache=cache2).token() + self.assertEqual(token, ID_TOKEN) + self.assertEqual(self.state.device_requests, 1) # no re-prompt + # File is mode 600 where supported. + if os.name == 'posix': + self.assertEqual(os.stat(path).st_mode & 0o777, 0o600) + + @unittest.skipUnless( + _HAS_FCNTL, 'cross-process file lock requires fcntl (POSIX)') + def test_concurrent_writes_preserve_all_entries(self): + # 20 writers (distinct instances, same file, distinct keys) racing: + # the sidecar lock + atomic unique-temp replace must keep every entry + # and never corrupt the file or leave a temp behind. + tmp = tempfile.mkdtemp() + path = os.path.join(tmp, 'cache.json') + + def writer(i): + FileCache(path).store( + f'key-{i}', + TokenSet(access_token=f'a{i}', id_token=f'id{i}', + refresh_token=f'r{i}', expires_at=1.0)) + + threads = [threading.Thread(target=writer, args=(i,)) + for i in range(20)] + for t in threads: + t.start() + for t in threads: + t.join(10) + + final = FileCache(path) + for i in range(20): + ts = final.load(f'key-{i}') + self.assertIsNotNone(ts, f'lost entry key-{i}') + self.assertEqual(ts.access_token, f'a{i}') + leftovers = [n for n in os.listdir(tmp) if n.endswith('.tmp')] + self.assertEqual(leftovers, [], f'temp files left behind: {leftovers}') + + +class TestDiscovery(AuthTestBase): + def test_from_questdb_reads_settings(self): + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.scope': 'openid groups', + 'acl.oidc.groups.encoded.in.token': True, + 'acl.oidc.token.endpoint': self.base + '/token', + 'acl.oidc.device.authorization.endpoint': self.base + '/device', + }} + auth = OidcDeviceAuth.from_questdb( + self.base, insecure=True, interactive=True, renderer=Renderer(), + _clock=FakeClock()) + self.assertEqual(auth.config.client_id, 'questdb') + self.assertTrue(auth.config.groups_in_token) + self.assertEqual(auth.config.device_authorization_endpoint, + self.base + '/device') + self.assertEqual(auth.token(), ID_TOKEN) + + def test_well_known_fallback_for_device_endpoint(self): + # Settings advertise OIDC + token endpoint but NOT the device endpoint. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.scope': 'openid', + 'acl.oidc.groups.encoded.in.token': False, + 'acl.oidc.token.endpoint': self.base + '/token', + }} + self.state.well_known = { + 'issuer': self.base, + 'token_endpoint': self.base + '/token', + 'device_authorization_endpoint': self.base + '/device', + } + auth = OidcDeviceAuth.from_questdb(self.base, insecure=True, + renderer=Renderer()) + self.assertEqual(auth.config.device_authorization_endpoint, + self.base + '/device') + + def test_oidc_disabled_raises(self): + self.state.settings = {'config': {'acl.oidc.enabled': False}} + with self.assertRaises(OidcConfigError): + OidcDeviceAuth.from_questdb(self.base, insecure=True) + + def test_missing_device_endpoint_raises(self): + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': self.base + '/token', + }} + self.state.well_known = {'issuer': self.base, + 'token_endpoint': self.base + '/token'} + with self.assertRaises(OidcConfigError): + OidcDeviceAuth.from_questdb(self.base, insecure=True) + + def test_loopback_flow_not_implemented(self): + # Reserved-but-unimplemented flow raises an OidcError subclass so it's + # caught by `except OidcError` like other config problems. + with self.assertRaises(OidcConfigError): + OidcDeviceAuth.from_questdb(self.base, flow='loopback', + insecure=True) + + def test_endpoint_origin_mismatch_rejected(self): + # /settings advertises the device endpoint on a different origin than + # the token endpoint: refuse rather than POST credentials off-origin. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': self.base + '/token', + 'acl.oidc.device.authorization.endpoint': + 'http://127.0.0.2:9/device', # different host:port + }} + with self.assertRaises(OidcConfigError): + OidcDeviceAuth.from_questdb(self.base, insecure=True) + + def test_issuer_pin_rejects_off_origin_endpoints(self): + # Endpoints are internally consistent, but an explicit issuer pins them + # to a different origin -> reject (a compromised /settings can't + # redirect the token POST when the IdP is pinned). + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': self.base + '/token', + 'acl.oidc.device.authorization.endpoint': self.base + '/device', + }} + with self.assertRaises(OidcConfigError): + OidcDeviceAuth.from_questdb( + self.base, issuer='https://idp.attacker.example', + insecure=True) + + def test_issuer_pin_accepts_matching_origin(self): + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': self.base + '/token', + 'acl.oidc.device.authorization.endpoint': self.base + '/device', + }} + auth = OidcDeviceAuth.from_questdb( + self.base, issuer=self.base, insecure=True, renderer=Renderer()) + self.assertEqual(auth.config.device_authorization_endpoint, + self.base + '/device') + + +@unittest.skipIf(pd is None, 'pandas not installed') +class TestRestAdapter(AuthTestBase): + def _connected(self): + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.scope': 'openid groups', + 'acl.oidc.groups.encoded.in.token': True, + 'acl.oidc.token.endpoint': self.base + '/token', + 'acl.oidc.device.authorization.endpoint': self.base + '/device', + }} + self.state.expected_bearer = ID_TOKEN + return connect(self.base, insecure=True, renderer=Renderer(), + interactive=True, _clock=FakeClock()) + + def test_sql_returns_dataframe(self): + qdb = self._connected() + df = qdb.sql('SELECT * FROM trades') + self.assertEqual(list(df.columns), ['ts', 'price']) + self.assertEqual(len(df), 2) + self.assertEqual(df['price'].tolist(), [1.5, 2.5]) + # TIMESTAMP column coerced to datetime. + self.assertTrue(str(df['ts'].dtype).startswith('datetime64')) + + def test_sql_unauthorized_maps_to_auth_error(self): + qdb = self._connected() + self.state.expected_bearer = 'something-else' # force 401 + with self.assertRaises(OidcAuthError): + qdb.sql('SELECT 1') + + def test_connect_is_eager(self): + qdb = self._connected() + self.assertIsInstance(qdb, QuestDB) + # Sign-in already happened during connect(). + self.assertEqual(self.state.device_requests, 1) + + def test_sql_query_error_maps_to_oidc_error(self): + qdb = self._connected() + self.state.exec_status = 400 + self.state.exec_response = {'error': 'unexpected token', 'position': 5} + with self.assertRaises(OidcError) as cm: + qdb.sql('SELEKT 1') + self.assertIn('unexpected token', str(cm.exception)) + self.assertNotIsInstance(cm.exception, OidcAuthError) + + def test_sql_passes_limit(self): + qdb = self._connected() + qdb.sql('SELECT * FROM trades', limit='1,10') + self.assertTrue(any('limit=1' in p for p in self.state.exec_requests)) + + def test_sql_handles_empty_dataset(self): + qdb = self._connected() + self.state.exec_response = {'ddl': 'OK'} # no columns / dataset + df = qdb.sql('CREATE TABLE x (a INT)') + self.assertEqual(len(df), 0) + + def test_sql_malformed_shape_raises_oidc_error(self): + qdb = self._connected() + self.state.exec_response = { # rows shorter than the column list + 'columns': [{'name': 'a', 'type': 'LONG'}, + {'name': 'b', 'type': 'LONG'}], + 'dataset': [[1]]} + with self.assertRaises(OidcError): + qdb.sql('SELECT a, b FROM t') + + +class TestConcurrency(AuthTestBase): + def test_valid_cached_token_does_not_block_during_signin(self): + # A caller with a valid cached token must NOT block behind another + # thread's in-progress sign-in: the fast path takes no lock. + auth = self.make_auth() + valid = TokenSet( + access_token='a', id_token=ID_TOKEN, refresh_token='r', + expires_at=self._clock.now() + 3600) + auth._cache.store(auth.cache_key, valid) + + auth._lock.acquire() # simulate another thread mid-sign-in + try: + result = {} + t = threading.Thread( + target=lambda: result.update(tok=auth.token())) + t.start() + t.join(timeout=5) + self.assertFalse( + t.is_alive(), 'token() blocked behind an in-progress sign-in') + self.assertEqual(result.get('tok'), ID_TOKEN) + finally: + auth._lock.release() + + def test_concurrent_signin_prompts_only_once(self): + # Two threads racing with an empty cache must trigger exactly ONE + # device flow; the loser reuses the winner's token. + auth = self.make_auth() + entered = threading.Event() + release = threading.Event() + + class GatingRenderer(Renderer): + def on_prompt(self, resp): + entered.set() # first thread is now inside the flow + release.wait(5) # ...holding the acquisition lock + + auth._renderer = GatingRenderer() + results = {} + + def call(name): + try: + results[name] = auth.token() + except Exception as e: # noqa: BLE001 + results[name] = e + + t1 = threading.Thread(target=call, args=('a',)) + t1.start() + self.assertTrue(entered.wait(5)) # t1 holds the lock in the flow + t2 = threading.Thread(target=call, args=('b',)) + t2.start() + release.set() # let t1 finish signing in + t1.join(5) + t2.join(5) + self.assertEqual(results.get('a'), ID_TOKEN) + self.assertEqual(results.get('b'), ID_TOKEN) + self.assertEqual(self.state.device_requests, 1) # no second prompt + + +class TestAdapters(unittest.TestCase): + """Connection adapters: tested via injected fake modules (the real + sqlalchemy / psycopg / questdb.ingress need not be installed).""" + + def _qdb(self, url='http://db.example.com:9000', token='TKN'): + return QuestDB(url, _FakeAuth(token), insecure=True) + + def test_sender_builds_conf_with_token(self): + qdb = self._qdb('http://db.example.com:9000', token='TKN') + captured = {} + + fake = types.ModuleType('questdb.ingress') + + class Sender: + @staticmethod + def from_conf(conf, *, token=None, **kw): + captured.update(conf=conf, token=token, kw=kw) + return 'SENDER' + + fake.Sender = Sender + with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): + sender = qdb.sender(auto_flush=False) + self.assertEqual(sender, 'SENDER') + self.assertEqual(captured['conf'], 'http::addr=db.example.com:9000;') + self.assertEqual(captured['token'], 'TKN') + self.assertEqual(captured['kw'], {'auto_flush': False}) + + def test_sender_https_defaults_to_443(self): + qdb = self._qdb('https://db.example.com') # no explicit port + captured = {} + fake = types.ModuleType('questdb.ingress') + + class Sender: + @staticmethod + def from_conf(conf, *, token=None, **kw): + captured['conf'] = conf + return 'S' + + fake.Sender = Sender + with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): + qdb.sender() + self.assertEqual(captured['conf'], 'https::addr=db.example.com:443;') + + def test_psycopg_connects_as_sso_with_token(self): + qdb = self._qdb('http://db.example.com:9000', token='TKN') + captured = {} + fake = types.ModuleType('psycopg') + + def connect(**kw): + captured.update(kw) + return 'CONN' + + fake.connect = connect + with mock.patch.dict(sys.modules, {'psycopg': fake}): + conn = qdb.psycopg(connect_timeout=3) + self.assertEqual(conn, 'CONN') + self.assertEqual(captured['user'], '_sso') + self.assertEqual(captured['password'], 'TKN') + self.assertEqual(captured['host'], 'db.example.com') + self.assertEqual(captured['port'], 8812) + self.assertEqual(captured['dbname'], 'qdb') + self.assertEqual(captured['connect_timeout'], 3) + # The token is fetched at connect time (fresh per connection). + self.assertEqual(qdb.auth.calls, 1) + + def test_sqlalchemy_engine_injects_fresh_token_per_connect(self): + auth = _FakeAuth('TKN') + qdb = QuestDB('http://db.example.com:9000', auth, insecure=True) + created = {} + events = {} + engine_obj = object() + + fake_sa = types.ModuleType('sqlalchemy') + fake_sa.__path__ = [] + + def create_engine(url, **kw): + created.update(url=url, engine_kw=kw) + return engine_obj + + class _Event: + @staticmethod + def listens_for(target, name): + def deco(fn): + events.update(name=name, fn=fn) + return fn + return deco + + fake_sa.create_engine = create_engine + fake_sa.event = _Event + + fake_eng = types.ModuleType('sqlalchemy.engine') + + class _URL: + @staticmethod + def create(**kw): + created.update(kw) + return 'URL' + + fake_eng.URL = _URL + fake_pg = types.ModuleType('psycopg') # drives the drivername choice + + with mock.patch.dict(sys.modules, { + 'sqlalchemy': fake_sa, + 'sqlalchemy.engine': fake_eng, + 'psycopg': fake_pg}): + engine = qdb.sqlalchemy_engine(pool_pre_ping=True) + + self.assertIs(engine, engine_obj) + self.assertEqual(created['drivername'], 'postgresql+psycopg') + self.assertEqual(created['username'], '_sso') + self.assertEqual(created['host'], 'db.example.com') + self.assertEqual(created['port'], 8812) + self.assertEqual(created['database'], 'qdb') + self.assertEqual(created['url'], 'URL') + self.assertEqual(created['engine_kw'], {'pool_pre_ping': True}) + self.assertEqual(events['name'], 'do_connect') + # The listener injects a fresh token on each new connection. + before = auth.calls + for _ in range(2): + cparams = {} + events['fn'](None, None, [], cparams) + self.assertEqual(cparams['password'], 'TKN') + self.assertEqual(auth.calls - before, 2) + + def test_sql_missing_pandas_raises(self): + qdb = self._qdb() + with mock.patch.dict(sys.modules, {'pandas': None}): + with self.assertRaises(ImportError): + qdb.sql('SELECT 1') + + @unittest.skipIf(importlib.util.find_spec('sqlalchemy') is not None, + 'sqlalchemy installed') + def test_sqlalchemy_engine_missing_dep_raises(self): + with self.assertRaises(ImportError): + self._qdb().sqlalchemy_engine() + + @unittest.skipIf(_HAS_PG_DRIVER, 'a PostgreSQL driver is installed') + def test_psycopg_missing_dep_raises(self): + with self.assertRaises(ImportError): + self._qdb().psycopg() + + @unittest.skipIf(importlib.util.find_spec('questdb.ingress') is not None, + 'questdb.ingress extension is built') + def test_sender_missing_extension_raises(self): + with self.assertRaises(ImportError): + self._qdb().sender() + + +class TestConfigHelpers(unittest.TestCase): + def test_as_bool_variants(self): + from questdb.auth._discovery import _as_bool + for v in ('true', 'True', '1', 'yes', 'on', True, 1): + self.assertIs(_as_bool(v), True) + for v in ('false', '0', 'no', 'off', '', False, 0): + self.assertIs(_as_bool(v), False) + self.assertIsNone(_as_bool(None)) + self.assertIs(_as_bool(None, default=True), True) + + def test_resolve_endpoint_relative_path(self): + from questdb.auth._discovery import _resolve_endpoint + cfg = {'acl.oidc.host': 'idp.example.com', + 'acl.oidc.tls.enabled': True, 'acl.oidc.port': 443} + self.assertEqual(_resolve_endpoint('/as/token.oauth2', cfg), + 'https://idp.example.com:443/as/token.oauth2') + self.assertEqual(_resolve_endpoint('https://idp/x', cfg), + 'https://idp/x') # absolute is kept verbatim + + def test_settings_config_nesting(self): + from questdb.auth._discovery import settings_config + self.assertEqual(settings_config({'config': {'a': 1}}), {'a': 1}) + self.assertEqual(settings_config({'a': 1}), {'a': 1}) # flat fallback + + +class TestEndpointValidation(unittest.TestCase): + def setUp(self): + from questdb.auth._discovery import validate_endpoint_origins + self._validate = validate_endpoint_origins + + def test_default_port_equivalence_accepted(self): + # https default (443) vs explicit :443 normalize to the same origin. + self._validate('https://idp/token', 'https://idp:443/device') + + def test_ipv6_same_origin_accepted(self): + self._validate('https://[::1]/token', 'https://[::1]/device') + + def test_off_origin_device_rejected(self): + with self.assertRaises(OidcConfigError): + self._validate('https://idp/token', 'https://evil.example/device') + + def test_both_endpoints_off_issuer_rejected(self): + # Endpoints agree with each other but not with the pinned issuer: + # the issuer-pin loop must check both, not just their consistency. + with self.assertRaises(OidcConfigError): + self._validate('https://idp/token', 'https://idp/device', + issuer='https://other-issuer.example') + + def test_explicit_constructor_enforces_co_location(self): + with self.assertRaises(OidcConfigError): + OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint='https://idp.example.com/device', + token_endpoint='https://attacker.example/token', + renderer=Renderer()) + + +class TestCacheKey(unittest.TestCase): + def _auth(self, **kw): + opts = dict( + client_id='questdb', + device_authorization_endpoint='https://idp.example.com/device', + token_endpoint='https://idp.example.com/token', + scope='openid groups', groups_in_token=True, cache='memory', + renderer=Renderer()) + opts.update(kw) + return OidcDeviceAuth(**opts) + + def test_realm_path_distinguishes_key(self): + # Multi-tenant IdP: same host, different realm path -> distinct keys + # (the old origin-only key collided, leaking one realm's token). + a = self._auth( + token_endpoint='https://idp.example.com/realmA/token', + device_authorization_endpoint='https://idp.example.com/realmA/dev') + b = self._auth( + token_endpoint='https://idp.example.com/realmB/token', + device_authorization_endpoint='https://idp.example.com/realmB/dev') + self.assertNotEqual(a.cache_key, b.cache_key) + + def test_scope_order_does_not_change_key(self): + self.assertEqual( + self._auth(scope='openid groups').cache_key, + self._auth(scope='groups openid').cache_key) + + def test_audience_distinguishes_key(self): + self.assertNotEqual( + self._auth(audience='aud-1').cache_key, + self._auth(audience='aud-2').cache_key) + + def test_default_port_normalized(self): + self.assertEqual( + self._auth(token_endpoint='https://idp.example.com/token').cache_key, + self._auth( + token_endpoint='https://idp.example.com:443/token').cache_key) + + +class TestTransportSecurity(unittest.TestCase): + def test_require_secure_policy(self): + from questdb.auth._http import _require_secure + # https is always fine. + _require_secure('https://idp.example.com/x', insecure=False) + # loopback http never leaves the host -> always allowed. + _require_secure('http://127.0.0.1:9000/x', insecure=False) + _require_secure('http://localhost/x', insecure=False) + _require_secure('http://[::1]:8080/x', insecure=False) + # non-loopback http is refused unless insecure is explicitly set. + with self.assertRaises(OidcConfigError): + _require_secure('http://idp.example.com/x', insecure=False) + _require_secure('http://idp.example.com/x', insecure=True) + + def test_insecure_does_not_downgrade_idp(self): + # insecure=True must NOT permit plaintext to a non-loopback IdP: the + # device code / refresh token must never traverse the network in clear. + auth = OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint='http://idp.example.com/device', + token_endpoint='http://idp.example.com/token', + scope='openid', groups_in_token=False, cache='memory', + insecure=True, interactive=True, renderer=Renderer(), + _clock=FakeClock()) + with self.assertRaises(OidcConfigError): + auth.token() + + +class TestRendererSecurity(unittest.TestCase): + """The Jupyter prompt must never turn an IdP-supplied URL into a + clickable/executable link unless it uses an http(s) scheme.""" + + def test_safe_link_url_allowlist(self): + from questdb.auth._render import _safe_link_url + self.assertEqual(_safe_link_url('https://idp/x'), 'https://idp/x') + self.assertEqual(_safe_link_url('http://idp/x'), 'http://idp/x') + self.assertEqual(_safe_link_url('HTTPS://idp/x'), 'HTTPS://idp/x') + for bad in ('javascript:alert(1)', 'data:text/html,x', + 'vbscript:x', 'file:///etc/passwd', '', None): + self.assertIsNone(_safe_link_url(bad)) + + def test_render_link_inert_for_dangerous_scheme(self): + from questdb.auth._render import _render_link + safe = _render_link('https://idp/x') + self.assertIn(' Date: Mon, 15 Jun 2026 22:18:44 +0100 Subject: [PATCH 002/104] fix: pandas 3 string dtype in test_parquet_roundtrip The fastparquet -> pyarrow parquet roundtrip decays the categorical column to a plain string column. On pandas >= 3 that reads back as the new default string dtype (StringDtype(na_value=nan)) rather than object, so the hardcoded np.dtype('O') in fallback_exp_dtypes no longer matched and the assertion failed. Derive the expected dtype from pd.Series(['x']).dtype instead of hardcoding it: this is object on pandas < 3 and the new string dtype on pandas >= 3, matching exactly what pyarrow's read_parquet produces, so the test is version-agnostic. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/test_dataframe.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/test_dataframe.py b/test/test_dataframe.py index 0bde05cf..de58758f 100644 --- a/test/test_dataframe.py +++ b/test/test_dataframe.py @@ -1897,14 +1897,19 @@ def df_eq(exp_df, deser_df, exp_dtypes): self.assertTrue(exp_df.equals(deser_df)) # fastparquet doesn't roundtrip with pyarrow parquet properly. - # It decays categories to object and UInt8 to float64. + # It decays categories to plain strings and UInt8 to float64. # We need to set up special case expected results for that. + # The decayed string column comes back as whatever this pandas + # infers for a string column: object on pandas < 3, but the new + # default string dtype (StringDtype(na_value=nan)) on pandas >= 3. + # Derive it instead of hardcoding so the test is version-agnostic. + str_dtype = pd.Series(['x']).dtype fallback_exp_dtypes = [ - np.dtype('O'), + str_dtype, np.dtype('int16'), np.dtype('float64'), np.dtype('float64')] - fallback_df = df.astype({'s': 'object', 'b': 'float64'}) + fallback_df = df.astype({'s': str_dtype, 'b': 'float64'}) df_eq(df, pa2pa_df, exp_dtypes) if fp_wrote: From e394fbdf2e8e24254d48fbc0b6e8cb11e954b82a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 15 Jun 2026 23:13:06 +0100 Subject: [PATCH 003/104] ci: keep 32-bit wheel tests on the pandas 2 / numpy 1 path Pandas 3 ships no 32-bit wheels. On win32 Python 3.11+ the pandas>=3 install was silently swallowed, fastparquet then pulled in a numpy-1-built pandas 2.0.3 alongside numpy 2, and importing pandas crashed at runtime. test.py tolerated the failed import and silently skipped every pandas test (skip count 39 vs 32), while the 64-bit-only import sanity check never fired to catch it. Gate should_use_pandas3() on a 64-bit interpreter so 32-bit targets stay on the consistent pandas 2 / numpy 1 stack and actually exercise the dataframe tests again. 64-bit targets keep testing pandas 3 unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- ci/pip_install_deps.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ci/pip_install_deps.py b/ci/pip_install_deps.py index d70b9761..e3ee25b4 100644 --- a/ci/pip_install_deps.py +++ b/ci/pip_install_deps.py @@ -77,12 +77,18 @@ def install_pandas3_and_numpy(): def should_use_pandas3(py_version=None): if py_version is None: py_version = sys.version_info[:2] - return py_version >= (3, 11) + # Pandas 3 ships no 32-bit wheels, so only take the pandas 3 / numpy 2 + # path on 64-bit interpreters. On 32-bit (e.g. win32) the pandas 3 install + # would be silently skipped, fastparquet would then drag in a numpy-1-built + # pandas 2.0.3 alongside numpy 2, and importing pandas would crash. + is_64bits = sys.maxsize > 2 ** 32 + return is_64bits and py_version >= (3, 11) def install_default_pandas_and_numpy(): - # Pandas 3 currently requires Python 3.11+, so keep 3.10 wheel tests on - # the pandas 2 / numpy 1.x-compatible path unless explicitly overridden. + # Pandas 3 requires Python 3.11+ and ships only 64-bit wheels, so keep + # 3.10 and all 32-bit wheel tests on the pandas 2 / numpy 1.x-compatible + # path unless explicitly overridden. if should_use_pandas3(): install_pandas3_and_numpy() else: From ae9217881a2fcce1579d1eccc01e27e0a494537b Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 16 Jun 2026 00:39:08 +0100 Subject: [PATCH 004/104] test: silence mock server tracebacks on Windows client disconnect The mock HTTP server only caught BrokenPipeError, but on Windows an abrupt client disconnect raises ConnectionAbortedError/ConnectionResetError -- siblings of BrokenPipeError under the common base ConnectionError, not subclasses of it. The timeout, min-throughput, and retry tests disconnect mid-request on purpose, so these slipped past the handler and the stdlib dumped tracebacks to stderr. The tests still passed, but the CI logs looked broken. Broaden the handler except clauses to ConnectionError, and override HTTPServer.handle_error to swallow connection errors that surface in the stdlib keep-alive readline of the next request line -- outside any request handler's try/except. Real errors are still reported. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/mock_server.py | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/test/mock_server.py b/test/mock_server.py index 6178a4f7..708be7f4 100644 --- a/test/mock_server.py +++ b/test/mock_server.py @@ -3,6 +3,7 @@ import select import re import http.server as hs +import sys import threading import time import struct @@ -121,6 +122,21 @@ def __exit__(self, _ex_type, _ex_value, _ex_tb): SETTINGS_WITH_PROTOCOL_VERSION_V1_V2_V3 = '{"config":{"release.type":"OSS","release.version":"[DEVELOPMENT]","line.proto.support.versions":[1,2,3],"ilp.proto.transports":["tcp","http"],"posthog.enabled":false,"posthog.api.key":null,"cairo.max.file.name.length":127},"preferences.version":0,"preferences":{}}' SETTINGS_WITHOUT_PROTOCOL_VERSION = '{ "release.type": "OSS", "release.version": "[DEVELOPMENT]", "acl.enabled": false, "posthog.enabled": false, "posthog.api.key": null }' +class _QuietHTTPServer(hs.HTTPServer): + """HTTPServer that stays quiet when a client disconnects abruptly. + + Several tests (e.g. the request-timeout and min-throughput cases) drop the + connection mid-request on purpose. The stdlib would otherwise print a + harmless but noisy traceback for the resulting connection error -- most + visibly on Windows, where the keep-alive read of the next request line + raises ConnectionResetError outside of any request handler's try/except. + """ + def handle_error(self, request, client_address): + if isinstance(sys.exc_info()[1], ConnectionError): + return + super().handle_error(request, client_address) + + class HttpServer: def __init__(self, settings=SETTINGS_WITH_PROTOCOL_VERSION_V1_V2_V3, delay_seconds=0): self.delay_seconds = delay_seconds @@ -162,7 +178,12 @@ def do_GET(self): else: self.send_error(404, "Endpoint not found") self.close_connection = False - except BrokenPipeError: + except ConnectionError: + # The client (sender under test) may disconnect mid-request, + # e.g. in the timeout / min-throughput tests. On Windows this + # surfaces as ConnectionAbortedError/ConnectionResetError + # rather than the BrokenPipeError seen on Unix; both derive + # from ConnectionError. pass def do_POST(self): @@ -187,7 +208,12 @@ def do_POST(self): if body: self.wfile.write(body) self.close_connection = False - except BrokenPipeError: + except ConnectionError: + # The client (sender under test) may disconnect mid-request, + # e.g. in the timeout / min-throughput tests. On Windows this + # surfaces as ConnectionAbortedError/ConnectionResetError + # rather than the BrokenPipeError seen on Unix; both derive + # from ConnectionError. pass return IlpHttpHandler @@ -195,7 +221,7 @@ def do_POST(self): def __enter__(self): self._stop_event = threading.Event() handler_class = self.create_handler() - self._http_server = hs.HTTPServer(('', 0), handler_class, bind_and_activate=True) + self._http_server = _QuietHTTPServer(('', 0), handler_class, bind_and_activate=True) self._http_server.timeout = 30 self._http_server_thread = threading.Thread(target=self._serve) self._http_server_thread.start() From 88308d5de98db7738e8522ba03c07d143cff9e34 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 17 Jun 2026 17:16:03 +0100 Subject: [PATCH 005/104] ci: skip readonly AZP_ENHANCED agent var in Windows wheel build The windows-2025 image injects a new readonly agent variable, AZP_ENHANCED_WORKER_CRASH_HANDLING. The Windows "Build wheels" step re-exports the vcvars environment via ##vso[task.setvariable ...], and attempting to set this readonly var made the agent emit an ##[error], marking the task failed even though all wheels built and tests passed. Add AZP_ENHANCED to the exclusion regex (prefix match also covers any future AZP_ENHANCED_* vars) in both windows_i686 and windows_x86_64. Co-Authored-By: Claude Opus 4.8 (1M context) --- ci/cibuildwheel.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/cibuildwheel.yaml b/ci/cibuildwheel.yaml index c0d31767..f6ca473f 100644 --- a/ci/cibuildwheel.yaml +++ b/ci/cibuildwheel.yaml @@ -107,7 +107,7 @@ stages: cmd /c "call `"$vsPath`" && set > env_vars.txt" Get-Content env_vars.txt | ForEach-Object { - if ($_ -match "^([^=]+?)=(.*)$" -and $matches[1] -notmatch '^(SYSTEM|AGENT|BUILD|RELEASE|VSTS|TASK|USE_|FAIL_|MSDEPLOY|AZP_75787|AZP_AGENT|AZP_ENABLE|AZURE_HTTP|COPYFILESOVERSSHV0|ENABLE_ISSUE_SOURCE_VALIDATION|MODIFY_NUMBER_OF_RETRIES_IN_ROBOCOPY|MSBUILDHELPERS_ENABLE_TELEMETRY|RETIRE_AZURERM_POWERSHELL_MODULE|ROSETTA2_WARNING|AZP_PS_ENABLE)') { + if ($_ -match "^([^=]+?)=(.*)$" -and $matches[1] -notmatch '^(SYSTEM|AGENT|BUILD|RELEASE|VSTS|TASK|USE_|FAIL_|MSDEPLOY|AZP_75787|AZP_AGENT|AZP_ENABLE|AZP_ENHANCED|AZURE_HTTP|COPYFILESOVERSSHV0|ENABLE_ISSUE_SOURCE_VALIDATION|MODIFY_NUMBER_OF_RETRIES_IN_ROBOCOPY|MSBUILDHELPERS_ENABLE_TELEMETRY|RETIRE_AZURERM_POWERSHELL_MODULE|ROSETTA2_WARNING|AZP_PS_ENABLE)') { [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2], "Process") Write-Host "##vso[task.setvariable variable=$($matches[1])]$($matches[2])" } @@ -137,7 +137,7 @@ stages: cmd /c "call `"$vsPath`" && set > env_vars.txt" Get-Content env_vars.txt | ForEach-Object { - if ($_ -match "^([^=]+?)=(.*)$" -and $matches[1] -notmatch '^(SYSTEM|AGENT|BUILD|RELEASE|VSTS|TASK|USE_|FAIL_|MSDEPLOY|AZP_75787|AZP_AGENT|AZP_ENABLE|AZURE_HTTP|COPYFILESOVERSSHV0|ENABLE_ISSUE_SOURCE_VALIDATION|MODIFY_NUMBER_OF_RETRIES_IN_ROBOCOPY|MSBUILDHELPERS_ENABLE_TELEMETRY|RETIRE_AZURERM_POWERSHELL_MODULE|ROSETTA2_WARNING|AZP_PS_ENABLE)') { + if ($_ -match "^([^=]+?)=(.*)$" -and $matches[1] -notmatch '^(SYSTEM|AGENT|BUILD|RELEASE|VSTS|TASK|USE_|FAIL_|MSDEPLOY|AZP_75787|AZP_AGENT|AZP_ENABLE|AZP_ENHANCED|AZURE_HTTP|COPYFILESOVERSSHV0|ENABLE_ISSUE_SOURCE_VALIDATION|MODIFY_NUMBER_OF_RETRIES_IN_ROBOCOPY|MSBUILDHELPERS_ENABLE_TELEMETRY|RETIRE_AZURERM_POWERSHELL_MODULE|ROSETTA2_WARNING|AZP_PS_ENABLE)') { [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2], "Process") Write-Host "##vso[task.setvariable variable=$($matches[1])]$($matches[2])" } From d5df18634965f9547be32e601d039c3c4392a495 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 17 Jun 2026 19:41:24 +0100 Subject: [PATCH 006/104] fix: harden OIDC device-flow auth; drop on-disk FileCache backend Review fixes on the OIDC device-flow auth module, plus a simplification that removes the file cache entirely. Hardening: - device-flow poll gates success on _has_required_token (id_token in groups mode, else access_token) instead of always access_token: a completed grant missing the required kind now fails once with a clear error instead of caching an unusable token or discarding a usable id_token. - QuestDB.sql() guards the 2xx path against non-JSON / non-dict bodies, raising OidcError instead of a raw JSONDecodeError / AttributeError. - discovery requires an explicit issuer= (or discovery_url=) before the IdP .well-known fallback; the discovery origin is never derived from a server-supplied token endpoint, so a tampered /settings can't redirect the device-code / refresh-token POSTs. - PG-wire / ILP adapters bracket IPv6 literals in the ILP addr= and raise a clear error on a host-less URL instead of passing None to the driver. - validate_endpoint_origins and cache-key normalization raise OidcConfigError (not a bare ValueError) on a malformed port, via a shared safe_urlparse helper. - the example imports questdb.ingress lazily, so it loads on the pure-Python path with no compiled extension. Simplification: - drop FileCache and its cross-process locking + at-rest refresh token; MemoryCache (process-global, survives notebook cell re-runs) is the only persistent backend, with NullCache for cache=None. This also removes the Windows msvcrt-lock no-op and corrupt-file edge cases entirely. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/auth.rst | 21 +-- examples/oidc_device_auth.py | 6 +- src/questdb/auth/__init__.py | 3 +- src/questdb/auth/_cache.py | 147 +------------------ src/questdb/auth/_device.py | 47 ++++++- src/questdb/auth/_discovery.py | 59 ++++---- src/questdb/auth/_http.py | 17 +++ src/questdb/auth/_questdb.py | 57 ++++++-- test/test.py | 1 - test/test_auth.py | 250 +++++++++++++++++++++++---------- 10 files changed, 335 insertions(+), 273 deletions(-) diff --git a/docs/auth.rst b/docs/auth.rst index 7e40ada5..d44e6b1e 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -103,9 +103,8 @@ order: ``acl.oidc.device.authorization.endpoint``. 2. If the device-authorization endpoint is not advertised, the helper falls back to the IdP discovery document - (``{issuer}/.well-known/openid-configuration``). The issuer is taken from an - explicit ``issuer=`` / ``discovery_url=`` argument, or derived from the token - endpoint's origin. + (``{issuer}/.well-known/openid-configuration``). This path **requires** an + explicit ``issuer=`` (or ``discovery_url=``) argument. Anything you pass explicitly overrides discovery. You can also skip discovery entirely: @@ -150,11 +149,12 @@ Cache backends (``cache=`` argument): * ``"memory"`` *(default)* — process-global, nothing written to disk. Re-running cells is silent; a kernel restart re-prompts once. -* ``"file"`` — ``~/.questdb/oidc-cache.json`` (mode ``600``). Survives kernel - restarts and is shared across kernels on the same host. **Security - trade-off:** the refresh token is stored at rest. * ``None`` — never persist; prompt every time. +Tokens are deliberately never written to disk: a kernel restart re-prompts +(an interactive sign-in is cheap relative to the risk of a refresh token +sitting in a plaintext file at rest). + Non-interactive contexts ------------------------- @@ -214,9 +214,12 @@ Security notes QuestDB ``/settings``. The helper requires both endpoints to share a single origin and rejects the configuration otherwise. Because ``/settings`` is authoritative-by-QuestDB, a compromised server could in principle point them - elsewhere; pass ``issuer=`` (or ``discovery_url=``) to **pin** the IdP so the - endpoints are verified to belong to it and credentials can't be redirected to - another host. + elsewhere; pass ``issuer=`` to **pin** the IdP so the endpoints are verified + to belong to it and credentials can't be redirected to another host. When the + server does not advertise the device-authorization endpoint (so it must be + discovered from the IdP), ``issuer=`` (or ``discovery_url=``) is **required** + for exactly this reason — the helper refuses to guess the discovery origin + from the server-supplied token endpoint. * Adapters avoid logging the token / PG DSN. Avoid logging them yourself. * Standard proxy / CA settings (``HTTPS_PROXY``, ``REQUESTS_CA_BUNDLE``, ``SSL_CERT_FILE``) are honoured; you can also pass ``ca_bundle=``. diff --git a/examples/oidc_device_auth.py b/examples/oidc_device_auth.py index 2d7d4535..1691659f 100644 --- a/examples/oidc_device_auth.py +++ b/examples/oidc_device_auth.py @@ -13,7 +13,6 @@ import sys from questdb.auth import connect, OidcDeviceAuth, OidcError -from questdb.ingress import TimestampNanos QUESTDB_URL = 'https://questdb.example.com:9000' @@ -32,6 +31,11 @@ def integrated(url: str = QUESTDB_URL): # Feed the same auto-refreshed token into your existing tooling: # engine = qdb.sqlalchemy_engine() # PG-wire, token as _sso password # with qdb.psycopg() as conn: ... # raw psycopg + # + # questdb.ingress is the compiled extension; import it lazily (only the + # ingestion path needs it) so this module also loads for the pure-Python + # bring_your_own_client() path, which needs no extension. + from questdb.ingress import TimestampNanos with qdb.sender() as sender: # ingestion (ILP over HTTP) sender.row( 'trades', diff --git a/src/questdb/auth/__init__.py b/src/questdb/auth/__init__.py index e3768bca..f6ba9d1f 100644 --- a/src/questdb/auth/__init__.py +++ b/src/questdb/auth/__init__.py @@ -60,7 +60,7 @@ from ._device import OidcDeviceAuth from ._discovery import OidcConfig -from ._cache import TokenCache, TokenSet, FileCache, MemoryCache, NullCache +from ._cache import TokenCache, TokenSet, MemoryCache, NullCache from ._errors import ( OidcError, OidcConfigError, @@ -80,7 +80,6 @@ 'TokenCache', 'TokenSet', 'MemoryCache', - 'FileCache', 'NullCache', 'OidcError', 'OidcConfigError', diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index 858e113e..be66ee9a 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -26,13 +26,8 @@ from __future__ import annotations -import contextlib -import json -import os -import pathlib -import tempfile import threading -from dataclasses import asdict, dataclass, replace +from dataclasses import dataclass, replace from typing import Dict, Optional, Union from ._errors import OidcConfigError @@ -67,14 +62,6 @@ def is_valid(self, now: float, skew: float = DEFAULT_SKEW_SECONDS) -> bool: skew = min(skew, lifetime / 2) return now < (self.expires_at - skew) - def to_dict(self) -> Dict[str, object]: - return asdict(self) - - @classmethod - def from_dict(cls, d: Dict[str, object]) -> 'TokenSet': - known = {f for f in cls.__dataclass_fields__} # noqa: C416 - return cls(**{k: v for k, v in d.items() if k in known}) - class TokenCache: """Interface for token caches.""" @@ -133,145 +120,17 @@ def clear(self, key: str) -> None: pass -# Cross-process file locking, used to serialize read-modify-write on the -# shared cache file. fcntl.flock (POSIX) also serializes across threads/ -# instances in one process (locks are per open file description). Where no OS -# primitive is available it degrades to a best-effort no-op; the atomic -# os.replace still guarantees readers never see a torn file. -try: - import fcntl - - def _lock_fd(fd: int) -> None: - fcntl.flock(fd, fcntl.LOCK_EX) - - def _unlock_fd(fd: int) -> None: - fcntl.flock(fd, fcntl.LOCK_UN) -except ImportError: # pragma: no cover - non-POSIX (e.g. Windows) - try: - import msvcrt - - def _lock_fd(fd: int) -> None: - try: - msvcrt.locking(fd, msvcrt.LK_LOCK, 1) - except OSError: - pass - - def _unlock_fd(fd: int) -> None: - try: - msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) - except OSError: - pass - except ImportError: # pragma: no cover - def _lock_fd(fd: int) -> None: - pass - - def _unlock_fd(fd: int) -> None: - pass - - -@contextlib.contextmanager -def _interprocess_lock(lock_path: pathlib.Path): - """Best-effort exclusive lock via a sidecar lock file.""" - fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR, 0o600) - try: - _lock_fd(fd) - try: - yield - finally: - _unlock_fd(fd) - finally: - os.close(fd) - - -class FileCache(TokenCache): - """ - Opt-in on-disk cache at ``~/.questdb/oidc-cache.json`` (mode ``600``). - - Survives kernel restarts and is shared across kernels on the same host. - Security trade-off: a refresh token is stored at rest. The file is created - owner-only (``0600``) from the start via an atomic temp-file replace, and a - sidecar lock file serializes concurrent read-modify-writes across kernels - so entries are not corrupted or lost. - """ - - def __init__(self, path: Optional[Union[str, os.PathLike]] = None): - if path is None: - path = pathlib.Path.home() / '.questdb' / 'oidc-cache.json' - self.path = pathlib.Path(path) - self._lock_path = self.path.with_name(self.path.name + '.lock') - - def _ensure_dir(self) -> None: - parent = self.path.parent - parent.mkdir(parents=True, exist_ok=True) - try: - os.chmod(parent, 0o700) - except OSError: - pass - - def _read_all(self) -> Dict[str, dict]: - try: - with open(self.path, 'r', encoding='utf-8') as f: - data = json.load(f) - if isinstance(data, dict): - return data - except (FileNotFoundError, ValueError, OSError): - pass - return {} - - def _write_all(self, data: Dict[str, dict]) -> None: - # Atomic, owner-only replace. mkstemp creates the file mode 0600 with a - # unique name, so concurrent writers never share a temp file and the - # refresh token is never group/world-readable, even briefly. - fd, tmp = tempfile.mkstemp( - dir=str(self.path.parent), prefix='.oidc-', suffix='.tmp') - try: - with os.fdopen(fd, 'w', encoding='utf-8') as f: - json.dump(data, f) - os.replace(tmp, self.path) - except BaseException: - with contextlib.suppress(OSError): - os.unlink(tmp) - raise - - def load(self, key: str) -> Optional[TokenSet]: - # Lock-free: the atomic replace guarantees a complete file is read. - entry = self._read_all().get(key) - if isinstance(entry, dict): - try: - return TokenSet.from_dict(entry) - except TypeError: - return None - return None - - def store(self, key: str, tokens: TokenSet) -> None: - self._ensure_dir() - with _interprocess_lock(self._lock_path): - data = self._read_all() - data[key] = tokens.to_dict() - self._write_all(data) - - def clear(self, key: str) -> None: - self._ensure_dir() - with _interprocess_lock(self._lock_path): - data = self._read_all() - if key in data: - del data[key] - self._write_all(data) - - _CacheSpec = Union[str, None, TokenCache] def make_cache(spec: _CacheSpec) -> TokenCache: - """Resolve a cache spec (``"memory"`` / ``"file"`` / ``None`` / instance).""" + """Resolve a cache spec (``"memory"`` / ``None`` / a TokenCache instance).""" if isinstance(spec, TokenCache): return spec if spec is None or spec == 'none': return NullCache() if spec == 'memory': return MemoryCache() - if spec == 'file': - return FileCache() raise OidcConfigError( f'Unknown cache backend {spec!r}; ' - "expected 'memory', 'file', None, or a TokenCache instance.") + "expected 'memory', None, or a TokenCache instance.") diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 45694c64..27711814 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -31,7 +31,6 @@ import json import threading import time -import urllib.parse import webbrowser from typing import Any, Dict, Optional @@ -45,7 +44,7 @@ OidcNetworkError, OidcTimeoutError, ) -from ._http import build_ssl_context, post_form +from ._http import build_ssl_context, post_form, safe_urlparse from ._render import ( Renderer, _safe_link_url, @@ -344,6 +343,24 @@ def _has_required_token(self, tokens: TokenSet) -> bool: return bool(tokens.id_token) return bool(tokens.access_token) + def _missing_required_token_error(self) -> OidcDeviceFlowError: + """ + Build the terminal error for a *completed* grant whose token response + omits the kind :meth:`_select` needs (the ``id_token`` in groups mode, + else the ``access_token``). Mirrors :meth:`_select`'s diagnostics, but + is an :class:`OidcDeviceFlowError` — a flow failure — so the device-flow + poll can raise it without first caching an unusable response. + """ + if self.config.groups_in_token: + return OidcDeviceFlowError( + 'Device authorization completed but the IdP returned no ' + 'id_token, which this server requires (it expects groups ' + 'encoded in the token). Ensure the "openid" scope is requested ' + f'(current scope: {self.config.scope!r}).') + return OidcDeviceFlowError( + 'Device authorization completed but the IdP returned no ' + 'access_token.') + def _obtain_tokens(self) -> TokenSet: # Fast path: return a valid cached token without taking the lock, so a # caller with a usable token never blocks behind another thread's @@ -538,8 +555,24 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: 'client_id': self.config.client_id, }) - if status == 200 and body.get('access_token'): - return self._tokenset_from_response(body) + if status == 200: + # A 200 is the RFC 6749 §5.1 token response: the grant + # completed. Accept it only if it actually carries the kind + # _select will hand to QuestDB (the id_token in groups mode, + # else the access_token), using the same predicate as the cache + # gate and the post-refresh check so the three can't disagree. + tokens = self._tokenset_from_response(body) + if self._has_required_token(tokens): + return tokens + # The grant completed but the required kind is absent: a stable + # misconfiguration, not a transient poll state. Raise a clear + # terminal error here instead of caching an unusable token and + # silently re-running the whole interactive flow on every later + # token() call. + self._renderer.on_failure( + 'Sign-in failed: the identity provider did not return the ' + 'token this server requires.') + raise self._missing_required_token_error() error = body.get('error') if error == 'authorization_pending': @@ -602,12 +635,12 @@ def _normalize_url(url: str) -> str: # Full URL with scheme/host lower-cased and the default port dropped, but # the path kept (it distinguishes multi-tenant realms). Used for the cache # key so trivial spelling differences don't cause a spurious re-prompt. - parts = urllib.parse.urlparse(url) + parts, port = safe_urlparse(url) scheme = (parts.scheme or '').lower() host = (parts.hostname or '').lower() default_port = {'https': 443, 'http': 80}.get(scheme) - if parts.port and parts.port != default_port: - netloc = f'{host}:{parts.port}' + if port and port != default_port: + netloc = f'{host}:{port}' else: netloc = host query = f'?{parts.query}' if parts.query else '' diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index a1cccbe8..359391c4 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -37,12 +37,11 @@ from __future__ import annotations import ssl -import urllib.parse from dataclasses import dataclass from typing import Any, Dict, Optional from ._errors import OidcConfigError -from ._http import get_json +from ._http import get_json, safe_urlparse # QuestDB /settings keys (see EntPropServerConfiguration.exportConfiguration()). _K_ENABLED = 'acl.oidc.enabled' @@ -116,22 +115,15 @@ def fetch_settings( return settings_config(data) -def _origin(url: str) -> Optional[str]: - parts = urllib.parse.urlparse(url) - if parts.scheme and parts.netloc: - return f'{parts.scheme}://{parts.netloc}' - return None - - _DEFAULT_PORTS = {'https': 443, 'http': 80} def _normalized_origin(url: str) -> tuple: """(scheme, host, port) with default ports filled in, for comparison.""" - parts = urllib.parse.urlparse(url) + parts, explicit_port = safe_urlparse(url) scheme = (parts.scheme or '').lower() host = (parts.hostname or '').lower() - port = parts.port or _DEFAULT_PORTS.get(scheme) + port = explicit_port or _DEFAULT_PORTS.get(scheme) return (scheme, host, port) @@ -213,7 +205,6 @@ def discover_device_endpoint_from_idp( *, issuer: Optional[str], discovery_url: Optional[str], - token_endpoint: Optional[str], ctx: Optional[ssl.SSLContext] = None, insecure: bool = False, timeout: float = 30) -> Dict[str, Any]: @@ -221,20 +212,18 @@ def discover_device_endpoint_from_idp( Fetch the IdP ``.well-known/openid-configuration`` and return it. The discovery URL is taken from ``discovery_url``, else built from - ``issuer``, else (best effort) from the origin of ``token_endpoint``. + ``issuer``. One of the two is required: the discovery origin is **never** + derived from a QuestDB-advertised endpoint, because that would let a + tampered ``/settings`` choose where the device code and refresh token are + sent (the resolved issuer and endpoints would then all share the attacker's + origin and pass the co-location / issuer-pin checks trivially). """ - url = discovery_url - if not url and issuer: - url = well_known_url(issuer) - if not url and token_endpoint: - origin = _origin(token_endpoint) - if origin: - url = well_known_url(origin) + url = discovery_url or (well_known_url(issuer) if issuer else None) if not url: raise OidcConfigError( - 'Cannot discover the IdP device-authorization endpoint: no ' - 'issuer / discovery_url given and none could be derived. Pass ' - 'issuer=... or device_authorization_endpoint=... explicitly.') + 'Cannot discover the IdP device-authorization endpoint: no issuer ' + 'or discovery_url was given. Pass issuer=... (or ' + 'device_authorization_endpoint=... to skip discovery).') return get_json(url, ctx=ctx, insecure=insecure, timeout=timeout) @@ -296,10 +285,30 @@ def resolve_config( # endpoint (and/or the token endpoint). This contacts the IdP, so it is # held to https/loopback (insecure=False) regardless of the QuestDB flag. if not device_authorization_endpoint or not token_endpoint: + # Require a caller-supplied trust anchor before contacting the IdP for + # discovery. Without issuer= / discovery_url=, the discovery target + # would have to be guessed from the token endpoint that /settings + # supplied; a tampered or MITM'd /settings (reachable in cleartext when + # QuestDB is http:// with insecure=True) could then steer discovery — + # and so the device-code and refresh-token POSTs — to an attacker + # origin, with the co-location and issuer-pin checks passing trivially + # because every value shares that one origin. issuer= is out-of-band, + # so the server cannot forge it. + if not issuer and not discovery_url: + raise OidcConfigError( + 'QuestDB did not advertise the OIDC device-authorization ' + 'endpoint (and/or the token endpoint), so it must be ' + 'discovered from the identity provider, but the IdP is not ' + 'pinned. Pass issuer="https://your-idp" (its origin) so a ' + 'tampered or intercepted /settings response cannot redirect ' + 'the device-code and refresh-token requests to an attacker. ' + 'Alternatively pass the endpoint(s) explicitly ' + '(device_authorization_endpoint=..., token_endpoint=...) to ' + 'skip discovery, or discovery_url=... to pin the discovery ' + 'document.') doc = discover_device_endpoint_from_idp( issuer=issuer, discovery_url=discovery_url, - token_endpoint=token_endpoint, ctx=ctx, insecure=False, - timeout=timeout) + ctx=ctx, insecure=False, timeout=timeout) device_authorization_endpoint = ( device_authorization_endpoint or doc.get('device_authorization_endpoint')) diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index fc5b158b..a845b749 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -91,6 +91,23 @@ def ok(self) -> bool: return 200 <= self.status < 300 +def safe_urlparse(url: str) -> tuple: + """ + ``urllib.parse.urlparse(url)`` paired with its port, but with a typed error. + + ``ParseResult.port`` raises a bare ``ValueError`` for a non-integer port + (e.g. ``https://idp:notaport``); re-raise it as :class:`OidcConfigError` so + a malformed endpoint URL stays within the package's error contract instead + of escaping as a raw ``ValueError``. Returns ``(parts, port)``. + """ + parts = urllib.parse.urlparse(url) + try: + return parts, parts.port + except ValueError as e: + raise OidcConfigError( + f'Malformed endpoint URL {url!r}: invalid port.') from e + + def _is_loopback(host: Optional[str]) -> bool: # Traffic to a loopback address never leaves the host, so plaintext http # carries no network interception risk and is always permitted. diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py index 2b95d75b..ac9aa45b 100644 --- a/src/questdb/auth/_questdb.py +++ b/src/questdb/auth/_questdb.py @@ -30,7 +30,7 @@ from typing import Any, Dict, Optional from ._device import OidcDeviceAuth -from ._errors import OidcAuthError, OidcError +from ._errors import OidcAuthError, OidcConfigError, OidcError from ._http import request _DEFAULT_PG_PORT = 8812 @@ -159,12 +159,51 @@ def sql(self, query: str, *, limit: Optional[str] = None, pass raise OidcError( f'QuestDB query failed (HTTP {resp.status}): {detail}') - return _exec_json_to_df(resp.json(), pandas) + try: + data = resp.json() + except (ValueError, UnicodeDecodeError): + # A 2xx body that isn't JSON (e.g. an HTML error/login page from a + # reverse proxy or captive portal) must surface as a clean + # OidcError, not a raw JSONDecodeError. Mirrors the error path and + # post_form(). + raise OidcError( + 'QuestDB returned a non-JSON success response from /exec: ' + f'{resp.text()[:300]}') + if not isinstance(data, dict): + # Valid JSON but not an object (e.g. a bare list) would make + # _exec_json_to_df fail with AttributeError on .get(); reject it. + raise OidcError( + 'QuestDB /exec returned JSON that is not an object ' + f'(got {type(data).__name__}); cannot build a DataFrame.') + return _exec_json_to_df(data, pandas) # -- connection adapters ------------------------------------------------ - def _host(self) -> Optional[str]: - return self._parts.hostname + def _require_host(self, host: Optional[str] = None) -> str: + """ + Resolve the PG-wire / ILP host: an explicit ``host`` override, else the + host from the QuestDB URL. Raises when neither yields one (e.g. a URL + with no authority such as ``"localhost"`` or ``"questdb:9000"``) instead + of passing a bare ``None`` down to the driver. + + The returned host is *unbracketed* — psycopg and SQLAlchemy take the + address and port as separate arguments. :meth:`_ilp_addr` adds the + brackets an IPv6 literal needs in the ILP ``addr=host:port`` form. + """ + resolved = host or self._parts.hostname + if not resolved: + raise OidcConfigError( + f'The QuestDB URL {self.url!r} has no host. Use a URL with an ' + 'explicit host (e.g. "https://questdb.example.com:9000"), or ' + 'pass host=... to the adapter.') + return resolved + + @staticmethod + def _ilp_addr(host: str, port: int) -> str: + # Bracket an IPv6 literal so the ILP conf parser reads host:port + # unambiguously; hostnames and IPv4 addresses never contain ':'. + bracketed = f'[{host}]' if ':' in host else host + return f'{bracketed}:{port}' def sqlalchemy_engine( self, @@ -200,7 +239,7 @@ def sqlalchemy_engine( url = URL.create( drivername=drivername, username='_sso', - host=host or self._host(), + host=self._require_host(host), port=pg_port, database=database) engine = create_engine(url, **engine_kwargs) @@ -229,7 +268,7 @@ def psycopg( """ mod = _pg_module() return mod.connect( - host=host or self._host(), + host=self._require_host(host), port=pg_port, dbname=database, user='_sso', @@ -256,7 +295,8 @@ def sender(self, *, port: Optional[int] = None, scheme = 'https' if self._parts.scheme == 'https' else 'http' resolved_port = port or self._parts.port or ( 443 if scheme == 'https' else 9000) - conf = f'{scheme}::addr={self._host()}:{resolved_port};' + conf = (f'{scheme}::addr=' + f'{self._ilp_addr(self._require_host(), resolved_port)};') return Sender.from_conf(conf, token=self.auth.token(), **sender_kwargs) @@ -288,8 +328,7 @@ def connect( :param flow: ``"auto"`` (default), ``"device"`` or ``"loopback"``. Today ``"auto"`` always resolves to the device flow (works on local and remote kernels); ``"loopback"`` is reserved for a future release. - :param cache: Token cache backend: ``"memory"`` (default), ``"file"`` or - ``None``. + :param cache: Token cache backend: ``"memory"`` (default) or ``None``. :param insecure: Allow plaintext ``http://`` URLs (development only). :param eager: If ``True`` (default), sign in immediately; otherwise defer until the first call that needs a token. diff --git a/test/test.py b/test/test.py index 6fdd418f..d6c7808a 100755 --- a/test/test.py +++ b/test/test.py @@ -39,7 +39,6 @@ TestDeviceFlow, TestNonInteractive, TestRefresh, - TestFileCache, TestDiscovery, TestRestAdapter, TestAdapters, diff --git a/test/test_auth.py b/test/test_auth.py index 427ccb57..6a4ed951 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -40,7 +40,6 @@ import json import os import sys -import tempfile import threading import types import unittest @@ -64,7 +63,7 @@ OidcNetworkError, TokenSet, ) -from questdb.auth._cache import FileCache, MemoryCache, _MEMORY_STORE # noqa: E402 +from questdb.auth._cache import MemoryCache, _MEMORY_STORE # noqa: E402 from questdb.auth._render import Renderer # noqa: E402 try: @@ -72,12 +71,6 @@ except ImportError: pd = None -try: - import fcntl as _fcntl # noqa: F401 - _HAS_FCNTL = True -except ImportError: - _HAS_FCNTL = False - _HAS_PG_DRIVER = ( importlib.util.find_spec('psycopg') is not None or importlib.util.find_spec('psycopg2') is not None) @@ -148,6 +141,7 @@ def __init__(self): self.expected_bearer = None # for /exec auth check self.exec_response = None self.exec_status = 200 + self.exec_raw = None # (status, content_type, bytes) override # Recording. self.device_requests = 0 self.token_requests = [] @@ -192,6 +186,14 @@ def do_GET(self): self._send_json(401, {'error': 'unauthorized'}) return self.state.exec_requests.append(self.path) + if self.state.exec_raw is not None: + status, ctype, raw = self.state.exec_raw + self.send_response(status) + self.send_header('Content-Type', ctype) + self.send_header('Content-Length', str(len(raw))) + self.end_headers() + self.wfile.write(raw) + return self._send_json(self.state.exec_status, self.state.exec_response or { 'columns': [ {'name': 'ts', 'type': 'TIMESTAMP'}, @@ -366,13 +368,28 @@ def test_token_caches_in_memory_across_instances(self): self.make_auth().token() self.assertEqual(self.state.device_requests, 1) - def test_missing_id_token_raises_config_error(self): + def test_groups_mode_missing_id_token_fails_without_caching(self): + # groups_in_token=True but the completed grant carries only an + # access_token: the poll must reject it as a terminal flow error and + # NOT cache it (otherwise every later token() re-runs the whole + # interactive flow). See M1. self.state.token_script = [(200, { 'access_token': ACCESS_TOKEN, 'token_type': 'Bearer', 'expires_in': 3600})] # no id_token auth = self.make_auth(groups_in_token=True) - with self.assertRaises(OidcConfigError): + with self.assertRaises(OidcDeviceFlowError): auth.token() + self.assertIsNone(auth._tokens) # nothing was cached + + def test_groups_mode_accepts_id_token_without_access_token(self): + # A completed grant that returns only an id_token (no access_token) is + # usable in groups mode and must be returned, not discarded as it was + # when success gated on access_token. See M1. + self.state.token_script = [(200, { + 'id_token': ID_TOKEN, 'token_type': 'Bearer', + 'expires_in': 3600})] # no access_token + auth = self.make_auth(groups_in_token=True) + self.assertEqual(auth.token(), ID_TOKEN) def test_200_without_access_token_is_not_success(self): # A 200 with no access_token must not be treated as a token. @@ -551,68 +568,6 @@ def test_refresh_network_error_propagates_without_reprompt(self): self.assertEqual(auth._tokens.refresh_token, 'REFRESH-1') -class TestFileCache(AuthTestBase): - def test_file_cache_works_without_os_lock(self): - # Exercise the no-fcntl/no-msvcrt fallback: with the lock primitives - # no-op'd, the atomic temp-file replace must still keep every entry. - import questdb.auth._cache as cache_mod - tmp = tempfile.mkdtemp() - path = os.path.join(tmp, 'cache.json') - with mock.patch.object(cache_mod, '_lock_fd', lambda fd: None), \ - mock.patch.object(cache_mod, '_unlock_fd', lambda fd: None): - cache = FileCache(path) - cache.store('k1', TokenSet(access_token='a1', expires_at=1.0)) - cache.store('k2', TokenSet(access_token='a2', expires_at=1.0)) - self.assertEqual(cache.load('k1').access_token, 'a1') - self.assertEqual(cache.load('k2').access_token, 'a2') - - def test_file_cache_survives_new_instance(self): - tmp = tempfile.mkdtemp() - path = os.path.join(tmp, 'cache.json') - cache1 = FileCache(path) - self.make_auth(cache=cache1).token() - self.assertEqual(self.state.device_requests, 1) - # New process simulation: fresh memory, load from file. - _MEMORY_STORE.clear() - cache2 = FileCache(path) - token = self.make_auth(cache=cache2).token() - self.assertEqual(token, ID_TOKEN) - self.assertEqual(self.state.device_requests, 1) # no re-prompt - # File is mode 600 where supported. - if os.name == 'posix': - self.assertEqual(os.stat(path).st_mode & 0o777, 0o600) - - @unittest.skipUnless( - _HAS_FCNTL, 'cross-process file lock requires fcntl (POSIX)') - def test_concurrent_writes_preserve_all_entries(self): - # 20 writers (distinct instances, same file, distinct keys) racing: - # the sidecar lock + atomic unique-temp replace must keep every entry - # and never corrupt the file or leave a temp behind. - tmp = tempfile.mkdtemp() - path = os.path.join(tmp, 'cache.json') - - def writer(i): - FileCache(path).store( - f'key-{i}', - TokenSet(access_token=f'a{i}', id_token=f'id{i}', - refresh_token=f'r{i}', expires_at=1.0)) - - threads = [threading.Thread(target=writer, args=(i,)) - for i in range(20)] - for t in threads: - t.start() - for t in threads: - t.join(10) - - final = FileCache(path) - for i in range(20): - ts = final.load(f'key-{i}') - self.assertIsNotNone(ts, f'lost entry key-{i}') - self.assertEqual(ts.access_token, f'a{i}') - leftovers = [n for n in os.listdir(tmp) if n.endswith('.tmp')] - self.assertEqual(leftovers, [], f'temp files left behind: {leftovers}') - - class TestDiscovery(AuthTestBase): def test_from_questdb_reads_settings(self): self.state.settings = {'config': { @@ -633,7 +588,8 @@ def test_from_questdb_reads_settings(self): self.assertEqual(auth.token(), ID_TOKEN) def test_well_known_fallback_for_device_endpoint(self): - # Settings advertise OIDC + token endpoint but NOT the device endpoint. + # Settings advertise OIDC + token endpoint but NOT the device endpoint; + # issuer= is pinned, so the IdP .well-known fallback is allowed. self.state.settings = {'config': { 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', @@ -646,8 +602,47 @@ def test_well_known_fallback_for_device_endpoint(self): 'token_endpoint': self.base + '/token', 'device_authorization_endpoint': self.base + '/device', } - auth = OidcDeviceAuth.from_questdb(self.base, insecure=True, - renderer=Renderer()) + auth = OidcDeviceAuth.from_questdb(self.base, issuer=self.base, + insecure=True, renderer=Renderer()) + self.assertEqual(auth.config.device_authorization_endpoint, + self.base + '/device') + + def test_device_fallback_without_issuer_is_rejected(self): + # M4: QuestDB advertises the token endpoint but not the device + # endpoint, and no issuer is pinned. Discovery would otherwise be + # steered by the (possibly tampered) /settings response, so refuse and + # demand an out-of-band issuer pin — even though a usable .well-known + # is reachable here, it must NOT be fetched. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': self.base + '/token', + }} + self.state.well_known = { + 'issuer': self.base, + 'token_endpoint': self.base + '/token', + 'device_authorization_endpoint': self.base + '/device', + } + with self.assertRaises(OidcConfigError) as cm: + OidcDeviceAuth.from_questdb(self.base, insecure=True) + self.assertIn('issuer', str(cm.exception)) + + def test_device_fallback_with_discovery_url_is_accepted(self): + # discovery_url= is an out-of-band pin too, accepted in lieu of issuer=. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': self.base + '/token', + }} + self.state.well_known = { + 'issuer': self.base, + 'token_endpoint': self.base + '/token', + 'device_authorization_endpoint': self.base + '/device', + } + auth = OidcDeviceAuth.from_questdb( + self.base, + discovery_url=self.base + '/.well-known/openid-configuration', + insecure=True, renderer=Renderer()) self.assertEqual(auth.config.device_authorization_endpoint, self.base + '/device') @@ -657,6 +652,9 @@ def test_oidc_disabled_raises(self): OidcDeviceAuth.from_questdb(self.base, insecure=True) def test_missing_device_endpoint_raises(self): + # issuer= is pinned (so the fallback is allowed), but the IdP's + # discovery doc carries no device_authorization_endpoint: that is the + # error under test, not the missing-issuer guard above. self.state.settings = {'config': { 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', @@ -664,6 +662,21 @@ def test_missing_device_endpoint_raises(self): }} self.state.well_known = {'issuer': self.base, 'token_endpoint': self.base + '/token'} + with self.assertRaises(OidcConfigError): + OidcDeviceAuth.from_questdb(self.base, issuer=self.base, + insecure=True) + + def test_malformed_endpoint_port_raises_config_error(self): + # /settings advertising a non-integer port in an endpoint must raise + # OidcConfigError (the typed contract), not a bare ValueError that + # callers catching OidcError would miss. See M6. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': 'https://idp:notaport/token', + 'acl.oidc.device.authorization.endpoint': + 'https://idp:notaport/device', + }} with self.assertRaises(OidcConfigError): OidcDeviceAuth.from_questdb(self.base, insecure=True) @@ -780,6 +793,24 @@ def test_sql_malformed_shape_raises_oidc_error(self): with self.assertRaises(OidcError): qdb.sql('SELECT a, b FROM t') + def test_sql_non_json_2xx_raises_oidc_error(self): + # A 2xx body that isn't JSON (e.g. an HTML page from a reverse proxy) + # must raise a clean OidcError, not a raw JSONDecodeError. See M3. + qdb = self._connected() + self.state.exec_raw = (200, 'text/html', b'proxy') + with self.assertRaises(OidcError) as cm: + qdb.sql('SELECT 1') + self.assertNotIsInstance(cm.exception, OidcAuthError) + + def test_sql_non_dict_json_raises_oidc_error(self): + # A valid-JSON-but-not-an-object 2xx body (e.g. a bare list) must raise + # OidcError, not AttributeError from .get(). See M3. + qdb = self._connected() + self.state.exec_response = ['not', 'an', 'object'] + with self.assertRaises(OidcError) as cm: + qdb.sql('SELECT 1') + self.assertNotIsInstance(cm.exception, OidcAuthError) + class TestConcurrency(AuthTestBase): def test_valid_cached_token_does_not_block_during_signin(self): @@ -962,6 +993,61 @@ def create(**kw): self.assertEqual(cparams['password'], 'TKN') self.assertEqual(auth.calls - before, 2) + def test_sender_brackets_ipv6_addr(self): + # An IPv6 literal must be bracketed in the ILP addr=host:port conf, + # else "::1:9000" is ambiguous to the conf parser. See M5. + qdb = self._qdb('https://[::1]:9000') + captured = {} + fake = types.ModuleType('questdb.ingress') + + class Sender: + @staticmethod + def from_conf(conf, *, token=None, **kw): + captured['conf'] = conf + return 'S' + + fake.Sender = Sender + with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): + qdb.sender() + self.assertEqual(captured['conf'], 'https::addr=[::1]:9000;') + + def test_psycopg_uses_bare_ipv6_host(self): + # psycopg takes host and port separately, so the IPv6 host is passed + # WITHOUT brackets (unlike the ILP addr= form). See M5. + qdb = self._qdb('http://[::1]:9000') + captured = {} + fake = types.ModuleType('psycopg') + + def connect(**kw): + captured.update(kw) + return 'CONN' + + fake.connect = connect + with mock.patch.dict(sys.modules, {'psycopg': fake}): + qdb.psycopg() + self.assertEqual(captured['host'], '::1') + + def test_require_host_rejects_hostless_url(self): + # A URL with no extractable host must raise, not pass None to a driver; + # an explicit host= override still resolves. See M5. + for bad in ('localhost', 'questdb:9000'): + with self.subTest(url=bad): + with self.assertRaises(OidcConfigError): + QuestDB(bad, _FakeAuth(), insecure=True)._require_host() + self.assertEqual( + QuestDB('localhost', _FakeAuth())._require_host('h.example'), + 'h.example') + + def test_sender_hostless_url_raises(self): + # The guard propagates through an adapter (not just the helper): + # sender() on a host-less URL raises OidcConfigError. See M5. + qdb = self._qdb('questdb:9000') + fake = types.ModuleType('questdb.ingress') + fake.Sender = object() # import must succeed so we reach the guard + with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): + with self.assertRaises(OidcConfigError): + qdb.sender() + def test_sql_missing_pandas_raises(self): qdb = self._qdb() with mock.patch.dict(sys.modules, {'pandas': None}): @@ -1034,6 +1120,13 @@ def test_both_endpoints_off_issuer_rejected(self): self._validate('https://idp/token', 'https://idp/device', issuer='https://other-issuer.example') + def test_malformed_port_raises_config_error(self): + # A non-integer port must surface as OidcConfigError, not urllib's bare + # ValueError (which callers catching OidcError would miss). See M6. + with self.assertRaises(OidcConfigError): + self._validate('https://idp:notaport/token', + 'https://idp:notaport/device') + def test_explicit_constructor_enforces_co_location(self): with self.assertRaises(OidcConfigError): OidcDeviceAuth( @@ -1054,6 +1147,13 @@ def _auth(self, **kw): opts.update(kw) return OidcDeviceAuth(**opts) + def test_normalize_url_malformed_port_raises_config_error(self): + # cache_key normalization shares the same typed-port guard: a malformed + # port raises OidcConfigError, not a bare ValueError. See M6. + from questdb.auth._device import _normalize_url + with self.assertRaises(OidcConfigError): + _normalize_url('https://idp:notaport/token') + def test_realm_path_distinguishes_key(self): # Multi-tenant IdP: same host, different realm path -> distinct keys # (the old origin-only key collided, leaking one realm's token). From 5ef892bb45ef515aff7de52a9f25f1d4015713f8 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 01:00:03 +0100 Subject: [PATCH 007/104] ci: build questdb master's -SNAPSHOT java client via local-client profile The linux-qdb-master job's "Compile QuestDB master" step failed because questdb master depends on a -SNAPSHOT java-questdb-client that is not published to Maven Central. Add a detect-local-client step template (adapted from questdb/questdb's ci/templates/detect-local-client.yml) that reads questdb.client.version from the cloned core/pom.xml: for a -SNAPSHOT client it inits the java-questdb-client submodule and builds it via the `local-client` Maven profile; for a released client it resolves from Maven Central. The "Compile QuestDB master" step appends the resulting $(CLIENT_PROFILE). Co-Authored-By: Claude Opus 4.8 (1M context) --- ci/run_tests_pipeline.yaml | 9 ++++++- ci/templates/detect-local-client.yml | 36 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 ci/templates/detect-local-client.yml diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 80099fb9..4c444658 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -63,12 +63,19 @@ stages: git clone --depth 1 https://github.com/questdb/questdb.git displayName: git clone questdb master condition: eq(variables.vsQuestDbMaster, true) + # Decide whether to build java-questdb-client from the bundled + # submodule (-P local-client, for a -SNAPSHOT client not on Maven + # Central) or resolve it from Maven Central. Sets $(CLIENT_PROFILE). + - template: templates/detect-local-client.yml + parameters: + qdbRepoPath: questdb + condition: eq(variables.vsQuestDbMaster, true) - task: Maven@3 displayName: "Compile QuestDB master" inputs: mavenPOMFile: "questdb/pom.xml" jdkVersionOption: "1.17" - options: "-DskipTests -Pbuild-web-console" + options: "-DskipTests -Pbuild-web-console $(CLIENT_PROFILE)" condition: eq(variables.vsQuestDbMaster, true) - script: python3 proj.py test 1 displayName: "Test vs released" diff --git a/ci/templates/detect-local-client.yml b/ci/templates/detect-local-client.yml new file mode 100644 index 00000000..55c9a5fb --- /dev/null +++ b/ci/templates/detect-local-client.yml @@ -0,0 +1,36 @@ +# Adapted from questdb/questdb's ci/templates/detect-local-client.yml. +# +# Decide how a cloned QuestDB checkout resolves its java-questdb-client +# dependency: a -SNAPSHOT client version is not published to Maven Central, so +# build it from the bundled java-questdb-client submodule via the `local-client` +# profile; a released version is taken from Maven Central. Sets the +# CLIENT_PROFILE pipeline variable (``-P local-client`` or empty) for the +# following Maven build, and inits the submodule only when it is needed. +# +# Unlike the upstream template, QuestDB is cloned into a subdirectory here, so +# the repo path is a parameter; ``condition`` lets the caller gate this to the +# matrix leg that builds QuestDB master. +parameters: + - name: qdbRepoPath + type: string + default: questdb + - name: condition + type: string + default: succeeded() + +steps: + - bash: | + set -eu + pom="${{ parameters.qdbRepoPath }}/core/pom.xml" + CLIENT_VERSION=$(sed -n 's/.*\(.*\)<\/questdb.client.version>.*/\1/p' "$pom" | head -1) + echo "questdb.client.version=$CLIENT_VERSION" + if echo "$CLIENT_VERSION" | grep -q '\-SNAPSHOT$'; then + echo "SNAPSHOT client detected -> build it locally (local-client profile)" + git -C "${{ parameters.qdbRepoPath }}" submodule update --init java-questdb-client + echo "##vso[task.setvariable variable=CLIENT_PROFILE]-P local-client" + else + echo "Release client detected -> resolve from Maven Central" + echo "##vso[task.setvariable variable=CLIENT_PROFILE]" + fi + displayName: "Detect QuestDB local client profile" + condition: ${{ parameters.condition }} From ae8baa77deb577550a2cae0bc4f6da2fd5088e2f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 02:47:46 +0100 Subject: [PATCH 008/104] ci: build/run questdb master on JDK 25 QuestDB master bumped its build to Java 25 (javac.target=25). Its maven-enforcer requireJavaVersion reads ${java.enforce.version}, which is only set by the JDK-activated 'java25+' profile ((24,)). Building the linux-qdb-master leg with JDK 17 left that property empty, so the enforcer failed with 'JDK version can't be empty' before compilation. Point the 'Compile QuestDB master' Maven task at $(JAVA_HOME_25_X64) via jdkVersionOption: path, and run 'Test vs master' on the same JDK 25 so the freshly compiled Java 25 bytecode can run. 'Test vs released' stays on JDK 17. Both ubuntu-latest and windows-2025 images preinstall Temurin 25 (JAVA_HOME_25_X64). Co-Authored-By: Claude Opus 4.8 (1M context) --- ci/run_tests_pipeline.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 4c444658..e8a69da0 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -74,7 +74,8 @@ stages: displayName: "Compile QuestDB master" inputs: mavenPOMFile: "questdb/pom.xml" - jdkVersionOption: "1.17" + jdkVersionOption: "path" + jdkDirectory: "$(JAVA_HOME_25_X64)" options: "-DskipTests -Pbuild-web-console $(CLIENT_PROFILE)" condition: eq(variables.vsQuestDbMaster, true) - script: python3 proj.py test 1 @@ -84,7 +85,7 @@ stages: - script: python3 proj.py test 1 displayName: "Test vs master" env: - JAVA_HOME: $(JAVA_HOME_17_X64) + JAVA_HOME: $(JAVA_HOME_25_X64) QDB_REPO_PATH: "./questdb" condition: eq(variables.vsQuestDbMaster, true) - job: TestsAgainstVariousNumpyVersion1x From a4b41c3fa4c66ab0aa3833a1192b01a53079dc0a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 03:02:02 +0100 Subject: [PATCH 009/104] ci: invoke Maven directly to build questdb master on JDK 25 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Maven@3 task crashes while parsing JDK 25 ('Cannot read properties of null (reading 'major')') — its JDK selection tops out at 21 and its Node-side version detector returns null for 25. Replace the task with a bash step that exports JAVA_HOME=$(JAVA_HOME_25_X64) and runs mvn directly, mirroring the task defaults (questdb/pom.xml, goal 'package', same -DskipTests -Pbuild-web-console $(CLIENT_PROFILE) options). Co-Authored-By: Claude Opus 4.8 (1M context) --- ci/run_tests_pipeline.yaml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index e8a69da0..c9dfff3f 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -70,13 +70,17 @@ stages: parameters: qdbRepoPath: questdb condition: eq(variables.vsQuestDbMaster, true) - - task: Maven@3 + # The Maven@3 task crashes parsing JDK 25 ("Cannot read properties of + # null (reading 'major')") since its JDK support tops out at 21, so + # invoke Maven directly on the preinstalled JDK 25 instead. Mirrors the + # task's defaults: POM questdb/pom.xml, goal "package". + - bash: | + set -eu + export JAVA_HOME="$(JAVA_HOME_25_X64)" + export PATH="$JAVA_HOME/bin:$PATH" + java -version + mvn -B -f questdb/pom.xml package -DskipTests -Pbuild-web-console $(CLIENT_PROFILE) displayName: "Compile QuestDB master" - inputs: - mavenPOMFile: "questdb/pom.xml" - jdkVersionOption: "path" - jdkDirectory: "$(JAVA_HOME_25_X64)" - options: "-DskipTests -Pbuild-web-console $(CLIENT_PROFILE)" condition: eq(variables.vsQuestDbMaster, true) - script: python3 proj.py test 1 displayName: "Test vs released" From 61abd4fc15ba6275c45af4a4fe782304621a731a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 14:21:27 +0100 Subject: [PATCH 010/104] ci: pass JDK 25 module access flags to questdb master server QuestDB master runs as the io.questdb JPMS module and now uses jdk.internal.vm.ContinuationScope, so on JDK 25 the server dies at startup with IllegalAccessError (java.base does not export jdk.internal.vm to io.questdb), plus Unsafe/native-access warnings. The test fixture launches questdb.jar directly rather than via questdb.sh, so the access flags questdb.sh normally supplies are absent. Set JDK_JAVA_OPTIONS on the 'Test vs master' step with the exact module access flags from questdb.sh (all targeting io.questdb). Scoped to that step, so the JDK 17 'Test vs released' run is unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- ci/run_tests_pipeline.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index c9dfff3f..3025ab1a 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -91,6 +91,18 @@ stages: env: JAVA_HOME: $(JAVA_HOME_25_X64) QDB_REPO_PATH: "./questdb" + # QuestDB master runs as the io.questdb JPMS module and needs these + # JDK 25 access flags (mirrors questdb.sh). The test fixture launches + # questdb.jar directly rather than via questdb.sh, so feed them to the + # java launcher through JDK_JAVA_OPTIONS. + JDK_JAVA_OPTIONS: >- + --sun-misc-unsafe-memory-access=allow + --enable-native-access=io.questdb + --add-opens=java.base/java.lang=io.questdb + --add-opens=java.base/java.lang.reflect=io.questdb + --add-opens=java.base/java.nio=io.questdb + --add-opens=java.base/java.time.zone=io.questdb + --add-exports=java.base/jdk.internal.vm=io.questdb condition: eq(variables.vsQuestDbMaster, true) - job: TestsAgainstVariousNumpyVersion1x pool: From 23fb823df08f3f42b174acf436415fb123286ff9 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 15:31:06 +0100 Subject: [PATCH 011/104] do not follow redirects --- src/questdb/auth/_http.py | 32 +++++++++++++++++++++++---- test/test_auth.py | 46 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index a845b749..ec29a1e3 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -137,12 +137,36 @@ def _require_secure(url: str, insecure: bool) -> None: 'insecure=True only to permit plaintext to a non-loopback host.') +class _NoRedirect(urllib.request.HTTPRedirectHandler): + """Refuse to follow HTTP redirects. + + The discovery / device / token / ``/settings`` / ``/exec`` endpoints never + legitimately redirect. Auto-following a ``30x`` is unsafe here because only + the *original* URL is vetted: ``_require_secure`` and + ``validate_endpoint_origins`` never see the redirect target. urllib also + does not strip the ``Authorization`` header on a cross-origin redirect, so a + single ``302`` from ``/exec`` would re-send ``Authorization: Bearer + `` to an attacker-chosen host — including a downgrade to plaintext + ``http`` — leaking the QuestDB token off-origin. + + Returning ``None`` makes urllib stop following and surface the ``30x`` as an + ``HTTPError`` (which :func:`request` turns into a non-2xx + :class:`HttpResponse`), so callers see a clean failure instead of a + silently-followed redirect. + """ + + def redirect_request(self, *args, **kwargs): + return None + + def _opener(ctx: Optional[ssl.SSLContext]) -> urllib.request.OpenerDirector: # build_opener keeps the default ProxyHandler (which reads *_PROXY env - # vars), while letting us pin our own TLS context. - if ctx is None: - return urllib.request.build_opener() - return urllib.request.build_opener(urllib.request.HTTPSHandler(context=ctx)) + # vars), while letting us pin our own TLS context and forbid redirects + # (the credential/token endpoints never legitimately redirect). + handlers: list = [_NoRedirect()] + if ctx is not None: + handlers.append(urllib.request.HTTPSHandler(context=ctx)) + return urllib.request.build_opener(*handlers) def request( diff --git a/test/test_auth.py b/test/test_auth.py index 6a4ed951..9f55b669 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1209,6 +1209,52 @@ def test_insecure_does_not_downgrade_idp(self): with self.assertRaises(OidcConfigError): auth.token() + def test_redirects_are_not_followed(self): + # A 30x must NOT be followed: urllib would otherwise re-send the + # Authorization: Bearer header (and downgrade to plaintext http) to the + # redirect target, leaking the QuestDB token off-origin (only the + # original URL is vetted, never the redirect target). The redirect must + # surface as a non-2xx response, and the off-origin host must never be + # contacted. See C1. + from questdb.auth import _http + + seen = [] + + class _Redir(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def do_GET(self): + seen.append((self.path, self.headers.get('Authorization'))) + if self.path == '/exec': + self.send_response(302) + self.send_header('Location', attacker + '/stolen') + self.end_headers() + else: + self.send_response(200) + self.send_header('Content-Length', '2') + self.end_headers() + self.wfile.write(b'{}') + + victim = http.server.HTTPServer(('127.0.0.1', 0), _Redir) + thief = http.server.HTTPServer(('127.0.0.1', 0), _Redir) + attacker = f'http://127.0.0.1:{thief.server_port}' + for srv in (victim, thief): + threading.Thread(target=srv.serve_forever, daemon=True).start() + try: + resp = _http.request( + 'GET', f'http://127.0.0.1:{victim.server_port}/exec', + headers={'Authorization': 'Bearer SECRET'}, timeout=5) + finally: + for srv in (victim, thief): + srv.shutdown() + srv.server_close() + + # The redirect surfaced as a non-2xx response, was not followed, and the + # off-origin target never saw the request (or the bearer token). + self.assertEqual(resp.status, 302) + self.assertEqual(seen, [('/exec', 'Bearer SECRET')]) + class TestRendererSecurity(unittest.TestCase): """The Jupyter prompt must never turn an IdP-supplied URL into a From c13cf698614770e9efed139b66d63aa503f76f1c Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 15:41:13 +0100 Subject: [PATCH 012/104] fix: require IdP pin for plaintext /settings When QuestDB is reached over plaintext http to a non-loopback host (only possible with insecure=True), its /settings response is MITM-able. The issuer-pin requirement previously fired only when an IdP endpoint was missing (the discovery path). A tampered /settings advertising BOTH the token and device-authorization endpoints at one attacker origin skipped that path: the co-location check passed trivially (same origin) and the issuer-pin check was vacuous (no issuer), so the device code and the long-lived refresh token were POSTed to the attacker. Require the same out-of-band pin (issuer= / discovery_url=) before trusting /settings-supplied credential endpoints fetched over such an untrusted channel. Endpoints the caller passed explicitly, and endpoints from an authenticated (https / loopback) /settings, are unaffected, so the https happy path and local-dev loopback are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 49 ++++++++++++++++++++++- test/test.py | 1 + test/test_auth.py | 71 ++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 359391c4..ccd39fce 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -41,7 +41,7 @@ from typing import Any, Dict, Optional from ._errors import OidcConfigError -from ._http import get_json, safe_urlparse +from ._http import get_json, safe_urlparse, _is_loopback # QuestDB /settings keys (see EntPropServerConfiguration.exportConfiguration()). _K_ENABLED = 'acl.oidc.enabled' @@ -132,6 +132,19 @@ def _origin_str(url: str) -> str: return f'{scheme}://{host}:{port}' if port else f'{scheme}://{host}' +def _settings_channel_is_plaintext(questdb_url: str) -> bool: + """ + True if QuestDB ``/settings`` was fetched over plaintext http to a + non-loopback host — a channel a network MITM can tamper (only reachable + with ``insecure=True``; ``_require_secure`` rejects it otherwise). IdP + endpoints advertised by such an unauthenticated ``/settings`` response must + not be trusted to route credentials without an out-of-band pin. + """ + parts, _ = safe_urlparse(questdb_url) + return (parts.scheme or '').lower() == 'http' and not _is_loopback( + parts.hostname) + + def validate_endpoint_origins( token_endpoint: str, device_authorization_endpoint: str, @@ -272,6 +285,12 @@ def resolve_config( if audience is None: audience = cfg.get(_K_AUDIENCE) or None + # Track which credential endpoints the caller supplied directly. Those are + # trusted; endpoints learned from /settings are only as trustworthy as the + # channel that delivered them (see the insecure-channel guard below). + explicit_token_endpoint = token_endpoint is not None + explicit_device_endpoint = device_authorization_endpoint is not None + token_endpoint = ( token_endpoint or _resolve_endpoint(cfg.get(_K_TOKEN_ENDPOINT), cfg)) authorization_endpoint = ( @@ -281,6 +300,34 @@ def resolve_config( device_authorization_endpoint or _resolve_endpoint(cfg.get(_K_DEVICE_ENDPOINT), cfg)) + # When QuestDB itself was reached over plaintext http to a non-loopback host + # (only possible with insecure=True), its /settings response can be tampered + # in transit. Any IdP credential endpoint it advertises would then route the + # device code and long-lived refresh token to an attacker origin. The + # missing-endpoint discovery path below already demands an out-of-band pin, + # but when a tampered /settings advertises BOTH endpoints at one attacker + # origin that path is skipped, the co-location check passes trivially (they + # share that origin) and the issuer-pin check is vacuous (no issuer) — so + # nothing else catches it. Require the same out-of-band pin (issuer= / + # discovery_url=) before trusting /settings-supplied endpoints over such a + # channel. Endpoints the caller passed explicitly, and endpoints from an + # authenticated (https / loopback) /settings, are unaffected. + settings_supplied_credentials = ( + (token_endpoint and not explicit_token_endpoint) + or (device_authorization_endpoint and not explicit_device_endpoint)) + if (questdb_url and settings_supplied_credentials + and not issuer and not discovery_url + and _settings_channel_is_plaintext(questdb_url)): + raise OidcConfigError( + 'QuestDB was reached over plaintext http (insecure=True), so its ' + '/settings response — and the OIDC endpoints it advertises — can be ' + 'tampered in transit and used to redirect the device-code and ' + 'refresh-token requests to an attacker. Pin the identity provider ' + 'out-of-band with issuer="https://your-idp" (or discovery_url=...), ' + 'pass the endpoints explicitly (token_endpoint=..., ' + 'device_authorization_endpoint=...), or connect to QuestDB over ' + 'https so /settings is authenticated.') + # Fall back to IdP discovery when QuestDB doesn't advertise the device # endpoint (and/or the token endpoint). This contacts the IdP, so it is # held to https/loopback (insecure=False) regardless of the QuestDB flag. diff --git a/test/test.py b/test/test.py index d6c7808a..04a2dcd3 100755 --- a/test/test.py +++ b/test/test.py @@ -40,6 +40,7 @@ TestNonInteractive, TestRefresh, TestDiscovery, + TestInsecureSettingsGuard, TestRestAdapter, TestAdapters, TestConcurrency, diff --git a/test/test_auth.py b/test/test_auth.py index 9f55b669..d2fb3d99 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -728,6 +728,77 @@ def test_issuer_pin_accepts_matching_origin(self): self.base + '/device') +class TestInsecureSettingsGuard(unittest.TestCase): + """ + M1: a /settings response fetched over plaintext http to a non-loopback host + (only reachable with insecure=True) is MITM-able, so IdP endpoints it + advertises must not be trusted to route the device code / refresh token + without an out-of-band issuer/discovery_url pin — even when BOTH endpoints + are present (so the co-location check would otherwise pass trivially). + """ + + _TAMPERED = { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': 'https://evil.example.com/token', + 'acl.oidc.device.authorization.endpoint': + 'https://evil.example.com/device', + } + + def _resolve(self, settings, **kw): + # Stub the network: /settings returns the given (possibly tampered) map, + # and IdP discovery must never be contacted in these guard paths. + from questdb.auth import _discovery + with mock.patch.object(_discovery, 'fetch_settings', + return_value=settings), \ + mock.patch.object( + _discovery, 'discover_device_endpoint_from_idp', + side_effect=AssertionError('IdP discovery must not run')): + return _discovery.resolve_config(**kw) + + def test_both_endpoints_over_plaintext_without_pin_rejected(self): + # The M1 case: both endpoints present at one (attacker) origin, plaintext + # channel, no pin -> refuse, and never contact the IdP. + with self.assertRaises(OidcConfigError) as cm: + self._resolve(self._TAMPERED, + questdb_url='http://qdb.internal.example:9000', + insecure=True) + self.assertIn('issuer', str(cm.exception)) + + def test_plaintext_guard_does_not_fire_for_loopback(self): + # Loopback http never leaves the host, so /settings is not MITM-able; + # the guard must not fire (the common local-dev path). + cfg = self._resolve(self._TAMPERED, + questdb_url='http://127.0.0.1:9000', insecure=True) + self.assertEqual(cfg.token_endpoint, 'https://evil.example.com/token') + + def test_plaintext_guard_does_not_fire_over_https(self): + # Over https /settings is authenticated by TLS; the documented + # trust-the-server behavior is preserved (issuer= stays optional). + cfg = self._resolve(self._TAMPERED, + questdb_url='https://qdb.example.com:9000') + self.assertEqual(cfg.device_authorization_endpoint, + 'https://evil.example.com/device') + + def test_explicit_endpoints_over_plaintext_are_trusted(self): + # Endpoints the caller passed explicitly are not /settings-supplied, so + # the guard must not force a pin even over a plaintext channel. + cfg = self._resolve( + {'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb'}, + questdb_url='http://qdb.internal.example:9000', insecure=True, + token_endpoint='https://idp.example.com/token', + device_authorization_endpoint='https://idp.example.com/device') + self.assertEqual(cfg.token_endpoint, 'https://idp.example.com/token') + + def test_pin_satisfies_guard_over_plaintext(self): + # With an out-of-band issuer pin the guard is satisfied (the actual + # origin validation then happens in OidcDeviceAuth.__init__). + cfg = self._resolve(self._TAMPERED, + questdb_url='http://qdb.internal.example:9000', + insecure=True, issuer='https://evil.example.com') + self.assertEqual(cfg.token_endpoint, 'https://evil.example.com/token') + + @unittest.skipIf(pd is None, 'pandas not installed') class TestRestAdapter(AuthTestBase): def _connected(self): From bb9147c98faec39fb98b09cebc37aac450c513cc Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 16:14:02 +0100 Subject: [PATCH 013/104] fix: clamp device-flow poll timing fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The device-authorization response's expires_in / interval were trusted verbatim, so a hostile or buggy IdP could break or stall the poll loop — which runs under the acquisition lock, so a stall freezes every other thread needing a token on that instance: * expires_in <= 0 set the deadline to "now", timing the flow out before its first poll even though the user could still authorize; * an unbounded interval (or repeated slow_down) produced a single enormous sleep() holding the lock. Clamp both: expires_in <= 0 -> default, capped at a max lifetime; interval to [1s, 60s] (including after slow_down); and never sleep past the deadline. RFC-typical values (interval=5, expires_in=600) are unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 32 +++++++++++++++++++++---- test/test_auth.py | 48 +++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 27711814..f2797e86 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -61,6 +61,15 @@ # A non-positive expires_in is non-conformant; treat it as "unknown". _DEFAULT_EXPIRES_IN = 3600 +# Bounds for the device-authorization response's timing fields (RFC 8628). The +# device code is short-lived, so the IdP-supplied values are clamped: a hostile +# or buggy response must not be able to time the flow out before its first poll, +# nor pin the polling thread — which holds the acquisition lock — in one +# enormous sleep, nor keep the loop (and the lock) alive indefinitely. +_DEFAULT_DEVICE_CODE_LIFETIME = 600 # expires_in fallback (absent/invalid/<=0) +_MAX_DEVICE_CODE_LIFETIME = 1800 # cap on how long we keep polling +_MAX_POLL_INTERVAL = 60 # cap on the poll interval (incl. slow_down) + class _SystemClock: """Real time source; the default for :class:`OidcDeviceAuth`.""" @@ -526,13 +535,24 @@ def _request_device_code(self) -> Dict[str, Any]: def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: device_code = resp['device_code'] try: - interval = max(1, int(resp.get('interval', self._default_interval))) + interval = int(resp.get('interval', self._default_interval)) except (TypeError, ValueError): interval = self._default_interval + # At least 1s (RFC 8628 floor), and capped so a hostile/huge value can't + # pin the polling thread (which holds the acquisition lock) in one + # enormous sleep. + interval = min(_MAX_POLL_INTERVAL, max(1, interval)) try: - expires_in = int(resp.get('expires_in', 600)) + expires_in = int(resp.get('expires_in', _DEFAULT_DEVICE_CODE_LIFETIME)) except (TypeError, ValueError): - expires_in = 600 + expires_in = _DEFAULT_DEVICE_CODE_LIFETIME + # A non-positive lifetime would time the flow out before the first poll + # (the user has already been shown the code); treat it as unknown. Cap + # the upper end so a hostile expires_in can't keep the loop — and the + # lock — alive indefinitely. + if expires_in <= 0: + expires_in = _DEFAULT_DEVICE_CODE_LIFETIME + expires_in = min(expires_in, _MAX_DEVICE_CODE_LIFETIME) deadline = self._monotonic() + expires_in while True: @@ -545,7 +565,9 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: 'Run the sign-in again.', error='expired_token') self._renderer.on_waiting(remaining) - self._sleep(interval) + # Never sleep past the deadline (remaining > 0 here): a clamped + # interval still shouldn't overshoot a short-lived code. + self._sleep(min(interval, remaining)) status, body = self._idp_post( self.config.token_endpoint, @@ -578,7 +600,7 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: if error == 'authorization_pending': continue if error == 'slow_down': - interval += 5 + interval = min(_MAX_POLL_INTERVAL, interval + 5) continue if error == 'expired_token': self._renderer.on_failure( diff --git a/test/test_auth.py b/test/test_auth.py index d2fb3d99..ea0e4658 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -342,6 +342,54 @@ def test_timeout_when_never_authorized(self): with self.assertRaises(OidcTimeoutError): auth.token() + def test_nonpositive_expires_in_still_polls(self): + # A non-positive expires_in in the device-auth response must be treated + # as unknown, not as "already expired" — otherwise the flow times out + # before its first poll even though the user can still authorize. M2. + self.state.device_response = { + 'device_code': 'DEV-CODE', 'user_code': 'X', + 'verification_uri': 'https://idp/device', + 'expires_in': 0, 'interval': 5, + } + self.state.token_script = [(200, None)] # success on the first poll + auth = self.make_auth() + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(len(self.state.token_requests), 1) # it actually polled + + def test_oversized_interval_is_clamped(self): + # A hostile/huge interval must not pin the polling thread (which holds + # the acquisition lock) in one enormous sleep; the per-poll sleep is + # capped at _MAX_POLL_INTERVAL. M2. + from questdb.auth._device import _MAX_POLL_INTERVAL + self.state.device_response = { + 'device_code': 'DEV-CODE', 'user_code': 'X', + 'verification_uri': 'https://idp/device', + 'expires_in': 600, 'interval': 10 ** 9, + } + self.state.token_script = [(200, None)] + auth = self.make_auth() + auth.token() + self.assertTrue(self._clock.sleeps) + self.assertLessEqual(max(self._clock.sleeps), _MAX_POLL_INTERVAL) + + def test_oversized_expires_in_is_capped(self): + # A hostile expires_in must not keep the poll loop (and the lock) alive + # indefinitely; the lifetime is capped so a never-authorized flow still + # terminates promptly rather than looping millions of times. M2. + from questdb.auth._device import ( + _MAX_DEVICE_CODE_LIFETIME, _MAX_POLL_INTERVAL) + self.state.device_response = { + 'device_code': 'DEV-CODE', 'user_code': 'X', + 'verification_uri': 'https://idp/device', + 'expires_in': 10 ** 9, 'interval': 10 ** 9, # interval clamps too + } + self.state.token_script = [(400, {'error': 'authorization_pending'})] + auth = self.make_auth() + with self.assertRaises(OidcTimeoutError): + auth.token() + max_polls = _MAX_DEVICE_CODE_LIFETIME // _MAX_POLL_INTERVAL + 1 + self.assertLessEqual(len(self.state.token_requests), max_polls) + def test_access_denied_is_surfaced(self): self.state.token_script = [ (400, {'error': 'access_denied', From 0edf9dc6c45564eec6f6acbffaae737d08b40882 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 16:22:47 +0100 Subject: [PATCH 014/104] fix: map malformed inputs to typed OidcError Several malformed-input paths escaped the package's typed-error contract (callers catch OidcError) with a bare ValueError / AttributeError / http.client.InvalidURL: * a non-string OIDC endpoint in /settings -> AttributeError from .startswith(); now treated as absent so resolution raises a clear OidcConfigError; * a /exec "columns" entry that isn't an object -> AttributeError from .get(); now raises OidcError; * a malformed port in the QuestDB URL -> bare ValueError when an adapter read .port; QuestDB now validates it at construction via safe_urlparse; * the same malformed port reaching the /settings or discovery fetch -> http.client.InvalidURL; request() now wraps InvalidURL as OidcConfigError and any other HTTPException as OidcNetworkError, so the single HTTP choke point never leaks a raw http.client exception. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 6 ++++++ src/questdb/auth/_http.py | 8 +++++++- src/questdb/auth/_questdb.py | 18 +++++++++++++++--- test/test_auth.py | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index ccd39fce..6160ce22 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -197,6 +197,12 @@ def _resolve_endpoint(value: Optional[str], cfg: Dict[str, Any]) -> Optional[str """ if not value: return None + if not isinstance(value, str): + # A non-string endpoint from /settings (e.g. a JSON number) is + # malformed; treat it as absent so resolution falls through to a clear + # OidcConfigError (or the IdP-discovery fallback) instead of an + # AttributeError from .startswith() escaping the typed-error contract. + return None if value.startswith('http://') or value.startswith('https://'): return value if value.startswith('/'): diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index ec29a1e3..e9f2aa59 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -38,6 +38,7 @@ from __future__ import annotations +import http.client import ipaddress import json import os @@ -221,7 +222,12 @@ def request( return HttpResponse(e.code, body, e.headers or {}) except urllib.error.URLError as e: raise OidcNetworkError(f'Failed to reach {url}: {e.reason}') from e - except (TimeoutError, OSError) as e: + except http.client.InvalidURL as e: + # A malformed URL (e.g. a non-integer port) can't be turned into a + # request; surface it as a config error rather than letting a raw + # http.client exception escape the package's typed-error contract. + raise OidcConfigError(f'Malformed URL {url!r}: {e}') from e + except (TimeoutError, OSError, http.client.HTTPException) as e: raise OidcNetworkError(f'Failed to reach {url}: {e}') from e diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py index ac9aa45b..7ecfdcd5 100644 --- a/src/questdb/auth/_questdb.py +++ b/src/questdb/auth/_questdb.py @@ -31,7 +31,7 @@ from ._device import OidcDeviceAuth from ._errors import OidcAuthError, OidcConfigError, OidcError -from ._http import request +from ._http import request, safe_urlparse _DEFAULT_PG_PORT = 8812 _DEFAULT_DATABASE = 'qdb' @@ -60,6 +60,15 @@ def _import_pandas(): def _exec_json_to_df(data: Dict[str, Any], pandas): columns = data.get('columns') or [] + # /exec returns a list of {"name", "type"} column descriptors. A malformed + # response (a non-list, or entries that aren't objects) must surface as a + # clean OidcError, not an AttributeError from .get() escaping the package's + # typed-error contract. + if not isinstance(columns, list) or not all( + isinstance(c, dict) for c in columns): + raise OidcError( + 'QuestDB /exec returned a malformed "columns" field; ' + 'cannot build a DataFrame.') names = [c.get('name') for c in columns] dataset = data.get('dataset') if dataset is None: @@ -114,7 +123,10 @@ def __init__( self.auth = auth self._insecure = insecure self._ctx = auth._ctx - self._parts = urllib.parse.urlparse(self.url) + # safe_urlparse validates the port up-front, raising OidcConfigError + # (not a bare ValueError) for a malformed one, so the adapters that read + # the port stay within the package's typed-error contract. + self._parts, self._port = safe_urlparse(self.url) # -- token access ------------------------------------------------------- @@ -293,7 +305,7 @@ def sender(self, *, port: Optional[int] = None, '(`pip install questdb`).') from e scheme = 'https' if self._parts.scheme == 'https' else 'http' - resolved_port = port or self._parts.port or ( + resolved_port = port or self._port or ( 443 if scheme == 'https' else 9000) conf = (f'{scheme}::addr=' f'{self._ilp_addr(self._require_host(), resolved_port)};') diff --git a/test/test_auth.py b/test/test_auth.py index ea0e4658..ec88184a 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -930,6 +930,15 @@ def test_sql_non_dict_json_raises_oidc_error(self): qdb.sql('SELECT 1') self.assertNotIsInstance(cm.exception, OidcAuthError) + def test_sql_non_dict_columns_raises_oidc_error(self): + # A /exec body whose "columns" entries aren't objects must raise a clean + # OidcError, not an AttributeError from .get() on the column. See M3. + qdb = self._connected() + self.state.exec_response = {'columns': [None], 'dataset': [[1]]} + with self.assertRaises(OidcError) as cm: + qdb.sql('SELECT 1') + self.assertNotIsInstance(cm.exception, OidcAuthError) + class TestConcurrency(AuthTestBase): def test_valid_cached_token_does_not_block_during_signin(self): @@ -1157,6 +1166,13 @@ def test_require_host_rejects_hostless_url(self): QuestDB('localhost', _FakeAuth())._require_host('h.example'), 'h.example') + def test_malformed_port_url_raises_config_error(self): + # A QuestDB URL with a non-integer port must raise OidcConfigError at + # construction, not a bare ValueError when an adapter reads .port. M3. + with self.assertRaises(OidcConfigError): + QuestDB('https://questdb.example.com:notaport', _FakeAuth(), + insecure=True) + def test_sender_hostless_url_raises(self): # The guard propagates through an adapter (not just the helper): # sender() on a host-less URL raises OidcConfigError. See M5. @@ -1210,6 +1226,13 @@ def test_resolve_endpoint_relative_path(self): self.assertEqual(_resolve_endpoint('https://idp/x', cfg), 'https://idp/x') # absolute is kept verbatim + def test_resolve_endpoint_ignores_non_string(self): + # A non-string endpoint from /settings (e.g. a JSON number) must be + # treated as absent, not raise AttributeError from .startswith(). M3. + from questdb.auth._discovery import _resolve_endpoint + self.assertIsNone(_resolve_endpoint(8080, {})) + self.assertIsNone(_resolve_endpoint(True, {})) + def test_settings_config_nesting(self): from questdb.auth._discovery import settings_config self.assertEqual(settings_config({'config': {'a': 1}}), {'a': 1}) @@ -1374,6 +1397,15 @@ def do_GET(self): self.assertEqual(resp.status, 302) self.assertEqual(seen, [('/exec', 'Bearer SECRET')]) + def test_malformed_url_raises_config_error(self): + # A non-integer port must surface as OidcConfigError, not a raw + # http.client.InvalidURL escaping the typed-error contract — this is the + # path the QuestDB /settings / discovery fetches go through. See M3. + from questdb.auth._http import request + with self.assertRaises(OidcConfigError): + request('GET', 'https://questdb.example.com:notaport/settings', + timeout=5) + class TestRendererSecurity(unittest.TestCase): """The Jupyter prompt must never turn an IdP-supplied URL into a From 35c3fbb0eb50e29febc660bc047c683d8cf5e953 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 16:28:42 +0100 Subject: [PATCH 015/104] fix: sanitize device-flow terminal output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plain-text terminal prompt wrote untrusted device-authorization response fields — verification_uri, user_code, the IdP error_description and the JWT-derived identity — verbatim to the TTY. A hostile or MITM'd response could embed ANSI escape sequences (cursor moves, screen clears) to spoof the sign-in prompt or hide the real verification URL. Strip C0/C1 control characters (incl. ESC) from those untrusted strings in format_prompt() and TerminalRenderer.on_success/on_failure before they reach the stream. The Jupyter renderer already html-escapes its output, and the QR path encodes the URL as image data rather than terminal text, so neither needed changes. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_render.py | 30 +++++++++++++++++++++++++----- test/test_auth.py | 27 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 8a0fa9fb..86fb5716 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -34,6 +34,7 @@ from __future__ import annotations import html +import re import sys import urllib.parse from typing import Any, Dict, Optional, TextIO @@ -118,11 +119,30 @@ def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: f'rel="noopener noreferrer">{label}') +_CONTROL_CHARS = re.compile(r'[\x00-\x1f\x7f-\x9f]') + + +def _strip_control(text: Optional[str]) -> str: + """ + Strip C0/C1 control characters (incl. ESC) from an untrusted string before + it is written to a terminal. + + The verification URL, user code and IdP error strings come from the device- + authorization response (untrusted). Writing them verbatim to a TTY would let + a hostile or MITM'd response inject ANSI escape sequences — cursor moves, + screen clears — to spoof the prompt or hide the real sign-in URL. The + Jupyter renderer html-escapes its output; the plain-text path needs this. + """ + if not text: + return '' + return _CONTROL_CHARS.sub('', text) + + def format_prompt(resp: Dict[str, Any]) -> str: """Plain-text sign-in prompt (also used as the notebook fallback).""" - uri = _verification_uri(resp) - code = resp.get('user_code', '') - complete = _verification_uri_complete(resp) + uri = _strip_control(_verification_uri(resp)) + code = _strip_control(str(resp.get('user_code', ''))) + complete = _strip_control(_verification_uri_complete(resp)) lines = [ '🔐 Sign in to QuestDB', f' Open {uri} and enter code: {code}', @@ -184,7 +204,7 @@ def on_success(self, identity: Optional[str], expires_in: float) -> None: if self._countdown_active: self._write('\n') self._countdown_active = False - who = f' as {identity}' if identity else '' + who = f' as {_strip_control(identity)}' if identity else '' mins = max(1, int(round(expires_in / 60))) self._write(f'✅ Signed in{who} — token cached, expires in {mins} min\n') @@ -192,7 +212,7 @@ def on_failure(self, message: str) -> None: if self._countdown_active: self._write('\n') self._countdown_active = False - self._write(f'❌ {message}\n') + self._write(f'❌ {_strip_control(message)}\n') class JupyterRenderer(Renderer): diff --git a/test/test_auth.py b/test/test_auth.py index ec88184a..4404ec49 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1459,6 +1459,33 @@ def _display(self, html_str): # avoid importing IPython self.assertIn(' Date: Thu, 18 Jun 2026 16:28:48 +0100 Subject: [PATCH 016/104] docs: correct questdb.auth changelog and API reference * CHANGELOG: drop the stale "optional on-disk cache" claim (the FileCache backend was removed; tokens are never written to disk) and note the python_requires bump to 3.10. * docs/auth.rst: import TimestampNanos in the integrated-session snippet so it runs as written. * docs/api.rst: document TokenCache, TokenSet, MemoryCache and NullCache, which are exported in __all__ but were missing from the reference. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.rst | 8 +++++++- docs/api.rst | 20 ++++++++++++++++++++ docs/auth.rst | 2 ++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 557b131c..672455bb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -39,7 +39,8 @@ Highlights: * Auto-discovery of OIDC config from the QuestDB ``/settings`` endpoint, with a fallback to the IdP ``.well-known`` document. -* In-process token cache with silent refresh; optional on-disk cache. +* In-process token cache with silent refresh (tokens are never written to + disk). * Adapters for pandas (REST ``/exec``), SQLAlchemy, psycopg and the ingestion ``Sender``. * ``token()`` / ``headers()`` require no dependencies beyond the standard @@ -48,6 +49,11 @@ Highlights: See the :ref:`OIDC authentication guide ` for details. +Python Version Support +~~~~~~~~~~~~~~~~~~~~~~~~ + +* Raised the minimum supported Python version to 3.10. + 4.1.0 (2025-11-28) ------------------ diff --git a/docs/api.rst b/docs/api.rst index 6b428050..9ff4daf3 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -90,6 +90,26 @@ See the :ref:`oidc_auth` guide for an overview. :undoc-members: :show-inheritance: +.. autoclass:: questdb.auth.TokenCache + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: questdb.auth.TokenSet + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: questdb.auth.MemoryCache + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: questdb.auth.NullCache + :members: + :undoc-members: + :show-inheritance: + .. autoexception:: questdb.auth.OidcError :show-inheritance: diff --git a/docs/auth.rst b/docs/auth.rst index d44e6b1e..d2eedfa1 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -71,6 +71,8 @@ paths. engine = qdb.sqlalchemy_engine() # PG-wire, token as _sso with qdb.psycopg() as conn: # raw psycopg ... + + from questdb.ingress import TimestampNanos # the compiled extension with qdb.sender() as sender: # ingestion (ILP/HTTP) sender.row("trades", columns={"price": 101.5}, at=TimestampNanos.now()) From e043561fc33a47ea7537dbfd680f9d07b64141eb Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 16:36:50 +0100 Subject: [PATCH 017/104] fix: make TokenSet immutable and keep tokens out of repr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lock-free fast path in OidcDeviceAuth reads a published TokenSet without holding a lock, which is only safe because its fields never change after construction — an invariant previously kept by convention alone. Mark TokenSet frozen so any future in-place mutation fails loudly instead of introducing a torn read, and convert the one such mutation (the refresh carry-forward in _refresh) to dataclasses.replace(). Also keep the access/id/refresh tokens out of repr() so a TokenSet that lands in a log line or traceback cannot leak credentials. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_cache.py | 21 +++++++++++++++------ src/questdb/auth/_device.py | 5 ++++- test/test_auth.py | 37 ++++++++++++++++++++++++++++++++----- 3 files changed, 51 insertions(+), 12 deletions(-) diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index be66ee9a..796611c1 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -27,7 +27,7 @@ from __future__ import annotations import threading -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from typing import Dict, Optional, Union from ._errors import OidcConfigError @@ -36,13 +36,22 @@ DEFAULT_SKEW_SECONDS = 30 -@dataclass +@dataclass(frozen=True) class TokenSet: - """A set of tokens obtained from the IdP, plus their expiry.""" + """ + A set of tokens obtained from the IdP, plus their expiry. + + Immutable (``frozen``): the lock-free fast path in + :class:`~questdb.auth._device.OidcDeviceAuth` reads a published ``TokenSet`` + without holding a lock, which is only safe because its fields never change + after construction. Derive a modified copy with :func:`dataclasses.replace` + rather than mutating in place. The three secret fields are kept out of + ``repr`` so a token can't leak into a log line or traceback. + """ - access_token: Optional[str] = None - id_token: Optional[str] = None - refresh_token: Optional[str] = None + access_token: Optional[str] = field(default=None, repr=False) + id_token: Optional[str] = field(default=None, repr=False) + refresh_token: Optional[str] = field(default=None, repr=False) expires_at: float = 0.0 # epoch seconds; 0 == unknown token_type: str = 'Bearer' scope: Optional[str] = None diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index f2797e86..39fdcafb 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -32,6 +32,7 @@ import threading import time import webbrowser +from dataclasses import replace from typing import Any, Dict, Optional from ._cache import TokenSet, make_cache @@ -474,8 +475,10 @@ def _refresh(self, tokens: TokenSet) -> TokenSet: if status == 200: refreshed = self._tokenset_from_response(body) # Many IdPs do not rotate the refresh token; keep the old one. + # TokenSet is frozen, so derive a copy rather than mutating. if not refreshed.refresh_token: - refreshed.refresh_token = tokens.refresh_token + refreshed = replace( + refreshed, refresh_token=tokens.refresh_token) return refreshed raise OidcDeviceFlowError( f"Token refresh failed: {body.get('error', 'unknown error')}", diff --git a/test/test_auth.py b/test/test_auth.py index 4404ec49..fc45a629 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -502,11 +502,38 @@ def test_open_browser_rejects_dangerous_scheme(self): def test_memory_cache_returns_independent_copy(self): cache = MemoryCache() - cache.store('k', TokenSet(access_token='a', refresh_token='r', - expires_at=1.0)) - loaded = cache.load('k') - loaded.refresh_token = 'MUTATED' - self.assertEqual(cache.load('k').refresh_token, 'r') + stored = TokenSet(access_token='a', refresh_token='r', expires_at=1.0) + cache.store('k', stored) + # Each load is a distinct copy — never the object handed to store(), nor + # shared between loads — so a cached entry can't be aliased and reused. + first = cache.load('k') + second = cache.load('k') + self.assertIsNot(first, stored) + self.assertIsNot(first, second) + self.assertEqual(first.refresh_token, 'r') + + def test_tokenset_is_frozen(self): + # TokenSet is immutable: the lock-free fast path reads a published + # TokenSet without a lock, which is only safe if its fields never change + # after construction. Mutating one must raise, not silently succeed. + import dataclasses + t = TokenSet(access_token='a', refresh_token='r', expires_at=1.0) + with self.assertRaises(dataclasses.FrozenInstanceError): + t.refresh_token = 'MUTATED' + # Deriving a modified copy is the supported idiom. + t2 = dataclasses.replace(t, refresh_token='r2') + self.assertEqual(t.refresh_token, 'r') + self.assertEqual(t2.refresh_token, 'r2') + + def test_tokenset_repr_redacts_secrets(self): + # The access/id/refresh tokens must never appear in repr() — a TokenSet + # landing in a log line or traceback would otherwise leak credentials. + r = repr(TokenSet(access_token='SECRET-A', id_token='SECRET-I', + refresh_token='SECRET-R', scope='openid')) + self.assertNotIn('SECRET-A', r) + self.assertNotIn('SECRET-I', r) + self.assertNotIn('SECRET-R', r) + self.assertIn('openid', r) # non-secret metadata still shown class TestNonInteractive(AuthTestBase): From 4e629387af1dc3fabf42af19d251864e9e450092 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 16:45:47 +0100 Subject: [PATCH 018/104] style: sort questdb.auth __all__ to satisfy Ruff RUF022 Reorder the export list using Ruff's isort-style ordering (CamelCase names first, natural-sorted, then lowercase 'connect' last). The public API is unchanged; this only resolves the RUF022 lint warning. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/questdb/auth/__init__.py b/src/questdb/auth/__init__.py index f6ba9d1f..cfacb6e8 100644 --- a/src/questdb/auth/__init__.py +++ b/src/questdb/auth/__init__.py @@ -73,19 +73,19 @@ from ._questdb import QuestDB, connect __all__ = [ - 'connect', - 'QuestDB', - 'OidcDeviceAuth', - 'OidcConfig', - 'TokenCache', - 'TokenSet', 'MemoryCache', 'NullCache', - 'OidcError', + 'OidcAuthError', + 'OidcConfig', 'OidcConfigError', - 'OidcNetworkError', - 'OidcInteractionRequired', + 'OidcDeviceAuth', 'OidcDeviceFlowError', + 'OidcError', + 'OidcInteractionRequired', + 'OidcNetworkError', 'OidcTimeoutError', - 'OidcAuthError', + 'QuestDB', + 'TokenCache', + 'TokenSet', + 'connect', ] From 4a67cc57a3063cb8e936b59eb76fb18f2c466df5 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 16:48:47 +0100 Subject: [PATCH 019/104] docs: make review-pr level-0/1 Step 2.5 rules consistent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The level table said level 0 skips all of Step 2.5, but Step 2.5e is documented as running at every level — a self-contradictory default path. Clarify that levels 0 and 1 skip Steps 2.5a-d while still running Step 2.5e (build & binding profile), aligning the table with the 'runs at every level' rule. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/skills/review-pr/SKILL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude/skills/review-pr/SKILL.md b/.claude/skills/review-pr/SKILL.md index 6b408b4a..01841f67 100644 --- a/.claude/skills/review-pr/SKILL.md +++ b/.claude/skills/review-pr/SKILL.md @@ -36,8 +36,8 @@ The level controls how much of the review below actually runs. Lower levels keep | Level | What runs | |-------|-----------| -| **0 (default)** | Steps 1, 2, 4. Skip Step 2.5. Skip Step 3 — no agent spawn; review the diff inline in the main loop, using Read/Grep on demand to resolve ambiguities. Skip Step 3b — verify each finding inline as you write it. Single-pass review covering correctness, Cython memory/refcount/GIL safety, C-ABI binding correctness, tests, and coding standards on the diff itself. | -| **1** | Adds Step 2.5a (semantic delta only — skip 2.5b/2.5c/2.5d). In Step 3, launch only Agent 1 (correctness), Agent 2 (Cython memory & refcount safety), and Agent 7 (tests) in parallel. Skip all other agents. Skip Step 3b — verify findings inline as you draft the report. | +| **0 (default)** | Steps 1, 2, 4. Skip Steps 2.5a-d, but still run Step 2.5e (build & binding profile — mandatory at every level). Skip Step 3 — no agent spawn; review the diff inline in the main loop, using Read/Grep on demand to resolve ambiguities. Skip Step 3b — verify each finding inline as you write it. Single-pass review covering correctness, Cython memory/refcount/GIL safety, C-ABI binding correctness, tests, and coding standards on the diff itself. | +| **1** | Adds Step 2.5a (semantic delta only — skip 2.5b/2.5c/2.5d; Step 2.5e still runs, as at every level). In Step 3, launch only Agent 1 (correctness), Agent 2 (Cython memory & refcount safety), and Agent 7 (tests) in parallel. Skip all other agents. Skip Step 3b — verify findings inline as you draft the report. | | **2** | Full Step 2.5, but in 2.5b restrict the callsite inventory to public Python symbols (exported in `__all__` / `ingress.pyi`) plus every `cdef`/`cpdef` function and every C-ABI symbol declared in the `.pxd` files. In Step 3, launch Agents 1-8. Skip Agent 9 (cross-context) and Agent 10 (adversarial fresh-context). Step 3b uses a single batched verification agent for all findings instead of one per finding. | | **3** | Every step below as written, all 10 agents, per-finding verification. The full mission-critical pass. | From 124d4c2afb0b771c12e447964ca4280e50b47c88 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 16:52:04 +0100 Subject: [PATCH 020/104] docs: exclude Agent 10 from review-pr Step 2.5 input contract The Step 3 preamble and Step 2.5 stated that every Step 3 agent receives the change-surface map and build/binding profile facts, but Agent 10 (the fresh-context adversarial agent) is documented as receiving only the diff and changed file names. Carve Agent 10 out of all three general-rule statements (Steps 2.5, 2.5e, and the Step 3 input contract) so they no longer contradict Agent 10's own section. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/skills/review-pr/SKILL.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.claude/skills/review-pr/SKILL.md b/.claude/skills/review-pr/SKILL.md index 01841f67..2ff79a80 100644 --- a/.claude/skills/review-pr/SKILL.md +++ b/.claude/skills/review-pr/SKILL.md @@ -68,7 +68,7 @@ Check: ## Step 2.5: Map the change surface -Before launching review agents, produce a structured change surface map. This step is mandatory and must use Grep/Glob — do not reason about callsites from memory. The output of this step is required input for every agent in Step 3. +Before launching review agents, produce a structured change surface map. This step is mandatory and must use Grep/Glob — do not reason about callsites from memory. The output of this step is required input for every Step 3 agent except Agent 10 (the fresh-context adversarial agent, which deliberately works from the diff alone). ### 2.5a Semantic delta per changed symbol @@ -152,11 +152,11 @@ Record, with file:line citations: - **Minimum numpy / Python versions** (`pyproject.toml`: `requires-python`, `numpy>=1.21.0`). Code that uses a newer numpy C-API or Python C-API symbol than the floor breaks the oldest supported build. State the floor. - **`abort()` is imported** (`from libc.stdlib cimport ... abort`). Any reachable `abort()` call, or any Rust panic that crosses the C ABI, terminates the host interpreter with no traceback. Flag the path. -A review without this section is incomplete. State the relevant facts (directives, exception default, submodule commit) in one line at the top of every Step 3 agent prompt so the agent reasons from the right premise. +A review without this section is incomplete. State the relevant facts (directives, exception default, submodule commit) in one line at the top of every Step 3 agent prompt (except Agent 10's, which works from the diff alone) so the agent reasons from the right premise. ## Step 3: Parallel review -Every agent receives: +Every agent except Agent 10 receives: 1. The PR diff 2. The full change surface map from Step 2.5 (semantic deltas, callsite inventory, implicit contracts, cross-context exposure list, build & binding profile facts) From a062a0afbd222fa9f14a98467df7f4c6f6a28323 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 18:08:36 +0100 Subject: [PATCH 021/104] fix: ignore user-writable /settings preferences QuestDB /settings nests server-authoritative values under a top-level "config" object alongside a user-writable "preferences" sibling (the web console persists UI prefs there via PUT /settings). Discovery now reads only "config" and refuses to fall back to the top level of a structured response, so a user who can write a preference cannot smuggle an acl.oidc.* key (e.g. a redirected token endpoint that points the device code / refresh token at an attacker) into the resolved OIDC config. Genuinely flat legacy responses are still tolerated at the top level. Ports the trust model from the Java client (java-questdb-client#52). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 36 ++++++++++++++------ test/test_auth.py | 62 ++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 10 deletions(-) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 6160ce22..50732e6c 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -89,17 +89,33 @@ def _as_bool(value: Any, default: Optional[bool] = None) -> Optional[bool]: def settings_config(settings: Any) -> Dict[str, Any]: """ - Return the flat config map from a ``/settings`` response. - - Modern servers nest values under a ``"config"`` object; older ones return - them at the top level. We tolerate both. + Return the trusted config map from a ``/settings`` response. + + Modern QuestDB nests the server-authoritative values under a top-level + ``"config"`` object, alongside a **user-writable** ``"preferences"`` sibling + (the web console persists UI preferences there via ``PUT /settings``). + Discovery must read only ``"config"`` and never the top level, so a user who + can write a preference cannot smuggle an ``acl.oidc.*`` key — e.g. a + redirected ``token.endpoint`` that points the device code / refresh token at + an attacker — into the resolved OIDC configuration. + + A genuinely flat, legacy ``/settings`` response (no ``"config"`` / + ``"preferences"`` split) is still tolerated at the top level. """ - if isinstance(settings, dict): - cfg = settings.get('config') - if isinstance(cfg, dict): - return cfg - return settings - return {} + if not isinstance(settings, dict): + return {} + cfg = settings.get('config') + if isinstance(cfg, dict): + return cfg + # A structured response carries the user-writable "preferences" sibling + # (and normally the "config" object). If either marker is present, the top + # level is NOT trusted config: read "config" or nothing — so user-writable + # preferences can never be mistaken for server-authoritative config, even + # when "config" is absent or malformed. + if 'config' in settings or 'preferences' in settings: + return {} + # Legacy flat response: no config/preferences split; tolerate top-level keys. + return settings def fetch_settings( diff --git a/test/test_auth.py b/test/test_auth.py index fc45a629..162a6ac5 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -662,6 +662,34 @@ def test_from_questdb_reads_settings(self): self.base + '/device') self.assertEqual(auth.token(), ID_TOKEN) + def test_user_writable_preferences_cannot_override_config(self): + # A user-writable "preferences" sibling in /settings must never override + # the trusted "config" object during discovery: end-to-end, the resolved + # credential endpoints come from "config", not the attacker's prefs. + self.state.settings = { + 'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.scope': 'openid groups', + 'acl.oidc.groups.encoded.in.token': True, + 'acl.oidc.token.endpoint': self.base + '/token', + 'acl.oidc.device.authorization.endpoint': self.base + '/device', + }, + 'preferences.version': 1, + 'preferences': { + 'acl.oidc.token.endpoint': 'https://evil.example.com/token', + 'acl.oidc.device.authorization.endpoint': + 'https://evil.example.com/device', + }, + } + auth = OidcDeviceAuth.from_questdb( + self.base, insecure=True, interactive=True, renderer=Renderer(), + _clock=FakeClock()) + self.assertEqual(auth.config.token_endpoint, self.base + '/token') + self.assertEqual(auth.config.device_authorization_endpoint, + self.base + '/device') + self.assertEqual(auth.token(), ID_TOKEN) + def test_well_known_fallback_for_device_endpoint(self): # Settings advertise OIDC + token endpoint but NOT the device endpoint; # issuer= is pinned, so the IdP .well-known fallback is allowed. @@ -1265,6 +1293,40 @@ def test_settings_config_nesting(self): self.assertEqual(settings_config({'config': {'a': 1}}), {'a': 1}) self.assertEqual(settings_config({'a': 1}), {'a': 1}) # flat fallback + def test_settings_config_ignores_user_writable_preferences(self): + # QuestDB /settings nests server-authoritative values under "config" + # alongside a user-writable "preferences" sibling (the web console + # persists UI prefs there). Discovery must read only "config", so a user + # who can write a preference cannot smuggle an acl.oidc.* key in to + # redirect the device code / refresh token. Ported from the Java client. + from questdb.auth._discovery import settings_config + resp = { + 'config': { + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': 'https://idp.example.com/token'}, + 'preferences.version': 0, + 'preferences': { + 'acl.oidc.token.endpoint': 'https://evil.example.com/token'}, + } + cfg = settings_config(resp) + self.assertEqual(cfg['acl.oidc.token.endpoint'], + 'https://idp.example.com/token') + self.assertNotIn('evil', str(cfg)) + # A structured response (one carrying the user-writable "preferences" + # sibling) must NOT fall back to trusting the top level when "config" is + # absent or malformed: read nothing rather than the top level. + self.assertEqual( + settings_config({'preferences': {'acl.oidc.token.endpoint': 'x'}}), + {}) + self.assertEqual( + settings_config({'config': None, + 'preferences': {'acl.oidc.client.id': 'x'}}), + {}) + # A genuinely flat / legacy response (no config/preferences split) is + # still tolerated at the top level. + self.assertEqual(settings_config({'acl.oidc.client.id': 'q'}), + {'acl.oidc.client.id': 'q'}) + class TestEndpointValidation(unittest.TestCase): def setUp(self): From fe34ffa31a6aac75f5ffdccc1b4f3afb7d52eaa4 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 18 Jun 2026 19:25:59 +0100 Subject: [PATCH 022/104] fix: reject conf metachars in QuestDB host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit QuestDB._require_host() passed the URL hostname unsanitized into the ILP conf string sender() builds (addr=host:port;). urlparse keeps ';' and '=' in .hostname, so a crafted or tampered URL such as "https://host;tls_verify=unsafe_off;x=" injected extra conf params — silently disabling the sender's TLS certificate verification (and exposing the bearer token to a MITM), or e.g. auto_flush=off for data loss. Reject ';', '=', whitespace and control characters in the resolved host (':' stays allowed for IPv6 literals) at the single chokepoint shared by sender()/psycopg()/sqlalchemy_engine(). Add a regression test. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_questdb.py | 18 ++++++++++++++++++ test/test_auth.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py index 7ecfdcd5..8f7d8be3 100644 --- a/src/questdb/auth/_questdb.py +++ b/src/questdb/auth/_questdb.py @@ -26,6 +26,7 @@ from __future__ import annotations +import re import urllib.parse from typing import Any, Dict, Optional @@ -36,6 +37,15 @@ _DEFAULT_PG_PORT = 8812 _DEFAULT_DATABASE = 'qdb' +# A hostname or IP literal never contains the ILP conf-string delimiters (';' +# separates parameters, '=' separates key from value) nor whitespace/control +# characters. Reject them in the resolved host so a crafted or tampered URL +# can't smuggle extra conf parameters — e.g. ';tls_verify=unsafe_off;', which +# silently disables TLS certificate verification — into the 'addr=host:port;' +# string sender() hands to Sender.from_conf. Note ':' is intentionally allowed +# (IPv6 literals contain it; _ilp_addr brackets them). +_ILLEGAL_HOST_CHARS = re.compile(r'[\x00-\x20\x7f;=]') + _AUTH_HINT = ( 'QuestDB rejected the token (HTTP {status}). Common causes:\n' " * scope / 'acl.oidc.groups.encoded.in.token' mismatch — the server may " @@ -208,6 +218,14 @@ def _require_host(self, host: Optional[str] = None) -> str: f'The QuestDB URL {self.url!r} has no host. Use a URL with an ' 'explicit host (e.g. "https://questdb.example.com:9000"), or ' 'pass host=... to the adapter.') + if _ILLEGAL_HOST_CHARS.search(resolved): + raise OidcConfigError( + f'The QuestDB host {resolved!r} contains an illegal character ' + "(';', '=', whitespace or a control character). A hostname or " + 'IP address never does; this indicates a malformed or tampered ' + 'URL. (Such a host could otherwise inject ILP conf parameters ' + 'such as "tls_verify=unsafe_off" into the sender, silently ' + 'disabling TLS certificate verification.)') return resolved @staticmethod diff --git a/test/test_auth.py b/test/test_auth.py index 162a6ac5..41305b3a 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1228,6 +1228,39 @@ def test_malformed_port_url_raises_config_error(self): QuestDB('https://questdb.example.com:notaport', _FakeAuth(), insecure=True) + def test_host_with_conf_metachars_rejected(self): + # C1: a host containing the ILP conf delimiters (';' / '=') or + # whitespace must be rejected, never spliced into the + # `addr=host:port;` conf string. Otherwise a crafted/tampered URL host + # injects extra conf params — e.g. `tls_verify=unsafe_off`, which + # silently disables the sender's TLS certificate verification, or + # `auto_flush=off` (data loss). urlparse() keeps ';'/'=' in .hostname. + for bad in ('https://realhost;tls_verify=unsafe_off;x=', + 'https://a=b'): + with self.subTest(url=bad): + with self.assertRaises(OidcConfigError): + self._qdb(bad)._require_host() + # An explicit host= override goes through the same guard (incl. + # whitespace, which is never valid in a host). + for bad_host in ('evil;tls_verify=unsafe_off', 'a=b', 'h ost'): + with self.subTest(host=bad_host): + with self.assertRaises(OidcConfigError): + self._qdb()._require_host(bad_host) + # A legitimate host (incl. an IPv6 literal, which contains ':') is + # still accepted — the guard must not over-reject. + self.assertEqual(self._qdb()._require_host('::1'), '::1') + self.assertEqual( + self._qdb()._require_host('questdb.example.com'), + 'questdb.example.com') + # The guard fires through the adapter (sender), before the conf string + # is built and handed to Sender.from_conf. + qdb = self._qdb('https://realhost;tls_verify=unsafe_off:9000') + fake = types.ModuleType('questdb.ingress') + fake.Sender = object() # import must succeed so we reach the guard + with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): + with self.assertRaises(OidcConfigError): + qdb.sender() + def test_sender_hostless_url_raises(self): # The guard propagates through an adapter (not just the helper): # sender() on a host-less URL raises OidcConfigError. See M5. From ae4c5ba10095ce22279abffc35243034fd48ba29 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 19 Jun 2026 11:22:31 +0100 Subject: [PATCH 023/104] fix: harden questdb.auth untrusted-input handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M1 — keep four untrusted-input failures within the OidcError contract instead of leaking a raw stdlib exception: * int(float('inf')) from a JSON Infinity (json.loads accepts it) raised OverflowError — not a ValueError — in the expires_in/interval parses (_device.py); add OverflowError to those handlers. * urllib.parse.urlparse() itself raises ValueError on a malformed IPv6 literal (e.g. "https://[::1") before .port is read. safe_urlparse now wraps the urlparse() call, and _require_secure routes through it, so a bad endpoint from /settings or discovery raises OidcConfigError. * build_ssl_context() leaked FileNotFoundError/ssl.SSLError for a mistyped ca_bundle path (or env var); map to OidcConfigError. * deeply-nested JSON makes json.loads raise RecursionError — not a ValueError; catch it in get_json/post_form and QuestDB.sql. M2 — _strip_control now also strips Unicode bidi-override / zero-width / line-separator ranges (U+200B-200F, U+2028-202E, U+2066-2069, U+FEFF), not just C0/C1. U+202E (RIGHT-TO-LEFT OVERRIDE) in an untrusted device response could otherwise reverse displayed text in the terminal prompt to disguise the real sign-in host. Add 8 regression tests (overflow expires_in/interval, malformed-IPv6 at safe_urlparse/_require_secure/_normalize_url/validate_endpoint_origins/ constructor, bad CA bundle, deeply-nested JSON, bidi/zero-width stripping). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 8 ++- src/questdb/auth/_http.py | 39 ++++++++--- src/questdb/auth/_questdb.py | 9 +-- src/questdb/auth/_render.py | 20 ++++-- test/test_auth.py | 125 +++++++++++++++++++++++++++++++++++ 5 files changed, 177 insertions(+), 24 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 39fdcafb..e727278b 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -437,7 +437,9 @@ def _store(self, tokens: TokenSet) -> None: def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: try: expires_in = int(body.get('expires_in', _DEFAULT_EXPIRES_IN)) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): + # OverflowError: a JSON Infinity (json.loads accepts it) → int(inf); + # it is not a ValueError, so list it to keep the typed contract. expires_in = _DEFAULT_EXPIRES_IN if expires_in <= 0: # A non-positive lifetime would mark a just-issued token as already @@ -539,7 +541,7 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: device_code = resp['device_code'] try: interval = int(resp.get('interval', self._default_interval)) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): interval = self._default_interval # At least 1s (RFC 8628 floor), and capped so a hostile/huge value can't # pin the polling thread (which holds the acquisition lock) in one @@ -547,7 +549,7 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: interval = min(_MAX_POLL_INTERVAL, max(1, interval)) try: expires_in = int(resp.get('expires_in', _DEFAULT_DEVICE_CODE_LIFETIME)) - except (TypeError, ValueError): + except (TypeError, ValueError, OverflowError): expires_in = _DEFAULT_DEVICE_CODE_LIFETIME # A non-positive lifetime would time the flow out before the first poll # (the user has already been shown the code); treat it as unknown. Cap diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index e9f2aa59..3e691159 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -64,11 +64,21 @@ def build_ssl_context(ca_bundle: Optional[str] = None) -> ssl.SSLContext: ca_bundle or os.environ.get('REQUESTS_CA_BUNDLE') or os.environ.get('SSL_CERT_FILE')) - if ca: + if not ca: + return ssl.create_default_context() + # A missing / unreadable / invalid bundle makes the stdlib raise a raw + # FileNotFoundError or ssl.SSLError; map it to the package's typed error so + # a mistyped ca_bundle path (or env var) fails clearly instead of leaking a + # bare stdlib exception. + try: if os.path.isdir(ca): return ssl.create_default_context(capath=ca) return ssl.create_default_context(cafile=ca) - return ssl.create_default_context() + except (OSError, ssl.SSLError) as e: + raise OidcConfigError( + f'Could not load the CA bundle {ca!r}: {e}. Check the path points ' + 'to a readable PEM/DER certificate file (or a directory of them).' + ) from e class HttpResponse: @@ -96,17 +106,18 @@ def safe_urlparse(url: str) -> tuple: """ ``urllib.parse.urlparse(url)`` paired with its port, but with a typed error. - ``ParseResult.port`` raises a bare ``ValueError`` for a non-integer port - (e.g. ``https://idp:notaport``); re-raise it as :class:`OidcConfigError` so - a malformed endpoint URL stays within the package's error contract instead - of escaping as a raw ``ValueError``. Returns ``(parts, port)``. + Both ``urlparse`` itself (e.g. ``https://[::1`` — a malformed IPv6 literal) + and ``ParseResult.port`` (e.g. ``https://idp:notaport`` — a non-integer + port) raise a bare ``ValueError``; re-raise it as :class:`OidcConfigError` + so a malformed endpoint URL stays within the package's error contract + instead of escaping as a raw ``ValueError``. Returns ``(parts, port)``. """ - parts = urllib.parse.urlparse(url) try: + parts = urllib.parse.urlparse(url) return parts, parts.port except ValueError as e: raise OidcConfigError( - f'Malformed endpoint URL {url!r}: invalid port.') from e + f'Malformed endpoint URL {url!r}: {e}.') from e def _is_loopback(host: Optional[str]) -> bool: @@ -123,7 +134,9 @@ def _is_loopback(host: Optional[str]) -> bool: def _require_secure(url: str, insecure: bool) -> None: - parts = urllib.parse.urlparse(url) + # safe_urlparse maps a malformed URL (bad IPv6 literal / non-integer port) + # to OidcConfigError instead of letting a bare ValueError escape. + parts, _ = safe_urlparse(url) scheme = parts.scheme.lower() if scheme == 'https': return @@ -247,7 +260,9 @@ def get_json( f'HTTP {resp.status} from {url}: {resp.text()[:200]}') try: return resp.json() - except (ValueError, UnicodeDecodeError) as e: + except (ValueError, UnicodeDecodeError, RecursionError) as e: + # RecursionError: deeply-nested JSON exhausts the decoder's stack; it is + # not a ValueError, so catch it explicitly to keep the typed contract. raise OidcError(f'Invalid JSON from {url}: {e}') from e @@ -270,7 +285,9 @@ def post_form( insecure=insecure) try: parsed = resp.json() - except (ValueError, UnicodeDecodeError): + except (ValueError, UnicodeDecodeError, RecursionError): + # RecursionError: deeply-nested JSON exhausts the decoder's stack; not a + # ValueError, so catch it explicitly to keep the typed contract. if resp.ok: raise OidcError( f'Expected JSON from {url}, got: {resp.text()[:200]}') diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py index 8f7d8be3..56caf88c 100644 --- a/src/questdb/auth/_questdb.py +++ b/src/questdb/auth/_questdb.py @@ -183,11 +183,12 @@ def sql(self, query: str, *, limit: Optional[str] = None, f'QuestDB query failed (HTTP {resp.status}): {detail}') try: data = resp.json() - except (ValueError, UnicodeDecodeError): + except (ValueError, UnicodeDecodeError, RecursionError): # A 2xx body that isn't JSON (e.g. an HTML error/login page from a - # reverse proxy or captive portal) must surface as a clean - # OidcError, not a raw JSONDecodeError. Mirrors the error path and - # post_form(). + # reverse proxy or captive portal), or deeply-nested JSON that + # exhausts the decoder's stack (RecursionError, not a ValueError), + # must surface as a clean OidcError, not a raw decoder exception. + # Mirrors the error path and post_form(). raise OidcError( 'QuestDB returned a non-JSON success response from /exec: ' f'{resp.text()[:300]}') diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 86fb5716..121456b8 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -119,19 +119,27 @@ def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: f'rel="noopener noreferrer">{label}') -_CONTROL_CHARS = re.compile(r'[\x00-\x1f\x7f-\x9f]') +# C0/C1 control chars (incl. ESC, which drives ANSI escape sequences) plus the +# Unicode bidi-control, zero-width and line/paragraph-separator ranges. All can +# spoof a terminal prompt: U+202E (RIGHT-TO-LEFT OVERRIDE) reverses displayed +# text to disguise a URL's host; U+2028/U+2029 inject fake lines; zero-width +# chars hide content. Stripped from untrusted device-response fields. +_CONTROL_CHARS = re.compile( + r'[\x00-\x1f\x7f-\x9f\u200b-\u200f\u2028-\u202e\u2066-\u2069\ufeff]') def _strip_control(text: Optional[str]) -> str: """ - Strip C0/C1 control characters (incl. ESC) from an untrusted string before - it is written to a terminal. + Strip control / format characters from an untrusted string before it is + written to a terminal. The verification URL, user code and IdP error strings come from the device- authorization response (untrusted). Writing them verbatim to a TTY would let - a hostile or MITM'd response inject ANSI escape sequences — cursor moves, - screen clears — to spoof the prompt or hide the real sign-in URL. The - Jupyter renderer html-escapes its output; the plain-text path needs this. + a hostile or MITM'd response inject ANSI escape sequences (C0/C1 control + chars — cursor moves, screen clears) or Unicode bidi overrides / zero-width + / line separators to spoof the prompt or hide the real sign-in URL (e.g. + U+202E visually reverses the displayed host). The Jupyter renderer + html-escapes its output; the plain-text path needs this. """ if not text: return '' diff --git a/test/test_auth.py b/test/test_auth.py index 41305b3a..ca30b4b1 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -491,6 +491,28 @@ def test_short_lived_token_valid_at_issue(self): self.assertTrue(t.is_valid(t.issued_at)) # usable right after issue self.assertFalse(t.is_valid(t.expires_at)) # but still does expire + def test_overflow_expires_in_treated_as_unknown(self): + # A non-finite expires_in (JSON Infinity, which json.loads accepts and + # int(inf) turns into an OverflowError — not a ValueError) must not + # crash; treat it as unknown so the token stays usable. See M1. + self.state.token_script = [(200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'token_type': 'Bearer', 'expires_in': float('inf')})] + auth = self.make_auth() + self.assertEqual(auth.token(), ID_TOKEN) + self.assertTrue(auth._tokens.is_valid(self._clock.now())) + + def test_overflow_device_timing_fields_do_not_crash(self): + # Non-finite interval / expires_in in the device-auth response (JSON + # Infinity) must be treated as unknown, not raise OverflowError. See M1. + self.state.device_response = { + 'device_code': 'DEV-CODE', 'user_code': 'X', + 'verification_uri': 'https://idp/device', + 'expires_in': float('inf'), 'interval': float('inf')} + self.state.token_script = [(200, None)] # success on the first poll + auth = self.make_auth() + self.assertEqual(auth.token(), ID_TOKEN) + def test_open_browser_rejects_dangerous_scheme(self): auth = self.make_auth(open_browser=True) with mock.patch('webbrowser.open') as opener: @@ -1391,6 +1413,19 @@ def test_malformed_port_raises_config_error(self): self._validate('https://idp:notaport/token', 'https://idp:notaport/device') + def test_malformed_ipv6_endpoint_raises_config_error(self): + # A malformed IPv6 literal makes urllib.parse.urlparse() itself raise + # ValueError (before .port is read); it must surface as OidcConfigError, + # not a bare ValueError escaping the typed-error contract. See M1. + with self.assertRaises(OidcConfigError): + self._validate('https://[::1', 'https://[::1') + with self.assertRaises(OidcConfigError): + OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint='https://[::1', + token_endpoint='https://[::1', + renderer=Renderer()) + def test_explicit_constructor_enforces_co_location(self): with self.assertRaises(OidcConfigError): OidcDeviceAuth( @@ -1418,6 +1453,14 @@ def test_normalize_url_malformed_port_raises_config_error(self): with self.assertRaises(OidcConfigError): _normalize_url('https://idp:notaport/token') + def test_normalize_url_malformed_ipv6_raises_config_error(self): + # cache_key normalization must also map a malformed IPv6 literal (which + # makes urlparse itself raise) to OidcConfigError, not a bare + # ValueError. See M1. + from questdb.auth._device import _normalize_url + with self.assertRaises(OidcConfigError): + _normalize_url('https://[::1') + def test_realm_path_distinguishes_key(self): # Multi-tenant IdP: same host, different realm path -> distinct keys # (the old origin-only key collided, leaking one realm's token). @@ -1528,6 +1571,69 @@ def test_malformed_url_raises_config_error(self): request('GET', 'https://questdb.example.com:notaport/settings', timeout=5) + def test_require_secure_rejects_malformed_ipv6(self): + # _require_secure routes through safe_urlparse, so a malformed IPv6 + # endpoint raises OidcConfigError instead of a bare ValueError (urlparse + # raises before the scheme is even inspected). See M1. + from questdb.auth._http import _require_secure + with self.assertRaises(OidcConfigError): + _require_secure('https://[::1', insecure=False) + # A well-formed IPv6 URL is still accepted (loopback http is allowed). + _require_secure('http://[::1]:8080/x', insecure=False) + + def test_bad_ca_bundle_raises_config_error(self): + # A missing or invalid CA bundle path (explicit or via env) must surface + # as OidcConfigError, not a raw FileNotFoundError / ssl.SSLError. See M1. + import tempfile + from questdb.auth._http import build_ssl_context + with self.assertRaises(OidcConfigError): + build_ssl_context('/no/such/path/ca.pem') + with tempfile.NamedTemporaryFile('w', suffix='.pem', delete=False) as f: + f.write('not a certificate') + bad = f.name + try: + with self.assertRaises(OidcConfigError): + build_ssl_context(bad) + finally: + os.unlink(bad) + + def test_deeply_nested_json_raises_oidc_error(self): + # Deeply-nested JSON makes json.loads raise RecursionError (not a + # ValueError); get_json / post_form must map it to OidcError rather than + # let it escape the typed-error contract. See M1. + from questdb.auth import _http + deep = (b'[' * 100000) + (b']' * 100000) + + class _Deep(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _send(self): + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(deep))) + self.end_headers() + self.wfile.write(deep) + + def do_GET(self): + self._send() + + def do_POST(self): + self.rfile.read(int(self.headers.get('Content-Length', 0))) + self._send() + + srv = http.server.HTTPServer(('127.0.0.1', 0), _Deep) + threading.Thread(target=srv.serve_forever, daemon=True).start() + base = f'http://127.0.0.1:{srv.server_port}' + try: + with self.assertRaises(OidcError): + _http.get_json(base + '/x', timeout=5) + with self.assertRaises(OidcError): + _http.post_form(base + '/x', {'a': 'b'}, timeout=5) + finally: + srv.shutdown() + srv.server_close() + class TestRendererSecurity(unittest.TestCase): """The Jupyter prompt must never turn an IdP-supplied URL into a @@ -1608,6 +1714,25 @@ def test_terminal_prompt_strips_control_chars(self): self.assertNotIn('\x1b', out) self.assertNotIn('\x07', out) + def test_strip_control_removes_bidi_and_zero_width(self): + # Beyond C0/C1, untrusted device-response fields must have Unicode + # bidi-override / zero-width / line-separator characters stripped before + # they reach a TTY: U+202E (RIGHT-TO-LEFT OVERRIDE) can visually reverse + # a URL to spoof the sign-in host. chr(cp) avoids embedding the + # (invisible) characters in the test source. See M2. + from questdb.auth._render import _strip_control, format_prompt + for cp in (0x202e, 0x202d, 0x2066, 0x2069, 0x200b, 0x200f, + 0x2028, 0x2029, 0xfeff): + self.assertEqual(_strip_control('a' + chr(cp) + 'b'), 'ab', + f'U+{cp:04X} not stripped') + # Legitimate text (incl. accents / CJK / printable ASCII) is preserved. + self.assertEqual(_strip_control('café 北京 user-1'), 'café 北京 user-1') + text = format_prompt({ + 'user_code': 'WD' + chr(0x202e) + 'JB', + 'verification_uri': 'https://idp.example.com/' + chr(0x202e)}) + self.assertNotIn(chr(0x202e), text) + self.assertIn('idp.example.com', text) + if __name__ == '__main__': unittest.main() From afbd8088e334d84db1b247c49f3e271268b44083 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 19 Jun 2026 12:01:48 +0100 Subject: [PATCH 024/104] fix: bound IdP timeout; broaden auth edge tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M3 — thread a per-request `timeout` (default 30s, unchanged) through every IdP call (device-code request, each poll, refresh) via _idp_post, and through the /settings + discovery fetches in from_questdb. It bounds how long a single network leg can hold the token-acquisition lock when the IdP stalls, so an IdP outage no longer freezes other acquiring threads (e.g. SQLAlchemy pool connections) for the urllib default per leg. Exposed on OidcDeviceAuth.__init__ / from_questdb and forwarded by connect(**opts): lower it for tighter lock-hold, raise it for a slow IdP. Total interactive poll duration stays capped by _MAX_DEVICE_CODE_LIFETIME. M4 — add the remaining edge / error-path tests: * device-code / poll / refresh use the configured timeout (M3) * connect(eager=False) defers sign-in until the first token use * IdP .well-known 404 -> OidcError (discovery non-2xx path) * token/device non-JSON 2xx and non-dict JSON -> OidcError; /settings and discovery non-2xx / non-JSON -> OidcError (only /exec had this before) * make_cache resolves 'memory' / None / 'none' / a TokenCache and rejects an unknown spec with OidcConfigError * QR helpers degrade to None when `qrcode` is absent or data is empty Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 19 ++++- src/questdb/auth/_questdb.py | 4 +- test/test_auth.py | 147 +++++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 4 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index e727278b..c3f66f73 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -166,6 +166,7 @@ def __init__( qr: bool = False, renderer: Optional[Renderer] = None, default_interval: int = 5, + timeout: float = 30, _clock=None): # injectable time source for testing if not client_id: raise OidcConfigError('client_id is required') @@ -203,6 +204,13 @@ def __init__( self.open_browser = open_browser self._interactive = interactive self._default_interval = default_interval + # Per-request network timeout for every IdP call (device-code request, + # each poll, refresh). It bounds how long a single network leg can pin + # the acquisition lock if the IdP stalls: lower it to reduce lock-hold + # (and connection-pool starvation) during an IdP outage; raise it for a + # slow IdP. The total interactive-poll duration is separately capped by + # _MAX_DEVICE_CODE_LIFETIME. + self._timeout = timeout self._cache = make_cache(cache) self._ctx = build_ssl_context(ca_bundle) self._renderer = renderer if renderer is not None else make_renderer(qr=qr) @@ -243,6 +251,7 @@ def from_questdb( interactive: Optional[bool] = None, qr: bool = False, renderer: Optional[Renderer] = None, + timeout: float = 30, _clock=None) -> 'OidcDeviceAuth': # injectable time source """ Build an :class:`OidcDeviceAuth` by discovering config from QuestDB. @@ -265,7 +274,8 @@ def from_questdb( issuer=issuer, discovery_url=discovery_url, ctx=ctx, - insecure=insecure) + insecure=insecure, + timeout=timeout) return cls( client_id=cfg.client_id, device_authorization_endpoint=cfg.device_authorization_endpoint, @@ -281,6 +291,7 @@ def from_questdb( interactive=interactive, qr=qr, renderer=renderer, + timeout=timeout, _clock=_clock) # -- public API --------------------------------------------------------- @@ -462,8 +473,10 @@ def _idp_post(self, url: str, form: Dict[str, Any]): # IdP POSTs carry the device code / refresh token, so they are always # required to be https (loopback http is fine for local dev); the # user's `insecure` flag — which is about the QuestDB link — never - # downgrades them. - return post_form(url, form, ctx=self._ctx, insecure=False) + # downgrades them. The timeout bounds how long this leg can hold the + # acquisition lock if the IdP stalls. + return post_form( + url, form, ctx=self._ctx, insecure=False, timeout=self._timeout) def _refresh(self, tokens: TokenSet) -> TokenSet: status, body = self._idp_post( diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py index 56caf88c..034cc8e6 100644 --- a/src/questdb/auth/_questdb.py +++ b/src/questdb/auth/_questdb.py @@ -365,7 +365,9 @@ def connect( until the first call that needs a token. :param opts: Forwarded to :meth:`OidcDeviceAuth.from_questdb` (e.g. ``client_id``, ``scope``, ``audience``, ``issuer``, ``open_browser``, - ``qr``, ``ca_bundle``). + ``qr``, ``ca_bundle``, ``timeout`` — the per-request IdP network + timeout, which also bounds how long a stalled IdP can hold the + token-acquisition lock). """ auth = OidcDeviceAuth.from_questdb( url, flow=flow, cache=cache, insecure=insecure, **opts) diff --git a/test/test_auth.py b/test/test_auth.py index ca30b4b1..270351b0 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -36,6 +36,7 @@ """ import base64 +import contextlib import importlib.util import json import os @@ -106,6 +107,41 @@ def b64(obj): ACCESS_TOKEN = _jwt({'sub': 'user-1', 'scope': 'openid'}) +@contextlib.contextmanager +def _raw_response_server(status, content_type, body): + """A throwaway HTTP server that returns one fixed (status, type, body). + + Used to exercise the transport's handling of responses the scripted mock + IdP can't produce (non-JSON 2xx, non-dict JSON, non-2xx) on the token / + device / settings / discovery endpoints. Yields the base URL. + """ + class _H(http.server.BaseHTTPRequestHandler): + def log_message(self, *a): + pass + + def _send(self): + self.send_response(status) + self.send_header('Content-Type', content_type) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + self._send() + + def do_POST(self): + self.rfile.read(int(self.headers.get('Content-Length', 0))) + self._send() + + srv = http.server.HTTPServer(('127.0.0.1', 0), _H) + threading.Thread(target=srv.serve_forever, daemon=True).start() + try: + yield f'http://127.0.0.1:{srv.server_port}' + finally: + srv.shutdown() + srv.server_close() + + class FakeClock: """Deterministic clock: ``sleep`` advances both monotonic and wall time.""" @@ -513,6 +549,47 @@ def test_overflow_device_timing_fields_do_not_crash(self): auth = self.make_auth() self.assertEqual(auth.token(), ID_TOKEN) + def test_idp_requests_use_configured_timeout(self): + # The device-code / poll / refresh POSTs must use the configured + # timeout, so a stalled IdP can't pin the acquisition lock for the + # urllib default (30s) per network leg. See M3. + seen = [] + + def fake_post_form(url, form, *, ctx=None, insecure=False, + timeout=None): + seen.append(timeout) + if url.endswith('/device'): + return 200, {'device_code': 'D', 'user_code': 'U', + 'verification_uri': 'https://idp/d', + 'expires_in': 600, 'interval': 5} + return 200, {'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'token_type': 'Bearer', 'expires_in': 3600} + + from questdb.auth import _device + auth = self.make_auth(timeout=3) + with mock.patch.object(_device, 'post_form', fake_post_form): + self.assertEqual(auth.token(), ID_TOKEN) + self.assertTrue(seen) + self.assertTrue( + all(t == 3 for t in seen), + f'IdP POSTs did not all use the configured timeout: {seen}') + + def test_connect_lazy_defers_signin(self): + # eager=False must return a session WITHOUT running the device flow; the + # first token-needing call then triggers exactly one sign-in. See M4. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.scope': 'openid groups', + 'acl.oidc.groups.encoded.in.token': True, + 'acl.oidc.token.endpoint': self.base + '/token', + 'acl.oidc.device.authorization.endpoint': self.base + '/device'}} + qdb = connect(self.base, insecure=True, eager=False, + renderer=Renderer(), interactive=True, _clock=FakeClock()) + self.assertEqual(self.state.device_requests, 0) # deferred + self.assertEqual(qdb.token(), ID_TOKEN) # first use signs in + self.assertEqual(self.state.device_requests, 1) + def test_open_browser_rejects_dangerous_scheme(self): auth = self.make_auth(open_browser=True) with mock.patch('webbrowser.open') as opener: @@ -852,6 +929,20 @@ def test_issuer_pin_accepts_matching_origin(self): self.assertEqual(auth.config.device_authorization_endpoint, self.base + '/device') + def test_well_known_404_raises_oidc_error(self): + # issuer pinned (so the IdP fallback is allowed), but the .well-known + # document 404s: get_json maps the non-2xx to OidcError rather than a + # silent miss that would later masquerade as a missing-endpoint error. + # See M4. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': self.base + '/token'}} + self.state.well_known = None # the handler returns 404 for /.well-known + with self.assertRaises(OidcError): + OidcDeviceAuth.from_questdb(self.base, issuer=self.base, + insecure=True) + class TestInsecureSettingsGuard(unittest.TestCase): """ @@ -1382,6 +1473,18 @@ def test_settings_config_ignores_user_writable_preferences(self): self.assertEqual(settings_config({'acl.oidc.client.id': 'q'}), {'acl.oidc.client.id': 'q'}) + def test_make_cache_variants(self): + # The cache factory resolves the documented specs and rejects an + # unknown one with a typed error. See M4. + from questdb.auth._cache import make_cache, MemoryCache, NullCache + self.assertIsInstance(make_cache('memory'), MemoryCache) + self.assertIsInstance(make_cache(None), NullCache) + self.assertIsInstance(make_cache('none'), NullCache) + custom = MemoryCache() + self.assertIs(make_cache(custom), custom) # a TokenCache passes through + with self.assertRaises(OidcConfigError): + make_cache('disk') + class TestEndpointValidation(unittest.TestCase): def setUp(self): @@ -1634,6 +1737,40 @@ def do_POST(self): srv.shutdown() srv.server_close() + def test_post_form_non_json_2xx_raises_oidc_error(self): + # A 2xx body from the token/device endpoint that isn't JSON (e.g. an + # HTML login page from a proxy in front of the IdP) must surface as + # OidcError, not a raw decoder error. Only /exec had this before. M4. + from questdb.auth import _http + with _raw_response_server(200, 'text/html', b'login') as b: + with self.assertRaises(OidcError): + _http.post_form(b + '/token', {'a': 'b'}, timeout=5) + + def test_post_form_non_dict_json_raises_oidc_error(self): + # A 2xx JSON array (valid JSON but not an object) from the token + # endpoint must surface as OidcError. See M4. + from questdb.auth import _http + with _raw_response_server(200, 'application/json', b'[1, 2, 3]') as b: + with self.assertRaises(OidcError): + _http.post_form(b + '/token', {'a': 'b'}, timeout=5) + + def test_get_json_non_2xx_raises_oidc_error(self): + # A non-2xx /settings or discovery response must surface as OidcError. + # See M4. + from questdb.auth import _http + with _raw_response_server(500, 'text/plain', b'boom') as b: + with self.assertRaises(OidcError): + _http.get_json(b + '/settings', timeout=5) + + def test_get_json_non_json_2xx_raises_oidc_error(self): + # A 2xx /settings or discovery body that isn't JSON must surface as + # OidcError, not a raw JSONDecodeError. See M4. + from questdb.auth import _http + with _raw_response_server(200, 'text/html', b'x') as b: + with self.assertRaises(OidcError): + _http.get_json( + b + '/.well-known/openid-configuration', timeout=5) + class TestRendererSecurity(unittest.TestCase): """The Jupyter prompt must never turn an IdP-supplied URL into a @@ -1733,6 +1870,16 @@ def test_strip_control_removes_bidi_and_zero_width(self): self.assertNotIn(chr(0x202e), text) self.assertIn('idp.example.com', text) + def test_qr_helpers_degrade_without_qrcode(self): + # The QR helpers must degrade gracefully (return None), never raise, + # when `qrcode` is absent or the data is empty. See M4. + from questdb.auth import _render + with mock.patch.dict(sys.modules, {'qrcode': None}): + self.assertIsNone(_render._qr_ascii('https://idp/x')) + self.assertIsNone(_render._qr_data_uri('https://idp/x')) + self.assertIsNone(_render._qr_ascii('')) + self.assertIsNone(_render._qr_data_uri('')) + if __name__ == '__main__': unittest.main() From 63edfcea52369fe8e6550b10e4968030c0a99cab Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 19 Jun 2026 12:35:17 +0100 Subject: [PATCH 025/104] fix: address 3 minor questdb.auth review nits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cache_key now includes the token-kind mode (groups_in_token): two sessions differing ONLY in that mode no longer collide on one in-memory cache entry and repeatedly evict each other's token. _select gates the served kind so no wrong token was ever returned, but the collision caused avoidable refreshes / re-prompts. (_device.py) * _resolve_endpoint returns None for a path-only endpoint ("/as/token") when acl.oidc.host is absent, so resolution fails with a clear config error (pin the IdP / pass the endpoint explicitly) — or recovers via IdP discovery when issuer= is pinned — instead of passing a scheme-less "/path" downstream that surfaced as a confusing "insecure/malformed URL". (_discovery.py) * _pg_module chains the underlying ImportError (raise ... from e) so the traceback preserves the real cause. (_questdb.py) Add 3 regression tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 12 ++++++++++-- src/questdb/auth/_discovery.py | 19 +++++++++++++------ src/questdb/auth/_questdb.py | 4 ++-- test/test_auth.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index c3f66f73..db793427 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -318,8 +318,15 @@ def cache_key(self) -> str: Two sessions share a cached token only when they would accept the same one: same IdP token endpoint (**path included**, so multi-tenant realms sharing a host don't collide), client id, scope *set* (order-insensitive), - and audience. The QuestDB URL is deliberately excluded — the same IdP + audience, and token-kind mode (``groups_in_token`` — id_token vs + access_token). The QuestDB URL is deliberately excluded — the same IdP token is valid against any QuestDB that trusts it. + + ``groups_in_token`` is part of the key because it selects which token + kind :meth:`_select` returns; without it two sessions that differ only + in that mode would collide on one entry and repeatedly evict each + other's token (the gate self-corrects, but at the cost of avoidable + refreshes / re-prompts). """ c = self.config scope = ' '.join(sorted(c.scope.split())) if c.scope else '' @@ -328,7 +335,8 @@ def cache_key(self) -> str: _normalize_url(c.token_endpoint), c.client_id, scope, - c.audience or '']) + c.audience or '', + 'groups' if c.groups_in_token else 'access']) def clear(self) -> None: """Forget the cached token (forces a fresh sign-in next time).""" diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 50732e6c..4b44645b 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -223,12 +223,19 @@ def _resolve_endpoint(value: Optional[str], cfg: Dict[str, Any]) -> Optional[str return value if value.startswith('/'): host = cfg.get(_K_HOST) - if host: - tls = _as_bool(cfg.get(_K_TLS_ENABLED), default=True) - scheme = 'https' if tls else 'http' - port = cfg.get(_K_PORT) - netloc = f'{host}:{port}' if port else str(host) - return f'{scheme}://{netloc}{value}' + if not host: + # A path-only endpoint with no acl.oidc.host to resolve it against + # can't be turned into a URL. Treat it as absent (return None) so + # resolution fails with the clear "could not resolve the ... + # endpoint" error rather than passing a scheme-less "/path" + # downstream, where it surfaces as a confusing "insecure/malformed + # URL" instead. + return None + tls = _as_bool(cfg.get(_K_TLS_ENABLED), default=True) + scheme = 'https' if tls else 'http' + port = cfg.get(_K_PORT) + netloc = f'{host}:{port}' if port else str(host) + return f'{scheme}://{netloc}{value}' return value diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py index 034cc8e6..338ff0fd 100644 --- a/src/questdb/auth/_questdb.py +++ b/src/questdb/auth/_questdb.py @@ -107,10 +107,10 @@ def _pg_module(): try: import psycopg2 # type: ignore return psycopg2 - except ImportError: + except ImportError as e: raise ImportError( 'A PostgreSQL driver is required: install `psycopg` (v3) or ' - '`psycopg2-binary`.') + '`psycopg2-binary`.') from e class QuestDB: diff --git a/test/test_auth.py b/test/test_auth.py index 270351b0..84d71f1b 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1401,6 +1401,15 @@ def test_psycopg_missing_dep_raises(self): with self.assertRaises(ImportError): self._qdb().psycopg() + @unittest.skipIf(_HAS_PG_DRIVER, 'a PostgreSQL driver is installed') + def test_pg_module_missing_chains_cause(self): + # The "no PG driver" ImportError chains the underlying import failure + # (raise ... from e) so the traceback preserves the real cause. + from questdb.auth._questdb import _pg_module + with self.assertRaises(ImportError) as cm: + _pg_module() + self.assertIsInstance(cm.exception.__cause__, ImportError) + @unittest.skipIf(importlib.util.find_spec('questdb.ingress') is not None, 'questdb.ingress extension is built') def test_sender_missing_extension_raises(self): @@ -1434,6 +1443,16 @@ def test_resolve_endpoint_ignores_non_string(self): self.assertIsNone(_resolve_endpoint(8080, {})) self.assertIsNone(_resolve_endpoint(True, {})) + def test_resolve_endpoint_relative_path_without_host_is_none(self): + # A path-only endpoint with no acl.oidc.host can't be resolved; it must + # be treated as absent (None) so resolution fails with a clear "could + # not resolve the ... endpoint" error rather than a scheme-less "/path" + # that later surfaces as a confusing "insecure/malformed URL". + from questdb.auth._discovery import _resolve_endpoint + self.assertIsNone(_resolve_endpoint('/as/token.oauth2', {})) + self.assertIsNone( # port present but host missing -> still unresolved + _resolve_endpoint('/as/token.oauth2', {'acl.oidc.port': 443})) + def test_settings_config_nesting(self): from questdb.auth._discovery import settings_config self.assertEqual(settings_config({'config': {'a': 1}}), {'a': 1}) @@ -1591,6 +1610,15 @@ def test_default_port_normalized(self): self._auth( token_endpoint='https://idp.example.com:443/token').cache_key) + def test_groups_in_token_distinguishes_key(self): + # groups_in_token selects which token kind _select returns, so two + # sessions differing ONLY in that mode must not collide on one cache + # entry (and evict each other). scope already has 'openid' here, so the + # keys can differ only by the mode. + self.assertNotEqual( + self._auth(groups_in_token=True).cache_key, + self._auth(groups_in_token=False).cache_key) + class TestTransportSecurity(unittest.TestCase): def test_require_secure_policy(self): From c869e31e772128db5e829dbbc20507e6c0f67355 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 19 Jun 2026 13:34:32 +0100 Subject: [PATCH 026/104] fix: address 4 follow-up auth review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M1 (issuer pin path-scope) — the issuer pin now scopes /settings-advertised credential endpoints to the issuer PATH (segment-aware), not just its origin, so a tampered /settings can't steer the device code / refresh token to a different realm on a path-based IdP (Keycloak issuers are https://host/realms/{realm}). The origin check (validate_endpoint_origins) stays universal; the path check lives in resolve_config and applies ONLY to /settings-supplied endpoints — caller-explicit and IdP-discovered endpoints are authoritative and not path-restricted, so IdPs that place endpoints outside the issuer path (e.g. Azure AD) still work. M2 (forward CA to Sender) — QuestDB.sender() now forwards the private CA (explicit ca_bundle=, else REQUESTS_CA_BUNDLE / SSL_CERT_FILE, same precedence as build_ssl_context) to the ILP Sender as tls_roots for an https QuestDB, so REST queries and ILP ingestion trust the same roots. PEM file only; caller can override via tls_roots=/tls_ca=. ca_bundle is now stored on OidcDeviceAuth and read by QuestDB. M4 (token-field race) — the lock-free fast path is now strictly READ-ONLY: the cache->field promotion of self._tokens moved into the locked slow-path re-check (and an expired token is still promoted there so _acquire can refresh it). Every write to self._tokens is now serialized; the lock-free read of the frozen TokenSet is kept. M5 (default_interval) — from_questdb (and thus connect(**opts)) now accepts and forwards default_interval; connect(default_interval=...) previously raised TypeError. (M3, malformed-IPv6 -> OidcConfigError, was already fixed in ae4c5ba.) docs/auth.rst: issuer pin documented as origin+path with the Azure caveat; sender documented to inherit the CA bundle. Add 6 regression tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/auth.rst | 24 ++++-- src/questdb/auth/_device.py | 30 ++++++-- src/questdb/auth/_discovery.py | 56 +++++++++++++- src/questdb/auth/_questdb.py | 20 +++++ test/test_auth.py | 129 +++++++++++++++++++++++++++++++++ 5 files changed, 247 insertions(+), 12 deletions(-) diff --git a/docs/auth.rst b/docs/auth.rst index d2eedfa1..fbef0b3e 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -217,14 +217,26 @@ Security notes origin and rejects the configuration otherwise. Because ``/settings`` is authoritative-by-QuestDB, a compromised server could in principle point them elsewhere; pass ``issuer=`` to **pin** the IdP so the endpoints are verified - to belong to it and credentials can't be redirected to another host. When the - server does not advertise the device-authorization endpoint (so it must be - discovered from the IdP), ``issuer=`` (or ``discovery_url=``) is **required** - for exactly this reason — the helper refuses to guess the discovery origin - from the server-supplied token endpoint. + to belong to it and credentials can't be redirected to another host. The pin + checks both the **origin** and, for endpoints advertised by ``/settings``, the + issuer **path** — so on a path-based multi-tenant IdP (e.g. Keycloak issuers + ``https://host/realms/{realm}``) a tampered ``/settings`` cannot redirect the + device code / refresh token to a *different realm on the same host*. (Caller- + supplied endpoints and endpoints from the IdP's own discovery document are + trusted as-is and not path-restricted, since some IdPs — e.g. Azure AD — place + their endpoints outside the issuer path; pass such endpoints explicitly or let + discovery resolve them.) When the server does not advertise the device- + authorization endpoint (so it must be discovered from the IdP), ``issuer=`` + (or ``discovery_url=``) is **required** for exactly this reason — the helper + refuses to guess the discovery origin from the server-supplied token endpoint. * Adapters avoid logging the token / PG DSN. Avoid logging them yourself. * Standard proxy / CA settings (``HTTPS_PROXY``, ``REQUESTS_CA_BUNDLE``, - ``SSL_CERT_FILE``) are honoured; you can also pass ``ca_bundle=``. + ``SSL_CERT_FILE``) are honoured; you can also pass ``ca_bundle=``. The same + private CA is forwarded to the ingestion :meth:`~questdb.auth.QuestDB.sender` + (as the ILP ``tls_roots``) for an ``https`` QuestDB, so REST queries and ILP + ingestion trust the same roots. (Only a PEM **file** is forwarded this way; + for a CA *directory*, or to override, pass ``tls_roots=``/``tls_ca=`` to + ``sender()``.) Dependencies =========== diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index db793427..a836626f 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -202,6 +202,10 @@ def __init__( # cleartext over the network even when this is set. self.insecure = insecure self.open_browser = open_browser + # Kept so adapters that build their own transport (QuestDB.sender's ILP + # Sender) can forward the same private CA the urllib _ctx uses, instead + # of falling back to the default trust roots. See QuestDB.sender. + self._ca_bundle = ca_bundle self._interactive = interactive self._default_interval = default_interval # Per-request network timeout for every IdP call (device-code request, @@ -251,6 +255,7 @@ def from_questdb( interactive: Optional[bool] = None, qr: bool = False, renderer: Optional[Renderer] = None, + default_interval: int = 5, timeout: float = 30, _clock=None) -> 'OidcDeviceAuth': # injectable time source """ @@ -291,6 +296,7 @@ def from_questdb( interactive=interactive, qr=qr, renderer=renderer, + default_interval=default_interval, timeout=timeout, _clock=_clock) @@ -391,9 +397,13 @@ def _missing_required_token_error(self) -> OidcDeviceFlowError: 'access_token.') def _obtain_tokens(self) -> TokenSet: - # Fast path: return a valid cached token without taking the lock, so a - # caller with a usable token never blocks behind another thread's - # in-progress refresh or interactive sign-in. + # Fast path: return a valid token without taking the lock, so a caller + # with a usable token never blocks behind another thread's in-progress + # refresh or interactive sign-in. This path is READ-ONLY: it never + # writes self._tokens (M4). Every write to that field happens under the + # lock (the promotion below, plus _store and clear), so the lock-free + # reader can't race a concurrent write / lose an update / resurrect a + # just-cleared token. tokens = self._valid_cached() if tokens is not None: return tokens @@ -401,17 +411,27 @@ def _obtain_tokens(self) -> TokenSet: # overlapping refreshes or double-prompt; the loser re-checks and # reuses the winner's freshly acquired token. with self._lock: + # Promote a cached token into the field under the lock (even an + # expired one, so _acquire can reuse its refresh_token for a silent + # refresh). Done here, not on the lock-free fast path, so every + # write to self._tokens stays serialized. + if self._tokens is None: + cached = self._cache.load(self.cache_key) + if cached is not None: + self._tokens = cached tokens = self._valid_cached() if tokens is not None: return tokens return self._acquire() def _valid_cached(self) -> Optional[TokenSet]: + # Read-only: reads the published field, falling back to a read of the + # shared cache backend. It never writes self._tokens — that write is + # done only under the lock (in _obtain_tokens' slow path / _store / + # clear) — so it is safe to call on the lock-free fast path. tokens = self._tokens if tokens is None: tokens = self._cache.load(self.cache_key) - if tokens is not None: - self._tokens = tokens if (tokens is not None and tokens.is_valid(self._now()) and self._has_required_token(tokens)): return tokens diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 4b44645b..f55d3aba 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -161,6 +161,24 @@ def _settings_channel_is_plaintext(questdb_url: str) -> bool: parts.hostname) +def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: + """ + True if ``endpoint``'s path is the issuer's path or a sub-path of it. + + Segment-aware, so ``/realms/prod`` does not match ``/realms/production``. A + root issuer (no path, e.g. ``https://idp.example.com``) constrains the + origin only and matches any path. Used to keep a tampered ``/settings`` from + redirecting credentials to a different tenant on a path-based multi-tenant + IdP (Keycloak issuers are ``https://host/realms/{realm}``), which an + origin-only check can't catch. + """ + base = (safe_urlparse(issuer)[0].path or '').rstrip('/') + if not base: + return True + ep = safe_urlparse(endpoint)[0].path or '' + return ep == base or ep.startswith(base + '/') + + def validate_endpoint_origins( token_endpoint: str, device_authorization_endpoint: str, @@ -177,7 +195,16 @@ def validate_endpoint_origins( * the two credential endpoints must share a single origin (they are always co-located on the authorization server per RFC 8628); and * when the ``issuer`` is known independently (passed explicitly or resolved - from the IdP ``.well-known``), both endpoints must belong to it. + from the IdP ``.well-known``), both endpoints must share its **origin**. + + This is an origin-level check: it does **not**, on its own, isolate + path-based multi-tenant realms (e.g. Keycloak issuers + ``https://host/realms/{realm}``, where every realm shares one origin). That + path-scoping is enforced separately in :func:`resolve_config`, and only for + endpoints advertised by the (untrusted) QuestDB ``/settings`` — endpoints + from IdP discovery (the issuer's own ``.well-known``) and caller-explicit + endpoints are authoritative and are not path-restricted (some IdPs, e.g. + Azure AD, legitimately place endpoints outside the issuer path). Pass ``issuer=`` to pin the IdP explicitly when QuestDB advertises the endpoints directly (so a compromised server cannot redirect the token POST). @@ -357,6 +384,33 @@ def resolve_config( 'device_authorization_endpoint=...), or connect to QuestDB over ' 'https so /settings is authenticated.') + # When the credential endpoints came from QuestDB /settings (not the + # caller) and an issuer is pinned out-of-band, require each to sit under the + # issuer's PATH, not merely its origin. Path-based IdPs put every tenant on + # one origin (Keycloak issuers are https://host/realms/{realm}), so the + # origin check alone (validate_endpoint_origins) can't stop a tampered + # /settings from steering the device code / refresh token to a different + # realm on the same host. The issuer is out-of-band, so the server can't + # forge it. Caller-explicit endpoints, and endpoints from IdP discovery (the + # issuer's own .well-known), are authoritative and skip this — some IdPs + # (e.g. Azure AD) legitimately place endpoints outside the issuer path. + if issuer: + for label, url, from_settings in ( + ('token endpoint', token_endpoint, + not explicit_token_endpoint), + ('device-authorization endpoint', + device_authorization_endpoint, not explicit_device_endpoint)): + if url and from_settings and not _endpoint_path_under_issuer( + url, issuer): + raise OidcConfigError( + f'The OIDC {label} advertised by QuestDB /settings ' + f'({url!r}) is not under the pinned issuer ({issuer!r}); ' + 'refusing to send credentials to an endpoint outside the ' + 'trusted issuer (e.g. a different realm on the same host). ' + 'If your IdP places endpoints outside the issuer path, pass ' + 'them explicitly (token_endpoint=..., ' + 'device_authorization_endpoint=...).') + # Fall back to IdP discovery when QuestDB doesn't advertise the device # endpoint (and/or the token endpoint). This contacts the IdP, so it is # held to https/loopback (insecure=False) regardless of the QuestDB flag. diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py index 338ff0fd..98c6c72a 100644 --- a/src/questdb/auth/_questdb.py +++ b/src/questdb/auth/_questdb.py @@ -26,6 +26,7 @@ from __future__ import annotations +import os import re import urllib.parse from typing import Any, Dict, Optional @@ -133,6 +134,10 @@ def __init__( self.auth = auth self._insecure = insecure self._ctx = auth._ctx + # Same private CA bundle the auth/REST transport uses, so sender() can + # forward it to the ILP Sender (which has its own TLS stack). getattr + # keeps test doubles that only set _ctx working. + self._ca_bundle = getattr(auth, '_ca_bundle', None) # safe_urlparse validates the port up-front, raising OidcConfigError # (not a bare ValueError) for a malformed one, so the adapters that read # the port stay within the package's typed-error contract. @@ -328,6 +333,21 @@ def sender(self, *, port: Optional[int] = None, 443 if scheme == 'https' else 9000) conf = (f'{scheme}::addr=' f'{self._ilp_addr(self._require_host(), resolved_port)};') + # Forward the private CA bundle (explicit ca_bundle=, else the + # REQUESTS_CA_BUNDLE / SSL_CERT_FILE env vars — same precedence as + # build_ssl_context) to the Sender's own TLS stack as tls_roots, so an + # https Sender against a private-CA QuestDB trusts the same roots the + # REST/IdP paths do. Only a PEM file works here (tls_roots is a file; + # the Sender has no capath equivalent), and only over https. The caller + # can still override via tls_roots=/tls_ca= in **sender_kwargs. + if (scheme == 'https' + and 'tls_roots' not in sender_kwargs + and 'tls_ca' not in sender_kwargs): + ca = (self._ca_bundle + or os.environ.get('REQUESTS_CA_BUNDLE') + or os.environ.get('SSL_CERT_FILE')) + if ca and os.path.isfile(ca): + sender_kwargs['tls_roots'] = ca return Sender.from_conf(conf, token=self.auth.token(), **sender_kwargs) diff --git a/test/test_auth.py b/test/test_auth.py index 84d71f1b..e2f43d74 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -943,6 +943,18 @@ def test_well_known_404_raises_oidc_error(self): OidcDeviceAuth.from_questdb(self.base, issuer=self.base, insecure=True) + def test_connect_forwards_default_interval(self): + # M5: connect(**opts) routes through from_questdb; default_interval must + # be accepted (it previously raised TypeError) and reach the auth. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': self.base + '/token', + 'acl.oidc.device.authorization.endpoint': self.base + '/device'}} + qdb = connect(self.base, insecure=True, eager=False, default_interval=9, + renderer=Renderer(), interactive=True, _clock=FakeClock()) + self.assertEqual(qdb.auth._default_interval, 9) + class TestInsecureSettingsGuard(unittest.TestCase): """ @@ -1014,6 +1026,48 @@ def test_pin_satisfies_guard_over_plaintext(self): insecure=True, issuer='https://evil.example.com') self.assertEqual(cfg.token_endpoint, 'https://evil.example.com/token') + def test_issuer_path_scopes_settings_endpoints(self): + # M1: a tampered /settings advertising a DIFFERENT realm's endpoints on + # the SAME host (Keycloak path-based multi-tenancy) is rejected when the + # issuer is pinned to a specific realm — the origin check alone can't + # catch it because both realms share one origin. + kc = 'https://idp.example.com/realms' + evil = { + 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': + kc + '/EVIL/protocol/openid-connect/token', + 'acl.oidc.device.authorization.endpoint': + kc + '/EVIL/protocol/openid-connect/auth/device'} + with self.assertRaises(OidcConfigError) as cm: + self._resolve(evil, questdb_url='https://qdb.example.com:9000', + issuer=kc + '/prod') + self.assertIn('issuer', str(cm.exception).lower()) + # The pinned realm's own endpoints are accepted. + good = { + 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': + kc + '/prod/protocol/openid-connect/token', + 'acl.oidc.device.authorization.endpoint': + kc + '/prod/protocol/openid-connect/auth/device'} + cfg = self._resolve(good, questdb_url='https://qdb.example.com:9000', + issuer=kc + '/prod') + self.assertEqual(cfg.token_endpoint, + kc + '/prod/protocol/openid-connect/token') + + def test_issuer_path_scope_skips_explicit_endpoints(self): + # Caller-explicit endpoints are trusted and NOT path-checked, so an IdP + # that places endpoints outside the issuer path (e.g. Azure AD) still + # works when the endpoints are passed explicitly. + cfg = self._resolve( + {'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb'}, + questdb_url='https://qdb.example.com:9000', + issuer='https://idp.example.com/realms/prod', + token_endpoint='https://idp.example.com/oauth2/v2.0/token', + device_authorization_endpoint=( + 'https://idp.example.com/oauth2/v2.0/devicecode')) + self.assertEqual(cfg.token_endpoint, + 'https://idp.example.com/oauth2/v2.0/token') + @unittest.skipIf(pd is None, 'pandas not installed') class TestRestAdapter(AuthTestBase): @@ -1164,6 +1218,19 @@ def call(name): self.assertEqual(results.get('b'), ID_TOKEN) self.assertEqual(self.state.device_requests, 1) # no second prompt + def test_fast_path_does_not_write_tokens_field(self): + # M4: the lock-free fast path must be READ-ONLY. Serving a valid token + # from the shared cache must not write self._tokens — only the locked + # slow path (and _store/clear) write it — so the lock-free reader can't + # race a concurrent write (lost update / clear() resurrection). + auth = self.make_auth() + valid = TokenSet(access_token='a', id_token=ID_TOKEN, refresh_token='r', + expires_at=self._clock.now() + 3600) + auth._cache.store(auth.cache_key, valid) + self.assertIsNone(auth._tokens) # nothing published yet + self.assertEqual(auth.token(), ID_TOKEN) # served via the fast path + self.assertIsNone(auth._tokens) # fast path did not write it + class TestAdapters(unittest.TestCase): """Connection adapters: tested via injected fake modules (the real @@ -1307,6 +1374,52 @@ def from_conf(conf, *, token=None, **kw): qdb.sender() self.assertEqual(captured['conf'], 'https::addr=[::1]:9000;') + def test_sender_forwards_ca_bundle_as_tls_roots(self): + # M2: an https Sender must inherit the private CA bundle (as tls_roots) + # so it trusts the same roots as the REST/IdP paths; http does not, and + # an explicit tls_roots= is never overridden. + import tempfile + + def captured_conf_kwargs(url, *, ca_bundle, **sender_kwargs): + auth = _FakeAuth('TKN') + auth._ca_bundle = ca_bundle + qdb = QuestDB(url, auth, insecure=True) + captured = {} + fake = types.ModuleType('questdb.ingress') + + class Sender: + @staticmethod + def from_conf(conf, *, token=None, **kw): + captured['kw'] = kw + return 'S' + + fake.Sender = Sender + with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): + qdb.sender(**sender_kwargs) + return captured['kw'] + + with tempfile.NamedTemporaryFile('w', suffix='.pem', delete=False) as f: + f.write('-----dummy-----') + ca = f.name + try: + # https + a real CA file -> forwarded as tls_roots. + self.assertEqual( + captured_conf_kwargs('https://db.example.com:9000', + ca_bundle=ca).get('tls_roots'), ca) + # http -> never forwarded (TLS roots are irrelevant). + self.assertNotIn( + 'tls_roots', + captured_conf_kwargs('http://db.example.com:9000', + ca_bundle=ca)) + # An explicit tls_roots= wins over the inherited bundle. + self.assertEqual( + captured_conf_kwargs('https://db.example.com:9000', + ca_bundle=ca, + tls_roots='/other/ca.pem').get('tls_roots'), + '/other/ca.pem') + finally: + os.unlink(ca) + def test_psycopg_uses_bare_ipv6_host(self): # psycopg takes host and port separately, so the IPv6 host is passed # WITHOUT brackets (unlike the ILP addr= form). See M5. @@ -1556,6 +1669,22 @@ def test_explicit_constructor_enforces_co_location(self): token_endpoint='https://attacker.example/token', renderer=Renderer()) + def test_endpoint_path_under_issuer(self): + # M1: segment-aware path containment used to isolate path-based realms. + from questdb.auth._discovery import _endpoint_path_under_issuer as under + iss = 'https://idp.example.com/realms/prod' + self.assertTrue(under(iss + '/protocol/openid-connect/token', iss)) + self.assertTrue(under(iss, iss)) # exact path + self.assertTrue(under(iss + '/', iss)) # trailing slash + self.assertFalse(under('https://idp.example.com/realms/EVIL/token', iss)) + self.assertFalse( # not a *segment* prefix: prod != production + under('https://idp.example.com/realms/production/token', iss)) + # A root issuer (no path) constrains the origin only -> any path is in. + self.assertTrue( + under('https://idp.example.com/anything', 'https://idp.example.com')) + self.assertTrue( + under('https://idp.example.com/x', 'https://idp.example.com/')) + class TestCacheKey(unittest.TestCase): def _auth(self, **kw): From bcfebd9b3a2020cd82e874376d22df0685bbfe6b Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 19 Jun 2026 15:01:03 +0100 Subject: [PATCH 027/104] fix: address 4 moderate questdb.auth review findings - _decode_jwt_claims: catch RecursionError so a deeply-nested JWT payload from a hostile/buggy IdP degrades to no-identity instead of crashing token()/refresh with a raw exception (mirrors _http/_questdb guards). - _endpoint_path_under_issuer: reject '.'/'..' path segments (decoding %2e first) so a tampered /settings can't steer the device code and refresh token to a different realm via a path the IdP normalizes after the prefix check passes. - _render: coerce non-string verification_uri/_complete to str/None and harden _safe_link_url, so an untrusted device response can't crash the prompt renderer with a bare TypeError/AttributeError. - OidcDeviceAuth docstring: document that a missing/expired-token caller blocks behind an in-progress interactive sign-in, and that connect( eager=True) (the default) is the mitigation. Adds regression tests for each (full auth suite: 127 pass, 11 skipped). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 16 ++++++++- src/questdb/auth/_discovery.py | 12 +++++++ src/questdb/auth/_render.py | 19 ++++++++--- test/test_auth.py | 60 ++++++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 6 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index a836626f..966bed5c 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -97,7 +97,11 @@ def _decode_jwt_claims(token: Optional[str]) -> Dict[str, Any]: raw = base64.urlsafe_b64decode(payload.encode('ascii')) claims = json.loads(raw) return claims if isinstance(claims, dict) else {} - except (ValueError, binascii.Error, UnicodeDecodeError): + except (ValueError, binascii.Error, UnicodeDecodeError, RecursionError): + # RecursionError: a deeply-nested JSON payload exhausts the decoder's + # stack; it is not a ValueError, so list it explicitly so a hostile or + # buggy token response can't crash token()/refresh with a raw exception + # here (mirrors the guards in _http.get_json / post_form / QuestDB.sql). return {} @@ -126,6 +130,16 @@ class OidcDeviceAuth: token is returned without blocking on another thread's in-progress sign-in. + **Concurrency note.** The serialization lock is held for the whole of an + interactive sign-in (up to the device-code lifetime, ~30 min). A caller + that already holds a *valid* cached token never blocks, but a caller whose + token is missing or expired blocks behind whoever is signing in; if that + sign-in is abandoned, each waiter then re-prompts in turn. When several + threads share one auth object (e.g. a SQLAlchemy / psycopg connection + pool), sign in once up front — :func:`questdb.auth.connect` does this for + you with ``eager=True`` (the default), so the interactive flow runs a + single time on the main thread before the pool opens connections. + .. code-block:: python from questdb.auth import OidcDeviceAuth diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index f55d3aba..b5f60ae9 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -37,6 +37,7 @@ from __future__ import annotations import ssl +import urllib.parse from dataclasses import dataclass from typing import Any, Dict, Optional @@ -171,11 +172,22 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: redirecting credentials to a different tenant on a path-based multi-tenant IdP (Keycloak issuers are ``https://host/realms/{realm}``), which an origin-only check can't catch. + + A ``.`` / ``..`` path segment is rejected outright: urllib puts the dotted + path on the wire verbatim, but the IdP (or a reverse proxy in front of it) + normalizes it, so ``/realms/prod/../attacker/token`` would satisfy a naive + prefix test yet resolve server-side to a *different* realm — defeating the + very isolation this check exists to provide. Percent-encoded dot segments + (``%2e``) are decoded before the segment scan, since a server may unescape + before normalizing; a legitimate endpoint path never contains dot segments. """ base = (safe_urlparse(issuer)[0].path or '').rstrip('/') if not base: return True ep = safe_urlparse(endpoint)[0].path or '' + decoded_segments = urllib.parse.unquote(ep).split('/') + if '.' in decoded_segments or '..' in decoded_segments: + return False return ep == base or ep.startswith(base + '/') diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 121456b8..86dd2338 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -74,13 +74,20 @@ def detect_interactive() -> bool: def _verification_uri(resp: Dict[str, Any]) -> str: # RFC 8628 uses ``verification_uri``; some IdPs (older Google) use - # ``verification_url``. - return resp.get('verification_uri') or resp.get('verification_url') or '' + # ``verification_url``. The device response is untrusted: coerce to str so a + # non-string value (e.g. a JSON number) can't crash the renderer + # (``re.sub`` / ``html.escape``) with a raw TypeError before the prompt is + # even shown — matching the defensive ``str(user_code)`` at the call sites. + uri = resp.get('verification_uri') or resp.get('verification_url') or '' + return uri if isinstance(uri, str) else '' def _verification_uri_complete(resp: Dict[str, Any]) -> Optional[str]: - return (resp.get('verification_uri_complete') - or resp.get('verification_url_complete')) + # Coerce to str / None for the same untrusted-input reason as + # _verification_uri (a non-string would crash the renderer / _safe_link_url). + uri = (resp.get('verification_uri_complete') + or resp.get('verification_url_complete')) + return uri if isinstance(uri, str) else None def _safe_link_url(url: Optional[str]) -> Optional[str]: @@ -93,7 +100,9 @@ def _safe_link_url(url: Optional[str]) -> Optional[str]: ``data:`` URL that executes in the notebook DOM when clicked (``html.escape`` guards markup, not the URL scheme). """ - if not url: + if not url or not isinstance(url, str): + # A non-string (e.g. a JSON number from an untrusted device response) + # has no scheme to vet and would make urlparse raise; treat it as unsafe. return None try: scheme = urllib.parse.urlparse(url).scheme.lower() diff --git a/test/test_auth.py b/test/test_auth.py index e2f43d74..92d2c3d0 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -549,6 +549,20 @@ def test_overflow_device_timing_fields_do_not_crash(self): auth = self.make_auth() self.assertEqual(auth.token(), ID_TOKEN) + def test_deeply_nested_jwt_payload_does_not_crash(self): + # A hostile/buggy IdP returning an id_token whose payload base64-decodes + # to deeply-nested JSON must not crash token() with a raw RecursionError + # from the best-effort identity decode (RecursionError is not a + # ValueError); the decode degrades to no-identity and the token is still + # returned. See _decode_jwt_claims. + payload = base64.urlsafe_b64encode( + (('[' * 60000) + (']' * 60000)).encode()).rstrip(b'=').decode() + nested = f'aaa.{payload}.sig' + self.state.token_script = [(200, { + 'id_token': nested, 'token_type': 'Bearer', 'expires_in': 3600})] + auth = self.make_auth() + self.assertEqual(auth.token(), nested) + def test_idp_requests_use_configured_timeout(self): # The device-code / poll / refresh POSTs must use the configured # timeout, so a stalled IdP can't pin the acquisition lock for the @@ -1068,6 +1082,24 @@ def test_issuer_path_scope_skips_explicit_endpoints(self): self.assertEqual(cfg.token_endpoint, 'https://idp.example.com/oauth2/v2.0/token') + def test_issuer_path_scope_rejects_dot_segment_traversal(self): + # A tampered /settings can't slip a different realm past the issuer-path + # scope with a '..' segment: '/realms/prod/../EVIL/...' satisfies a naive + # prefix test but the IdP normalizes it to the EVIL realm. The dotted + # path must be rejected (even percent-encoded). See + # _endpoint_path_under_issuer. + kc = 'https://idp.example.com/realms' + for ep in (kc + '/prod/../EVIL/protocol/openid-connect', + kc + '/prod/%2e%2e/EVIL/protocol/openid-connect'): + evil = { + 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': ep + '/token', + 'acl.oidc.device.authorization.endpoint': ep + '/auth/device'} + with self.assertRaises(OidcConfigError) as cm: + self._resolve(evil, questdb_url='https://qdb.example.com:9000', + issuer=kc + '/prod') + self.assertIn('issuer', str(cm.exception).lower()) + @unittest.skipIf(pd is None, 'pandas not installed') class TestRestAdapter(AuthTestBase): @@ -1684,6 +1716,12 @@ def test_endpoint_path_under_issuer(self): under('https://idp.example.com/anything', 'https://idp.example.com')) self.assertTrue( under('https://idp.example.com/x', 'https://idp.example.com/')) + # A '.' / '..' segment (even percent-encoded) is rejected: urllib sends + # the dotted path verbatim and the IdP / proxy normalizes it to a + # DIFFERENT realm, which an origin check can't catch. + self.assertFalse(under(iss + '/../EVIL/protocol/token', iss)) + self.assertFalse(under(iss + '/%2e%2e/EVIL/token', iss)) + self.assertFalse(under(iss + '/./token', iss)) class TestCacheKey(unittest.TestCase): @@ -2037,6 +2075,28 @@ def test_qr_helpers_degrade_without_qrcode(self): self.assertIsNone(_render._qr_ascii('')) self.assertIsNone(_render._qr_data_uri('')) + def test_non_string_verification_uri_does_not_crash(self): + # A hostile/buggy device response with a non-string verification_uri / + # _complete (e.g. a JSON number or list) must not crash the renderer + # with a raw TypeError/AttributeError before the prompt is shown; the + # field is coerced away. See _verification_uri / _safe_link_url. + import io + from questdb.auth._render import ( + format_prompt, TerminalRenderer, JupyterRenderer) + resp = {'user_code': 'WDJB-MJHT', 'verification_uri': 12345, + 'verification_uri_complete': ['not', 'a', 'str'], + 'expires_in': 600, 'interval': 5} + self.assertIn('WDJB-MJHT', format_prompt(resp)) # plain-text path + TerminalRenderer(stream=io.StringIO()).on_prompt(resp) # must not raise + captured = {} + + class _Cap(JupyterRenderer): + def _display(self, html_str): + captured['html'] = html_str + + _Cap().on_prompt(resp) # must not raise + self.assertNotIn(' Date: Fri, 19 Jun 2026 15:47:25 +0100 Subject: [PATCH 028/104] fix: make deeply-nested-JSON auth test robust on Python 3.14 test_deeply_nested_json_raises_oidc_error sent a fixed-depth (100000) JSON body and relied on json.loads raising RecursionError. Python 3.14's C json scanner parses far deeper than 3.13 (and the limit ignores sys.setrecursionlimit), so the body parses and get_json returns instead of raising -> 'OidcError not raised' on the cp314 wheel-test leg. Inject the RecursionError via mock so the test deterministically exercises the get_json / post_form -> OidcError mapping (the actual contract) on every Python version, instead of depending on a version-specific nesting depth. The library guard is unchanged and still correct for <=3.13. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/test_auth.py | 42 ++++++++++++------------------------------ 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/test/test_auth.py b/test/test_auth.py index 92d2c3d0..73d67604 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1896,41 +1896,23 @@ def test_bad_ca_bundle_raises_config_error(self): os.unlink(bad) def test_deeply_nested_json_raises_oidc_error(self): - # Deeply-nested JSON makes json.loads raise RecursionError (not a - # ValueError); get_json / post_form must map it to OidcError rather than - # let it escape the typed-error contract. See M1. + # A RecursionError from json.loads (a deeply-nested JSON body exhausts + # the decoder's stack) must be mapped to OidcError, not escape the + # typed-error contract. The depth at which json actually raises is a + # Python-version detail — the C scanner ignores sys.setrecursionlimit, + # and 3.14 parses far deeper than 3.13, so a fixed-depth body no longer + # raises there — so inject the RecursionError directly to test the + # mapping deterministically across versions. See M1. from questdb.auth import _http - deep = (b'[' * 100000) + (b']' * 100000) - - class _Deep(http.server.BaseHTTPRequestHandler): - def log_message(self, *a): - pass - - def _send(self): - self.send_response(200) - self.send_header('Content-Type', 'application/json') - self.send_header('Content-Length', str(len(deep))) - self.end_headers() - self.wfile.write(deep) - - def do_GET(self): - self._send() - - def do_POST(self): - self.rfile.read(int(self.headers.get('Content-Length', 0))) - self._send() - - srv = http.server.HTTPServer(('127.0.0.1', 0), _Deep) - threading.Thread(target=srv.serve_forever, daemon=True).start() - base = f'http://127.0.0.1:{srv.server_port}' - try: + with _raw_response_server( + 200, 'application/json', b'{"ok": true}') as base, \ + mock.patch.object( + _http.json, 'loads', + side_effect=RecursionError('nesting too deep')): with self.assertRaises(OidcError): _http.get_json(base + '/x', timeout=5) with self.assertRaises(OidcError): _http.post_form(base + '/x', {'a': 'b'}, timeout=5) - finally: - srv.shutdown() - srv.server_close() def test_post_form_non_json_2xx_raises_oidc_error(self): # A 2xx body from the token/device endpoint that isn't JSON (e.g. an From a9c58c8b503e803459d802b547eca755673243ea Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 19 Jun 2026 15:59:46 +0100 Subject: [PATCH 029/104] fix: make questdb.auth clear() reliable across shared-cache instances MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OidcDeviceAuth.clear() took the per-instance self._lock, but the default MemoryCache is process-global. Two instances with the same cache_key have separate locks, so instance A's in-flight _store could resurrect a token instance B had just cleared — clear()'s 'a concurrent refresh/sign-in can't re-populate the cache' guarantee held only within a single instance. Add a per-key generation counter to MemoryCache: clear() bumps it, and the new store_if_current() drops a write whose captured generation is stale. _obtain_tokens captures the generation before the cache read / IdP round-trip and threads it through _acquire -> _store, so a clear() on any instance sharing the global store invalidates a racing store instead of being undone. Backends without the hooks (NullCache / custom TokenCache) store unconditionally as before, so the public TokenCache interface is unchanged. clear()'s docstring/comment now scope the guarantee honestly (local/process cache reset, not an IdP-side revocation). Adds a deterministic end-to-end regression test (clear() on instance B during instance A's sign-in) plus a unit test for the CAS primitive; verified the end-to-end test fails without the fix. Full auth suite: 129 pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_cache.py | 35 +++++++++++++++++++++++++++ src/questdb/auth/_device.py | 48 ++++++++++++++++++++++++++++++------- test/test_auth.py | 41 ++++++++++++++++++++++++++++++- 3 files changed, 114 insertions(+), 10 deletions(-) diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index 796611c1..b3e0ff95 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -88,6 +88,12 @@ def clear(self, key: str) -> None: # pragma: no cover # Module-global so that re-running a notebook cell (which constructs a fresh # ``OidcDeviceAuth``) reuses the already-acquired token instead of re-prompting. _MEMORY_STORE: Dict[str, TokenSet] = {} +# Per-key counter bumped on every clear(). store_if_current() uses it to drop a +# write from an acquisition that began before a concurrent clear() — including a +# clear() on a *different* OidcDeviceAuth that shares this process-global store, +# whose per-instance lock does not serialize against this one — so clear() can't +# be silently undone by an in-flight sign-in / refresh. +_MEMORY_GENERATION: Dict[str, int] = {} _MEMORY_LOCK = threading.Lock() @@ -114,6 +120,35 @@ def store(self, key: str, tokens: TokenSet) -> None: def clear(self, key: str) -> None: with _MEMORY_LOCK: _MEMORY_STORE.pop(key, None) + _MEMORY_GENERATION[key] = _MEMORY_GENERATION.get(key, 0) + 1 + + def generation(self, key: str) -> int: + """ + Current clear()-generation for ``key``. + + Captured before an acquisition's IdP round-trip and handed back to + :meth:`store_if_current`, which drops the write if a ``clear()`` bumped + the counter meanwhile (see :meth:`store_if_current`). + """ + with _MEMORY_LOCK: + return _MEMORY_GENERATION.get(key, 0) + + def store_if_current( + self, key: str, tokens: TokenSet, generation: int) -> bool: + """ + Store ``tokens`` only if no :meth:`clear` happened since ``generation``. + + If a concurrent ``clear()`` — on this or any other + :class:`~questdb.auth.OidcDeviceAuth` sharing this process-global store — + bumped the counter after ``generation`` was captured, the write is + dropped (returns ``False``) so the just-cleared entry is not resurrected + with a now-stale token. Returns ``True`` when the token was stored. + """ + with _MEMORY_LOCK: + if _MEMORY_GENERATION.get(key, 0) != generation: + return False + _MEMORY_STORE[key] = replace(tokens) + return True class NullCache(TokenCache): diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 966bed5c..71060b17 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -360,8 +360,12 @@ def cache_key(self) -> str: def clear(self) -> None: """Forget the cached token (forces a fresh sign-in next time).""" - # Serialize against acquisition so a concurrent refresh/sign-in can't - # re-populate the cache right after we clear it. + # self._lock serializes against THIS instance's acquisition; the shared + # MemoryCache additionally bumps a per-key generation here, so an + # in-flight acquisition on ANOTHER OidcDeviceAuth that shares the + # process-global store can't repopulate the entry after this clear (its + # _store sees the bumped generation and drops the write). This resets the + # local / process cache only — it does not revoke the token at the IdP. with self._lock: self._tokens = None self._cache.clear(self.cache_key) @@ -425,6 +429,12 @@ def _obtain_tokens(self) -> TokenSet: # overlapping refreshes or double-prompt; the loser re-checks and # reuses the winner's freshly acquired token. with self._lock: + # Capture the cache generation before reading or acquiring, so a + # clear() that races this acquisition — including one on another + # OidcDeviceAuth that shares the process-global MemoryCache (whose + # per-instance lock does not serialize against ours) — invalidates + # the store below instead of resurrecting the just-cleared entry. + generation = self._cache_generation() # Promote a cached token into the field under the lock (even an # expired one, so _acquire can reuse its refresh_token for a silent # refresh). Done here, not on the lock-free fast path, so every @@ -436,7 +446,7 @@ def _obtain_tokens(self) -> TokenSet: tokens = self._valid_cached() if tokens is not None: return tokens - return self._acquire() + return self._acquire(generation) def _valid_cached(self) -> Optional[TokenSet]: # Read-only: reads the published field, falling back to a read of the @@ -451,9 +461,11 @@ def _valid_cached(self) -> Optional[TokenSet]: return tokens return None - def _acquire(self) -> TokenSet: + def _acquire(self, generation: int) -> TokenSet: # Called while holding self._lock. Try a silent refresh, else run the - # interactive device flow. + # interactive device flow. `generation` was captured before the cache + # read in _obtain_tokens; _store drops its write if a concurrent clear() + # has bumped it since (see _store / _cache_generation). tokens = self._tokens if tokens is not None and tokens.refresh_token: try: @@ -476,16 +488,34 @@ def _acquire(self) -> TokenSet: # a response is unusable, so fall through to the interactive # flow rather than caching it and looping on every call. if self._has_required_token(refreshed): - self._store(refreshed) + self._store(refreshed, generation) return refreshed fresh = self._run_device_flow() - self._store(fresh) + self._store(fresh, generation) return fresh - def _store(self, tokens: TokenSet) -> None: + def _store(self, tokens: TokenSet, generation: int) -> None: + # self._tokens is this instance's own view, so always set it — the + # caller uses the token it just acquired. The shared-cache write is + # conditional: a clear() (here or on another instance sharing the + # process-global store) that bumped the generation since it was captured + # drops the write, so clear() is not silently undone. Backends without + # generation support (NullCache / a custom TokenCache) store + # unconditionally, exactly as before. self._tokens = tokens - self._cache.store(self.cache_key, tokens) + store_if_current = getattr(self._cache, 'store_if_current', None) + if store_if_current is not None: + store_if_current(self.cache_key, tokens, generation) + else: + self._cache.store(self.cache_key, tokens) + + def _cache_generation(self) -> int: + # MemoryCache tracks a per-key clear()-generation for the cross-instance + # CAS in _store; other backends don't, so default to 0 (the store is + # then unconditional, matching the pre-existing behavior). + generation = getattr(self._cache, 'generation', None) + return generation(self.cache_key) if generation is not None else 0 def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: try: diff --git a/test/test_auth.py b/test/test_auth.py index 73d67604..0f79bc1e 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -64,7 +64,8 @@ OidcNetworkError, TokenSet, ) -from questdb.auth._cache import MemoryCache, _MEMORY_STORE # noqa: E402 +from questdb.auth._cache import ( # noqa: E402 + MemoryCache, _MEMORY_GENERATION, _MEMORY_STORE) from questdb.auth._render import Renderer # noqa: E402 try: @@ -303,6 +304,7 @@ def __init__(self): class AuthTestBase(unittest.TestCase): def setUp(self): _MEMORY_STORE.clear() + _MEMORY_GENERATION.clear() self.server = _MockServer() self.state = self.server.state self.thread = threading.Thread( @@ -1263,6 +1265,43 @@ def test_fast_path_does_not_write_tokens_field(self): self.assertEqual(auth.token(), ID_TOKEN) # served via the fast path self.assertIsNone(auth._tokens) # fast path did not write it + def test_clear_on_other_instance_survives_inflight_acquire(self): + # Two OidcDeviceAuth instances share the process-global MemoryCache + # (same cache_key) but have separate per-instance locks. If instance B + # clears the entry while instance A's sign-in is in flight, A's store + # must NOT resurrect it: the per-key generation A captured before its + # round-trip no longer matches, so the write is dropped and the cache + # stays cleared (the next fresh load re-prompts, honoring clear()). A + # still returns the token it just acquired. See store_if_current. + a = self.make_auth() + b = self.make_auth() + self.assertEqual(a.cache_key, b.cache_key) + + class _ClearMidFlow(Renderer): + def on_prompt(self, resp): + b.clear() # concurrent clear during A's sign-in + + a._renderer = _ClearMidFlow() + self.assertEqual(a.token(), ID_TOKEN) # A still gets its token + # A's store was dropped, so the shared cache is NOT repopulated; a fresh + # instance therefore re-signs in rather than reusing the cleared token. + self.assertNotIn(a.cache_key, _MEMORY_STORE) + + def test_store_if_current_drops_write_after_concurrent_clear(self): + # Unit cover for the CAS primitive the cross-instance guard relies on: + # a generation captured before a clear() must not be allowed to store. + cache = MemoryCache() + key = 'k' + gen = cache.generation(key) # captured before clear + cache.clear(key) # concurrent clear + self.assertFalse( + cache.store_if_current(key, TokenSet(access_token='T1'), gen)) + self.assertIsNone(cache.load(key)) # write dropped + gen2 = cache.generation(key) # unraced store succeeds + self.assertTrue( + cache.store_if_current(key, TokenSet(access_token='T2'), gen2)) + self.assertIsNotNone(cache.load(key)) + class TestAdapters(unittest.TestCase): """Connection adapters: tested via injected fake modules (the real From 7ed6f05946e2d26d7c4fd4155715b28fd4296bab Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 19 Jun 2026 16:04:01 +0100 Subject: [PATCH 030/104] fix: tolerate non-string acl.oidc.* from QuestDB /settings and IdP discovery resolve_config read acl.oidc.scope / client.id / audience from /settings, and the token / device / authorization endpoints and issuer from the IdP .well-known discovery document, without a type check. A non-string value (a JSON list/number from a buggy or tampered server/IdP) flowed through to scope.split(), safe_urlparse() and the cache-key '\x1f'.join, escaping the typed-error contract with a bare AttributeError / TypeError. Add _str_setting() (mirroring _resolve_endpoint, which already drops a non-string endpoint): accept a non-empty string, else treat as absent. Apply it to both the /settings reads and the IdP-discovery doc values. scope falls back to 'openid', audience/issuer drop to None, and a non-string client.id or discovered endpoint surfaces the existing clear OidcConfigError instead of crashing later. Adds unit + end-to-end regression tests for both the /settings and discovery paths; verified each fails without the fix. Full auth suite: 132 pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 40 ++++++++++++++---- test/test_auth.py | 75 ++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 7 deletions(-) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index b5f60ae9..4fea20c5 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -88,6 +88,20 @@ def _as_bool(value: Any, default: Optional[bool] = None) -> Optional[bool]: return default +def _str_setting(value: Any) -> Optional[str]: + """ + A ``/settings`` value as a non-empty string, else ``None``. + + ``/settings`` is server-controlled (and tamperable over a plaintext insecure + channel). A non-string ``acl.oidc.*`` value — a JSON list/number from a buggy + or hostile server — must not reach ``scope.split()`` or the cache-key join as + a raw object, where it would escape the package's typed-error contract with a + bare ``AttributeError`` / ``TypeError``. Mirrors :func:`_resolve_endpoint`, + which already drops a non-string endpoint. + """ + return value if isinstance(value, str) and value else None + + def settings_config(settings: Any) -> Dict[str, Any]: """ Return the trusted config map from a ``/settings`` response. @@ -340,18 +354,22 @@ def resolve_config( f'QuestDB at {questdb_url} reports OIDC is disabled ' f'({_K_ENABLED}=false). Nothing to authenticate against.') - client_id = client_id or cfg.get(_K_CLIENT_ID) + # _str_setting drops a non-string /settings value (e.g. a JSON list) so it + # can't reach scope.split() / the cache-key join as a raw object and escape + # the typed-error contract; a non-string client.id thus reads as absent and + # surfaces the clear "Missing client_id" error below. + client_id = client_id or _str_setting(cfg.get(_K_CLIENT_ID)) if not client_id: raise OidcConfigError( 'Missing OIDC client_id. QuestDB did not advertise ' f'{_K_CLIENT_ID!r} via /settings; pass client_id=... explicitly.') if scope is None: - scope = cfg.get(_K_SCOPE) or 'openid' + scope = _str_setting(cfg.get(_K_SCOPE)) or 'openid' if groups_in_token is None: groups_in_token = _as_bool(cfg.get(_K_GROUPS_IN_TOKEN), default=True) if audience is None: - audience = cfg.get(_K_AUDIENCE) or None + audience = _str_setting(cfg.get(_K_AUDIENCE)) # Track which credential endpoints the caller supplied directly. Those are # trusted; endpoints learned from /settings are only as trustworthy as the @@ -451,13 +469,21 @@ def resolve_config( doc = discover_device_endpoint_from_idp( issuer=issuer, discovery_url=discovery_url, ctx=ctx, insecure=False, timeout=timeout) + # The IdP discovery document is untrusted too: coerce its values the + # same way as /settings values. A non-string endpoint / issuer (a JSON + # number/list from a buggy or hostile IdP) must read as absent — the + # clear "could not resolve" OidcConfigError below, or no issuer pin — + # rather than reach safe_urlparse / the cache-key join as a raw object + # and escape the typed-error contract with a bare AttributeError. device_authorization_endpoint = ( device_authorization_endpoint - or doc.get('device_authorization_endpoint')) - token_endpoint = token_endpoint or doc.get('token_endpoint') + or _str_setting(doc.get('device_authorization_endpoint'))) + token_endpoint = ( + token_endpoint or _str_setting(doc.get('token_endpoint'))) authorization_endpoint = ( - authorization_endpoint or doc.get('authorization_endpoint')) - issuer = issuer or doc.get('issuer') + authorization_endpoint + or _str_setting(doc.get('authorization_endpoint'))) + issuer = issuer or _str_setting(doc.get('issuer')) if not token_endpoint: raise OidcConfigError( diff --git a/test/test_auth.py b/test/test_auth.py index 0f79bc1e..56ec2e8c 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1627,6 +1627,81 @@ def test_resolve_endpoint_ignores_non_string(self): self.assertIsNone(_resolve_endpoint(8080, {})) self.assertIsNone(_resolve_endpoint(True, {})) + def test_str_setting_ignores_non_string(self): + # A non-empty string passes through; anything else (a JSON list / + # number / dict, None, empty string) reads as absent so it can't reach + # scope.split() / the cache-key join as a raw object. + from questdb.auth._discovery import _str_setting + self.assertEqual(_str_setting('openid email'), 'openid email') + for bad in (['openid'], 12345, {'x': 1}, True, '', None): + self.assertIsNone(_str_setting(bad)) + + def test_non_string_settings_do_not_crash_resolution(self): + # A buggy/tampered /settings advertising non-string acl.oidc.* values + # must stay within the typed-error contract instead of crashing later + # with a bare AttributeError / TypeError (scope.split() / the cache-key + # join). scope falls back to 'openid', audience drops to None, and a + # non-string client.id reads as absent -> clear OidcConfigError. + from questdb.auth import _discovery + base = { + 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': 'https://idp.example.com/token', + 'acl.oidc.device.authorization.endpoint': + 'https://idp.example.com/device'} + + def from_settings(settings): + with mock.patch.object(_discovery, 'fetch_settings', + return_value=settings): + return OidcDeviceAuth.from_questdb( + 'https://qdb.example.com:9000', renderer=Renderer()) + + auth = from_settings({**base, 'acl.oidc.scope': ['openid', 'groups'], + 'acl.oidc.audience': {'x': 1}}) + self.assertEqual(auth.config.scope, 'openid') # non-string -> default + self.assertIsNone(auth.config.audience) # non-string -> dropped + self.assertTrue(auth.cache_key) # crash site now safe + # A non-string client.id reads as absent -> clear typed error. + with self.assertRaises(OidcConfigError): + from_settings({**base, 'acl.oidc.client.id': 12345}) + + def test_non_string_idp_discovery_values_do_not_crash(self): + # The IdP .well-known discovery document is untrusted too: a non-string + # endpoint / issuer (a JSON number/list from a buggy or hostile IdP) + # must read as absent -> a clear OidcConfigError, not a bare + # AttributeError from safe_urlparse later. See resolve_config discovery. + from questdb.auth import _discovery + settings = {'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb'} + + def from_discovery(well_known, **kw): + with mock.patch.object(_discovery, 'fetch_settings', + return_value=settings), \ + mock.patch.object( + _discovery, 'discover_device_endpoint_from_idp', + return_value=well_known): + return OidcDeviceAuth.from_questdb( + 'https://qdb.example.com:9000', renderer=Renderer(), **kw) + + # Non-string token / device endpoint -> absent -> clear typed error. + with self.assertRaises(OidcConfigError): + from_discovery( + {'device_authorization_endpoint': 'https://idp.example.com/device', + 'token_endpoint': 12345}, + issuer='https://idp.example.com') + with self.assertRaises(OidcConfigError): + from_discovery( + {'device_authorization_endpoint': ['nope'], + 'token_endpoint': 'https://idp.example.com/token'}, + issuer='https://idp.example.com') + # A non-string discovered issuer is dropped (no pin); valid endpoints + # still resolve and the cache key builds (the former crash site). + auth = from_discovery( + {'device_authorization_endpoint': 'https://idp.example.com/device', + 'token_endpoint': 'https://idp.example.com/token', + 'issuer': ['not', 'a', 'string']}, + discovery_url='https://idp.example.com/.well-known/openid-configuration') + self.assertIsNone(auth.config.issuer) + self.assertTrue(auth.cache_key) + def test_resolve_endpoint_relative_path_without_host_is_none(self): # A path-only endpoint with no acl.oidc.host can't be resolved; it must # be treated as absent (None) so resolution fails with a clear "could From 63a3f7aa53fd2c55168e196ddabac335036e6360 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sun, 21 Jun 2026 21:04:54 +0100 Subject: [PATCH 031/104] fix: keep device-flow poll alive through transient IdP errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The RFC 8628 poll loop aborted the whole sign-in on any transient failure during polling — a dropped connection, DNS blip, or per-request timeout (OidcNetworkError), an HTML 502/503/504 from a proxy/LB in front of the IdP (a bare OidcError from post_form), or a 5xx/429 carrying a JSON body (which hit the terminal error branch). Because the device flow targets flaky remote kernels and the failure lands after the user has already authorized in the browser, a single blip discarded a completed sign-in and forced a full restart. Treat these as transient and keep polling until the device-code deadline (RFC 8628 section 3.4): wrap the per-poll request in try/except OidcError -> continue, and continue on a 5xx/429 status, backing off the interval on a 429 rate-limit. Success, authorization_pending, slow_down, expired_token, and genuine OAuth rejections are unchanged; the deadline still bounds the total wait. Add regression tests covering a transient network error mid-poll and a 503 -> 429 -> 200 sequence (both fail against the pre-fix code). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 36 ++++++++++++++++++++++++++------- test/test_auth.py | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 7 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 71060b17..41a07ef7 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -659,13 +659,26 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: # interval still shouldn't overshoot a short-lived code. self._sleep(min(interval, remaining)) - status, body = self._idp_post( - self.config.token_endpoint, - { - 'grant_type': DEVICE_CODE_GRANT, - 'device_code': device_code, - 'client_id': self.config.client_id, - }) + try: + status, body = self._idp_post( + self.config.token_endpoint, + { + 'grant_type': DEVICE_CODE_GRANT, + 'device_code': device_code, + 'client_id': self.config.client_id, + }) + except OidcError: + # Transient failure mid-poll, not a terminal OAuth decision: a + # dropped connection / DNS blip / per-request timeout + # (OidcNetworkError), or a non-2xx response with a non-JSON body + # such as an HTML 502/503/504 from a proxy or load balancer in + # front of the IdP (a bare OidcError from post_form). The user + # may already have authorized in the browser, and RFC 8628 §3.4 + # expects polling to continue until the device code expires, so + # poll again instead of discarding the in-progress sign-in. The + # deadline check at the top of the loop bounds the total wait; a + # genuine rejection arrives as a JSON error body (handled below). + continue if status == 200: # A 200 is the RFC 6749 §5.1 token response: the grant @@ -686,6 +699,15 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: 'token this server requires.') raise self._missing_required_token_error() + # A 5xx or 429 that did carry a JSON body is also transient (a + # server-side error or a rate-limit) rather than a terminal OAuth + # rejection: back off on a rate-limit and keep polling until the + # deadline, matching the connection-failure handling above. + if status >= 500 or status == 429: + if status == 429: + interval = min(_MAX_POLL_INTERVAL, interval + 5) + continue + error = body.get('error') if error == 'authorization_pending': continue diff --git a/test/test_auth.py b/test/test_auth.py index 56ec2e8c..6d604e08 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -369,6 +369,46 @@ def test_slow_down_backs_off(self): # interval starts at 5, +5 after slow_down. self.assertEqual(self._clock.sleeps, [5, 10]) + def test_transient_network_error_during_poll_keeps_polling(self): + # A dropped connection / DNS blip / timeout on a single poll must not + # abort a sign-in the user may already have completed in the browser: + # the loop keeps polling until the deadline (RFC 8628 §3.4). M1. + self.state.token_script = [(200, None)] # success once actually polled + auth = self.make_auth() + real_idp_post = auth._idp_post + token_polls = {'n': 0} + + def flaky(url, form): + # Fail only the first poll of the token endpoint; pass the device- + # code request and later polls through to the real transport. + if url == auth.config.token_endpoint: + token_polls['n'] += 1 + if token_polls['n'] == 1: + raise OidcNetworkError('connection reset mid-poll') + return real_idp_post(url, form) + + auth._idp_post = flaky + self.assertEqual(auth.token(), ID_TOKEN) + # First poll raised (transient, retried); second poll reached the IdP. + self.assertEqual(token_polls['n'], 2) + self.assertEqual(len(self.state.token_requests), 1) + self.assertEqual(self._clock.sleeps, [5, 5]) + + def test_transient_5xx_and_429_during_poll_keep_polling(self): + # A 5xx server error or a 429 rate-limit (even carrying a JSON body) is + # transient, not a terminal OAuth rejection: keep polling, backing off + # on the rate-limit. M1. + self.state.token_script = [ + (503, {'error': 'server_error'}), + (429, {'error': 'slow_down'}), + (200, None), + ] + auth = self.make_auth() + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(len(self.state.token_requests), 3) + # 503 polled at the base interval; 429 bumps the interval by 5. + self.assertEqual(self._clock.sleeps, [5, 5, 10]) + def test_timeout_when_never_authorized(self): self.state.device_response = { 'device_code': 'DEV-CODE', 'user_code': 'X', From 954f8a87499279dced16cbfc7675f83f60d4a79a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sun, 21 Jun 2026 21:09:35 +0100 Subject: [PATCH 032/104] fix: map malformed IdP discovery / /exec payloads to OidcError Two server-payload paths escaped the package's typed-error contract with a raw exception instead of an OidcError: * A valid-JSON-but-not-an-object IdP discovery document (a list/null/ number/string from a captive portal, a misconfigured proxy, or a hostile IdP) reached resolve_config's doc.get(...) calls and raised a bare AttributeError. discover_device_endpoint_from_idp now coerces a non-dict document to {} so resolution fails with the clear 'could not resolve the ... endpoint' OidcConfigError, mirroring settings_config. * A /exec column descriptor with a non-hashable name (a JSON list/object) plus a TIMESTAMP/DATE type raised 'TypeError: unhashable type' from 'name in df.columns' during timestamp coercion. _exec_json_to_df now rejects a non-string column name in its columns guard, and also catches TypeError (not only ValueError) from the DataFrame constructor. Both surface as OidcError now; add regression tests (each fails against the pre-fix code with the raw AttributeError / TypeError). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 11 ++++++++++- src/questdb/auth/_questdb.py | 16 +++++++++++----- test/test_auth.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 4fea20c5..21e3c8db 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -319,7 +319,16 @@ def discover_device_endpoint_from_idp( 'Cannot discover the IdP device-authorization endpoint: no issuer ' 'or discovery_url was given. Pass issuer=... (or ' 'device_authorization_endpoint=... to skip discovery).') - return get_json(url, ctx=ctx, insecure=insecure, timeout=timeout) + doc = get_json(url, ctx=ctx, insecure=insecure, timeout=timeout) + # get_json guarantees valid JSON, not a JSON *object*. A discovery document + # that is valid-JSON-but-not-a-dict (a list/null/number/string from a + # captive portal, a misconfigured proxy, or a hostile IdP) must not reach + # resolve_config's doc.get(...) calls as a raw object, where it would escape + # the package's typed-error contract with a bare AttributeError. Coerce it + # to empty so resolution fails with the clear "could not resolve the ... + # endpoint" OidcConfigError instead — mirroring settings_config, which + # applies the same guard to the QuestDB /settings response. + return doc if isinstance(doc, dict) else {} def resolve_config( diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py index 98c6c72a..a2186442 100644 --- a/src/questdb/auth/_questdb.py +++ b/src/questdb/auth/_questdb.py @@ -72,11 +72,15 @@ def _import_pandas(): def _exec_json_to_df(data: Dict[str, Any], pandas): columns = data.get('columns') or [] # /exec returns a list of {"name", "type"} column descriptors. A malformed - # response (a non-list, or entries that aren't objects) must surface as a - # clean OidcError, not an AttributeError from .get() escaping the package's - # typed-error contract. + # response — a non-list, entries that aren't objects, or a non-string name — + # must surface as a clean OidcError, not a raw AttributeError from .get(), + # nor a TypeError from `name in df.columns` below when a name is + # non-hashable (a JSON list/object), escaping the package's typed-error + # contract. A real QuestDB column name is always a string. if not isinstance(columns, list) or not all( - isinstance(c, dict) for c in columns): + isinstance(c, dict) + and isinstance(c.get('name'), (str, type(None))) + for c in columns): raise OidcError( 'QuestDB /exec returned a malformed "columns" field; ' 'cannot build a DataFrame.') @@ -86,7 +90,9 @@ def _exec_json_to_df(data: Dict[str, Any], pandas): dataset = data.get('data') or [] try: df = pandas.DataFrame(dataset, columns=names or None) - except ValueError as e: + except (ValueError, TypeError) as e: + # TypeError too: a hostile/malformed dataset shape can make the pandas + # constructor raise it (not only ValueError); keep it within OidcError. raise OidcError( f'Unexpected shape in QuestDB /exec response: {e}') from e for col in columns: diff --git a/test/test_auth.py b/test/test_auth.py index 6d604e08..ca35d723 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -924,6 +924,22 @@ def test_missing_device_endpoint_raises(self): OidcDeviceAuth.from_questdb(self.base, issuer=self.base, insecure=True) + def test_non_dict_well_known_doc_raises_config_error(self): + # M2: an IdP discovery document that is valid JSON but not an object + # (a list/null/number/string from a captive portal, a misconfigured + # proxy, or a hostile IdP) must surface as a typed OidcConfigError, not + # a raw AttributeError from doc.get(...). issuer= is pinned so the + # fallback is allowed; the doc's shape is the error under test. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': self.base + '/token', + }} + self.state.well_known = [] # valid JSON, but not an object + with self.assertRaises(OidcConfigError): + OidcDeviceAuth.from_questdb(self.base, issuer=self.base, + insecure=True) + def test_malformed_endpoint_port_raises_config_error(self): # /settings advertising a non-integer port in an endpoint must raise # OidcConfigError (the typed contract), not a bare ValueError that @@ -1235,6 +1251,20 @@ def test_sql_non_dict_columns_raises_oidc_error(self): qdb.sql('SELECT 1') self.assertNotIsInstance(cm.exception, OidcAuthError) + def test_sql_non_string_column_name_raises_oidc_error(self): + # M2: a column descriptor with a non-hashable name (a JSON list/object) + # and a TIMESTAMP/DATE type must raise a clean OidcError, not a raw + # TypeError ("unhashable type") from `name in df.columns` during the + # timestamp coercion. + qdb = self._connected() + self.state.exec_response = { + 'columns': [{'name': ['evil'], 'type': 'TIMESTAMP'}, + {'name': 'b', 'type': 'LONG'}], + 'dataset': [['2021-01-01T00:00:00.000000Z', 2]]} + with self.assertRaises(OidcError) as cm: + qdb.sql('SELECT 1') + self.assertNotIsInstance(cm.exception, OidcAuthError) + class TestConcurrency(AuthTestBase): def test_valid_cached_token_does_not_block_during_signin(self): From 6e2970bc295399f90920c44b2d4094fae7f50acd Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sun, 21 Jun 2026 23:18:43 +0100 Subject: [PATCH 033/104] fix: sanitize untrusted device fields on the Jupyter prompt path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _strip_control (control / bidi / zero-width removal) was applied only on the terminal renderer. The Jupyter renderer — the module's primary surface — relied on html.escape alone, which neutralizes markup but NOT a U+202E bidi override or zero-width chars. A hostile or MITM'd IdP could put such chars in user_code, the verification URI, the JWT-derived identity, or error_description to visually spoof the sign-in prompt in the notebook DOM (the exact attack _strip_control exists to prevent). Route every untrusted Jupyter field through _strip_control before html.escape: factor the shared header/link/code into _prompt_head() (so on_prompt and _render_with_status can't diverge), and strip identity in on_success and the message in on_failure. Also complete _CONTROL_CHARS with the format/bidi code points it implied but missed (U+00AD, U+061C, U+115F, U+180E, U+2060-2064, U+FFF9-FFFB) and correct the _strip_control docstring claim that html-escaping suffices for Jupyter. Add a Jupyter-path regression test and extend the strip test to the new code points (both fail against the pre-fix code; the former shows U+202E reaching the rendered DOM). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_render.py | 73 ++++++++++++++++++++++--------------- test/test_auth.py | 38 ++++++++++++++++++- 2 files changed, 80 insertions(+), 31 deletions(-) diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 86dd2338..283153b7 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -128,13 +128,17 @@ def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: f'rel="noopener noreferrer">{label}') -# C0/C1 control chars (incl. ESC, which drives ANSI escape sequences) plus the -# Unicode bidi-control, zero-width and line/paragraph-separator ranges. All can -# spoof a terminal prompt: U+202E (RIGHT-TO-LEFT OVERRIDE) reverses displayed -# text to disguise a URL's host; U+2028/U+2029 inject fake lines; zero-width -# chars hide content. Stripped from untrusted device-response fields. +# C0/C1 control chars (incl. ESC, which drives ANSI escape sequences), the +# Unicode bidi controls, the zero-width / invisible-format chars, the +# line/paragraph separators and the interlinear-annotation controls. All can +# spoof a prompt: U+202E (RIGHT-TO-LEFT OVERRIDE) reverses displayed text to +# disguise a URL's host; U+2028/U+2029 inject fake lines; zero-width / invisible +# chars hide or join content. Stripped from untrusted device-response fields on +# BOTH the terminal and the Jupyter path (html.escape neutralizes markup, not +# these). Covers the dangerous Unicode Cc/Cf code points for our inputs. _CONTROL_CHARS = re.compile( - r'[\x00-\x1f\x7f-\x9f\u200b-\u200f\u2028-\u202e\u2066-\u2069\ufeff]') + r'[\x00-\x1f\x7f-\x9f\u00ad\u061c\u115f\u180e\u200b-\u200f' + r'\u2028-\u202e\u2060-\u2064\u2066-\u2069\ufeff\ufff9-\ufffb]') def _strip_control(text: Optional[str]) -> str: @@ -147,8 +151,9 @@ def _strip_control(text: Optional[str]) -> str: a hostile or MITM'd response inject ANSI escape sequences (C0/C1 control chars — cursor moves, screen clears) or Unicode bidi overrides / zero-width / line separators to spoof the prompt or hide the real sign-in URL (e.g. - U+202E visually reverses the displayed host). The Jupyter renderer - html-escapes its output; the plain-text path needs this. + U+202E visually reverses the displayed host). Needed on BOTH paths: the + plain-text terminal path (raw bytes to the TTY) and the Jupyter path — + ``html.escape`` neutralizes markup, not bidi/zero-width spoofing. """ if not text: return '' @@ -253,11 +258,25 @@ def _panel(self, body: str) -> str: 'padding:12px 16px;font-family:sans-serif;max-width:520px">' + body + '') - def on_prompt(self, resp: Dict[str, Any]) -> None: - self._resp = resp - uri = _verification_uri(resp) - code = html.escape(str(resp.get('user_code', ''))) + def _prompt_head(self): + """Header + sanitized verification link and user code. + + Shared by :meth:`on_prompt` and :meth:`_render_with_status` so the + sanitization can't be applied to one path and forgotten on the other. + ``verification_uri`` / ``user_code`` / ``verification_uri_complete`` are + untrusted device-response fields: strip control / bidi / zero-width + chars (which ``html.escape`` does NOT remove) before rendering, so a + hostile or MITM'd response can't inject a U+202E bidi override or + zero-width chars to visually spoof the prompt in the notebook DOM. + ``_render_link`` additionally html-escapes and scheme-vets the URL. + Returns ``(body, uri, complete)`` — the sanitized URLs are handed back + so the QR target isn't re-derived (and re-sanitized). + """ + resp = self._resp + uri = _strip_control(_verification_uri(resp)) + code = html.escape(_strip_control(str(resp.get('user_code', '')))) complete = _verification_uri_complete(resp) + complete = _strip_control(complete) if complete else None body = [ '
' '🔐 Sign in to QuestDB
', @@ -270,6 +289,11 @@ def on_prompt(self, resp: Dict[str, Any]) -> None: '
' + _render_link( complete, text='Click here to authorize directly →') + '
') + return body, uri, complete + + def on_prompt(self, resp: Dict[str, Any]) -> None: + self._resp = resp + body, uri, complete = self._prompt_head() if self._qr: qr_target = _safe_link_url(complete) or _safe_link_url(uri) data_uri = _qr_data_uri(qr_target) if qr_target else None @@ -292,7 +316,9 @@ def on_waiting(self, seconds_left: float) -> None: color='#888') def on_success(self, identity: Optional[str], expires_in: float) -> None: - who = html.escape(identity) if identity else '' + # identity is derived from the (untrusted) JWT claims — strip control / + # bidi chars before html-escaping, as for the other rendered fields. + who = html.escape(_strip_control(identity)) if identity else '' mins = max(1, int(round(expires_in / 60))) suffix = f' as {who}' if who else '' self._render_with_status( @@ -300,25 +326,12 @@ def on_success(self, identity: Optional[str], expires_in: float) -> None: color='#2e7d32') def on_failure(self, message: str) -> None: - self._render_with_status('❌ ' + html.escape(message), color='#c62828') + # message may interpolate the IdP's (untrusted) error_description. + self._render_with_status( + '❌ ' + html.escape(_strip_control(message)), color='#c62828') def _render_with_status(self, status_html: str, color: str) -> None: - resp = self._resp - uri = _verification_uri(resp) - code = html.escape(str(resp.get('user_code', ''))) - complete = _verification_uri_complete(resp) - body = [ - '
' - '🔐 Sign in to QuestDB
', - f'
Open {_render_link(uri)} and enter code:
', - f'
{code}
', - ] - if _safe_link_url(complete): - body.append( - '
' + _render_link( - complete, text='Click here to authorize directly →') - + '
') + body, _uri, _complete = self._prompt_head() body.append( f'
{status_html}
') self._display(self._panel(''.join(body))) diff --git a/test/test_auth.py b/test/test_auth.py index ca35d723..17553051 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2185,6 +2185,40 @@ def _display(self, html_str): # avoid importing IPython self.assertIn(' Date: Sun, 21 Jun 2026 23:36:44 +0100 Subject: [PATCH 034/104] fix: bind discovery_url pin to the IdP origin; close auth test gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M4: discovery_url= is documented as an out-of-band IdP pin, but when it was supplied without issuer=, resolve_config trusted the discovery document's self-declared issuer as the anchor that validate_endpoint_ origins checks the endpoints against. A hostile/confused/multi-tenant discovery host (or one returning no/absent issuer) could declare attacker endpoints all on one origin and pass the co-location + issuer-origin checks vacuously, redirecting the device-code / refresh-token POSTs. Anchor to the caller-pinned discovery_url itself instead: require the discovered credential endpoints to share its origin (OIDC Discovery §4.3 / RFC 8414 §3). This also closes the absent/non-string-issuer variant, and keeps the legitimate case (endpoints on the discovery origin, issuer dropped) working. M5: close auth test-coverage gaps flagged in review: * 401/403 -> OidcAuthError mapping in QuestDB.sql now has a pandas-independent test (new TestRestAdapterAuthErrors, registered in test.py) so it runs on every CI leg, not only where pandas is installed (the status check precedes any DataFrame build); adds the previously-missing 403 case. * test_concurrent_signin_prompts_only_once now asserts both threads finished (is_alive() == False) so a deadlock regression fails loudly instead of leaking a hung thread and passing on a stale result. * adds coverage for the IdP returning error=expired_token during the poll (distinct from the local deadline) and for a rotated refresh token being stored. Adds a regression test for the M4 fix (off-origin endpoints via a pinned discovery_url are refused; fails against the pre-fix code). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 30 +++++++++++ test/test.py | 1 + test/test_auth.py | 93 ++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 21e3c8db..de95b58f 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -492,6 +492,36 @@ def resolve_config( authorization_endpoint = ( authorization_endpoint or _str_setting(doc.get('authorization_endpoint'))) + # OIDC Discovery §4.3 / RFC 8414 §3: when the IdP is pinned ONLY by + # discovery_url (no out-of-band issuer=), the document's self-declared + # issuer would otherwise be the trust anchor validate_endpoint_origins + # (in OidcDeviceAuth.__init__) checks the endpoints against — but that + # issuer comes from the same (possibly hostile, confused, or + # multi-tenant) document, so the check is vacuous, and an absent or + # non-string issuer makes it vacuous too (a document declaring attacker + # endpoints all on one origin would pass co-location trivially). Anchor + # instead to the caller-pinned discovery_url itself: require the + # credential endpoints to live on its origin, so a document can't + # redirect the device-code / refresh-token POSTs to an attacker origin. + # Origin-level, matching validate_endpoint_origins; pass issuer= and the + # endpoints explicitly if your IdP serves discovery and tokens from + # different origins. + if discovery_url and not issuer: + discovery_origin = _normalized_origin(discovery_url) + for label, url in ( + ('token endpoint', token_endpoint), + ('device-authorization endpoint', + device_authorization_endpoint)): + if url and _normalized_origin(url) != discovery_origin: + raise OidcConfigError( + f'The OIDC {label} ({url!r}) discovered via the pinned ' + f'discovery_url ({discovery_url!r}) is on a different ' + 'origin; refusing to let a discovery document redirect ' + 'credentials off the pinned IdP origin (OIDC Discovery ' + '§4.3). Pin the IdP with issuer="https://your-idp" and ' + 'pass token_endpoint=/device_authorization_endpoint= ' + 'explicitly if it serves discovery and tokens from ' + 'different origins.') issuer = issuer or _str_setting(doc.get('issuer')) if not token_endpoint: diff --git a/test/test.py b/test/test.py index 04a2dcd3..054efe5e 100755 --- a/test/test.py +++ b/test/test.py @@ -42,6 +42,7 @@ TestDiscovery, TestInsecureSettingsGuard, TestRestAdapter, + TestRestAdapterAuthErrors, TestAdapters, TestConcurrency, TestConfigHelpers, diff --git a/test/test_auth.py b/test/test_auth.py index 17553051..336ef6a1 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -420,6 +420,16 @@ def test_timeout_when_never_authorized(self): with self.assertRaises(OidcTimeoutError): auth.token() + def test_idp_expired_token_error_raises_timeout(self): + # The token endpoint can itself answer a poll with error=expired_token + # (RFC 8628) — distinct from the local-deadline timeout. It must surface + # as OidcTimeoutError carrying that error, not loop or mis-classify it. + self.state.token_script = [(400, {'error': 'expired_token'})] + auth = self.make_auth() + with self.assertRaises(OidcTimeoutError) as cm: + auth.token() + self.assertEqual(cm.exception.error, 'expired_token') + def test_nonpositive_expires_in_still_polls(self): # A non-positive expires_in in the device-auth response must be treated # as unknown, not as "already expired" — otherwise the flow times out @@ -733,6 +743,20 @@ def test_refresh_token_preserved_when_not_rotated(self): auth.token() self.assertEqual(auth._tokens.refresh_token, 'REFRESH-1') + def test_rotated_refresh_token_is_stored(self): + # When the IdP DOES rotate the refresh token, the new one must replace + # the old in the cached token set — else an IdP with one-time-use + # refresh tokens breaks on the NEXT refresh. + auth = self.make_auth() + self._seed_expired(auth) + self.state.refresh_response = (200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'refresh_token': 'REFRESH-2', # rotated + 'token_type': 'Bearer', 'expires_in': 3600}) + auth.token() + self.assertEqual(auth._tokens.refresh_token, 'REFRESH-2') + self.assertEqual(self.state.device_requests, 0) # no re-prompt + def test_refresh_without_id_token_falls_back_to_device_flow(self): # groups_in_token=True but the IdP's refresh omits the id_token: the # refresh is unusable, so fall back to the interactive flow rather than @@ -904,6 +928,30 @@ def test_device_fallback_with_discovery_url_is_accepted(self): self.assertEqual(auth.config.device_authorization_endpoint, self.base + '/device') + def test_discovery_url_rejects_off_origin_issuer_in_doc(self): + # M4: discovery_url= is advertised as an out-of-band pin, but the doc it + # points to could declare an attacker issuer AND endpoints all on one + # (attacker) origin — which passes co-location / issuer-origin vacuously. + # The discovered issuer must share the pinned discovery_url origin (OIDC + # Discovery §4.3), else refuse. /settings advertises NO endpoints, so + # both come from the (hostile) doc — the exact gap the fix closes. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + }} + self.state.well_known = { + 'issuer': 'https://attacker.example.net', + 'token_endpoint': 'https://attacker.example.net/token', + 'device_authorization_endpoint': + 'https://attacker.example.net/device', + } + with self.assertRaises(OidcConfigError) as cm: + OidcDeviceAuth.from_questdb( + self.base, + discovery_url=self.base + '/.well-known/openid-configuration', + insecure=True) + self.assertIn('origin', str(cm.exception).lower()) + def test_oidc_disabled_raises(self): self.state.settings = {'config': {'acl.oidc.enabled': False}} with self.assertRaises(OidcConfigError): @@ -1266,6 +1314,47 @@ def test_sql_non_string_column_name_raises_oidc_error(self): self.assertNotIsInstance(cm.exception, OidcAuthError) +class TestRestAdapterAuthErrors(AuthTestBase): + """QuestDB.sql maps 401/403 to OidcAuthError BEFORE it builds a DataFrame, + so the mapping is testable without a real pandas. Kept out of the + pandas-gated TestRestAdapter so this security-relevant mapping runs on EVERY + CI leg, not just the ones where pandas is installed. M5.""" + + def _connected(self): + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.scope': 'openid groups', + 'acl.oidc.groups.encoded.in.token': True, + 'acl.oidc.token.endpoint': self.base + '/token', + 'acl.oidc.device.authorization.endpoint': self.base + '/device', + }} + self.state.expected_bearer = ID_TOKEN + return connect(self.base, insecure=True, renderer=Renderer(), + interactive=True, _clock=FakeClock()) + + @staticmethod + def _stub_pandas(): + # sql() reaches the 401/403 check before it touches pandas, so a bare + # stub module is enough to exercise the mapping without the real + # (possibly absent) dependency. + return mock.patch.dict( + sys.modules, {'pandas': types.ModuleType('pandas')}) + + def test_sql_401_maps_to_auth_error_without_pandas(self): + qdb = self._connected() + self.state.expected_bearer = 'something-else' # force 401 + with self._stub_pandas(), self.assertRaises(OidcAuthError): + qdb.sql('SELECT 1') + + def test_sql_403_maps_to_auth_error_without_pandas(self): + qdb = self._connected() + self.state.exec_status = 403 # bearer matches; server forbids + self.state.exec_response = {'error': 'forbidden'} + with self._stub_pandas(), self.assertRaises(OidcAuthError): + qdb.sql('SELECT 1') + + class TestConcurrency(AuthTestBase): def test_valid_cached_token_does_not_block_during_signin(self): # A caller with a valid cached token must NOT block behind another @@ -1318,6 +1407,10 @@ def call(name): release.set() # let t1 finish signing in t1.join(5) t2.join(5) + # Fail loudly on a deadlock regression: a hung thread would otherwise + # leak and let the assertions below pass on a stale/half-filled dict. + self.assertFalse(t1.is_alive(), 'sign-in thread deadlocked') + self.assertFalse(t2.is_alive(), 'waiter thread deadlocked') self.assertEqual(results.get('a'), ID_TOKEN) self.assertEqual(results.get('b'), ID_TOKEN) self.assertEqual(self.state.device_requests, 1) # no second prompt From 20bf2cd7ba9f15f270daa30607afc6a93bd2fc4d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 22 Jun 2026 11:28:12 +0100 Subject: [PATCH 035/104] fix: classify IdP token-endpoint errors transient vs terminal The device-flow poll loop and the silent refresh both POST to the IdP token endpoint but disagreed on which failures are transient, causing two user-visible bugs: - A non-JSON 4xx during polling (a WAF/proxy HTML error page, or a non-conformant IdP) was treated as transient, so the flow polled until the device code expired and reported a misleading "code expired" instead of failing fast (M1). - A transient 5xx/429 during a silent refresh tore the session down and re-ran the interactive sign-in -- hard-failing as OidcInteractionRequired in non-interactive pools/CI -- even though the refresh token was still valid and a retry would succeed (M2). Preserve the HTTP status on the error (OidcError.status, set by post_form for a non-JSON body) and classify both paths the same way: 4xx => terminal, 5xx/429/network => transient. Add regression tests for both paths plus the status-propagation contract they rely on. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 87 ++++++++++++++++++++++++++++++------- src/questdb/auth/_errors.py | 9 ++++ src/questdb/auth/_http.py | 12 +++-- test/test_auth.py | 74 +++++++++++++++++++++++++++++++ 4 files changed, 164 insertions(+), 18 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 41a07ef7..376ca27d 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -113,6 +113,25 @@ def _identity_from_claims(claims: Dict[str, Any]) -> Optional[str]: return None +def _http_status_is_terminal_4xx(status: Optional[int]) -> bool: + """ + True for a client-error HTTP status that is a definitive rejection. + + A non-JSON response body carrying such a status (e.g. an HTML/plain ``403`` + from a WAF or reverse proxy in front of the IdP, or a non-conformant IdP) is + never a RFC-conformant ``authorization_pending`` / ``slow_down`` — those are + always JSON — so the device-flow poll must fail fast rather than keep + retrying to a misleading "code expired". ``429`` is excluded: it is a + rate-limit, handled as transient with back-off. + """ + return status is not None and 400 <= status < 500 and status != 429 + + +def _http_status_is_transient(status: Optional[int]) -> bool: + """True for a server-side (5xx) or rate-limit (429) status worth retrying.""" + return status is not None and (status >= 500 or status == 429) + + class OidcDeviceAuth: """ Acquire and refresh an OIDC token via the device authorization grant. @@ -551,14 +570,29 @@ def _idp_post(self, url: str, form: Dict[str, Any]): url, form, ctx=self._ctx, insecure=False, timeout=self._timeout) def _refresh(self, tokens: TokenSet) -> TokenSet: - status, body = self._idp_post( - self.config.token_endpoint, - { - 'grant_type': REFRESH_GRANT, - 'refresh_token': tokens.refresh_token, - 'client_id': self.config.client_id, - 'scope': self.config.scope, - }) + try: + status, body = self._idp_post( + self.config.token_endpoint, + { + 'grant_type': REFRESH_GRANT, + 'refresh_token': tokens.refresh_token, + 'client_id': self.config.client_id, + 'scope': self.config.scope, + }) + except OidcNetworkError: + # Already transient (socket drop / DNS / per-request timeout): + # propagate so _acquire keeps the still-valid refresh token and + # retries later instead of re-prompting. + raise + except OidcError as e: + # Non-JSON HTTP error body (e.g. an HTML 5xx from a proxy in front + # of the IdP). A 5xx / 429 is a transient hiccup — re-raise as a + # network error so _acquire keeps the refresh token; a 4xx is a + # genuine rejection, so let it fall through (as an OidcError) to a + # fresh interactive sign-in. + if _http_status_is_transient(getattr(e, 'status', None)): + raise OidcNetworkError(str(e)) from e + raise if status == 200: refreshed = self._tokenset_from_response(body) # Many IdPs do not rotate the refresh token; keep the old one. @@ -567,6 +601,16 @@ def _refresh(self, tokens: TokenSet) -> TokenSet: refreshed = replace( refreshed, refresh_token=tokens.refresh_token) return refreshed + # A transient IdP error (5xx / 429) during a silent refresh must not + # tear down the session: the refresh token is still valid, so surface it + # as a network error and let _acquire keep it and retry later — matching + # the poll loop, which also treats 5xx/429 as transient. Only a genuine + # rejection (an expired/revoked refresh token, a 4xx invalid_grant) + # falls through to a fresh interactive sign-in. + if _http_status_is_transient(status): + raise OidcNetworkError( + f'Token refresh hit a transient IdP error (HTTP {status}); ' + 'the refresh token is still valid — retry later.') raise OidcDeviceFlowError( f"Token refresh failed: {body.get('error', 'unknown error')}", error=body.get('error'), @@ -667,17 +711,30 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: 'device_code': device_code, 'client_id': self.config.client_id, }) - except OidcError: - # Transient failure mid-poll, not a terminal OAuth decision: a - # dropped connection / DNS blip / per-request timeout - # (OidcNetworkError), or a non-2xx response with a non-JSON body - # such as an HTML 502/503/504 from a proxy or load balancer in - # front of the IdP (a bare OidcError from post_form). The user + except OidcError as e: + # A non-JSON 4xx is a terminal rejection (e.g. an HTML/plain + # error page from a WAF or reverse proxy in front of the IdP, or + # a non-conformant IdP): a conformant OAuth error is JSON, so it + # can never be authorization_pending / slow_down. Fail fast + # instead of polling on to a misleading "code expired". + if _http_status_is_terminal_4xx(getattr(e, 'status', None)): + self._renderer.on_failure( + 'Sign-in failed: the identity provider rejected the ' + 'request.') + raise OidcDeviceFlowError( + f'Device flow failed: the IdP rejected the token ' + f'request ({e}).') from e + # Otherwise transient, not a terminal OAuth decision: a dropped + # connection / DNS blip / per-request timeout (OidcNetworkError), + # or a non-JSON 5xx/429 such as an HTML 502/503/504 from a proxy + # in front of the IdP (a bare OidcError from post_form). The user # may already have authorized in the browser, and RFC 8628 §3.4 # expects polling to continue until the device code expires, so # poll again instead of discarding the in-progress sign-in. The # deadline check at the top of the loop bounds the total wait; a - # genuine rejection arrives as a JSON error body (handled below). + # genuine JSON rejection arrives as a JSON error body (below). + if getattr(e, 'status', None) == 429: + interval = min(_MAX_POLL_INTERVAL, interval + 5) continue if status == 200: diff --git a/src/questdb/auth/_errors.py b/src/questdb/auth/_errors.py index 7262f0cc..b4b7f55f 100644 --- a/src/questdb/auth/_errors.py +++ b/src/questdb/auth/_errors.py @@ -32,6 +32,15 @@ class OidcError(Exception): """Base class for every error raised by :mod:`questdb.auth`.""" + def __init__(self, *args, status: Optional[int] = None): + super().__init__(*args) + # HTTP status that produced this error, when it originated from a + # non-JSON HTTP response (else None). Lets the device-flow poll loop and + # the silent refresh tell a terminal 4xx rejection (e.g. a WAF/proxy + # error page) from a transient 5xx/429/network blip even when the body + # was not a conformant JSON OAuth error. + self.status = status + class OidcConfigError(OidcError): """ diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 3e691159..f43afeca 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -290,9 +290,15 @@ def post_form( # ValueError, so catch it explicitly to keep the typed contract. if resp.ok: raise OidcError( - f'Expected JSON from {url}, got: {resp.text()[:200]}') - # Non-JSON error body: surface the status + text. - raise OidcError(f'HTTP {resp.status} from {url}: {resp.text()[:200]}') + f'Expected JSON from {url}, got: {resp.text()[:200]}', + status=resp.status) + # Non-JSON error body: surface the status + text. Attach the HTTP status + # so callers (the device-flow poll loop / silent refresh) can tell a + # terminal 4xx rejection from a transient 5xx/429 even though the body + # was not a conformant JSON OAuth error. + raise OidcError( + f'HTTP {resp.status} from {url}: {resp.text()[:200]}', + status=resp.status) if not isinstance(parsed, dict): raise OidcError(f'Unexpected JSON shape from {url}: {parsed!r}') return resp.status, parsed diff --git a/test/test_auth.py b/test/test_auth.py index 336ef6a1..b9a5588b 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -409,6 +409,27 @@ def test_transient_5xx_and_429_during_poll_keep_polling(self): # 503 polled at the base interval; 429 bumps the interval by 5. self.assertEqual(self._clock.sleeps, [5, 5, 10]) + def test_non_json_4xx_during_poll_is_terminal(self): + # A non-JSON 4xx during polling (an HTML/plain error page from a WAF or + # reverse proxy in front of the IdP, or a non-conformant IdP) is a + # terminal rejection: a conformant OAuth error is JSON, so it can't be + # authorization_pending / slow_down. Fail fast with a device-flow error + # instead of polling on to a misleading "code expired". M1. + auth = self.make_auth() + with _raw_response_server( + 403, 'text/html', b'denied') as raw: + # Point only the poll (token) endpoint at the non-JSON 403; the + # device-code request still hits the JSON mock IdP. Set it post- + # construction so the (already-satisfied) co-location check isn't + # re-run against the throwaway origin. + auth.config.token_endpoint = raw + '/token' + with self.assertRaises(OidcDeviceFlowError) as cm: + auth.token() + # Terminal on the first poll: not a timeout, and it did not keep polling + # to the device-code deadline. + self.assertNotIsInstance(cm.exception, OidcTimeoutError) + self.assertLessEqual(len(self._clock.sleeps), 1) + def test_timeout_when_never_authorized(self): self.state.device_response = { 'device_code': 'DEV-CODE', 'user_code': 'X', @@ -821,6 +842,44 @@ def test_refresh_network_error_propagates_without_reprompt(self): self.assertIn('/token', str(cm.exception)) self.assertEqual(auth._tokens.refresh_token, 'REFRESH-1') + def test_refresh_transient_5xx_kept_for_retry(self): + # A transient IdP error (5xx) during a silent refresh must NOT tear the + # session down and re-prompt: the refresh token is still valid, so it is + # surfaced as a retryable OidcNetworkError and the cached token (with its + # refresh token) is kept for a later retry — matching the poll loop, + # which also treats 5xx/429 as transient. M2. + auth = self.make_auth() + self._seed_expired(auth) + self.state.refresh_response = (503, {'error': 'temporarily_unavailable'}) + with self.assertRaises(OidcNetworkError): + auth.token() + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual(self.state.device_requests, 0) # NOT re-prompted + self.assertEqual(auth._tokens.refresh_token, 'REFRESH-1') # kept + + def test_refresh_transient_429_kept_for_retry(self): + # Same as the 5xx case for a 429 rate-limit. M2. + auth = self.make_auth() + self._seed_expired(auth) + self.state.refresh_response = (429, {'error': 'slow_down'}) + with self.assertRaises(OidcNetworkError): + auth.token() + self.assertEqual(self.state.device_requests, 0) + self.assertEqual(auth._tokens.refresh_token, 'REFRESH-1') + + def test_refresh_transient_5xx_non_interactive_does_not_hard_fail(self): + # The worst case: in a non-interactive context (papermill / cron / CI) a + # transient refresh error must surface as a retryable OidcNetworkError, + # NOT escalate to OidcInteractionRequired — which a fall-through to the + # device flow would raise, hard-failing a session whose refresh token is + # still valid and would succeed on the next attempt. M2. + auth = self.make_auth(interactive=False) + self._seed_expired(auth) + self.state.refresh_response = (503, {'error': 'temporarily_unavailable'}) + with self.assertRaises(OidcNetworkError): + auth.token() + self.assertEqual(self.state.device_requests, 0) + class TestDiscovery(AuthTestBase): def test_from_questdb_reads_settings(self): @@ -2078,6 +2137,21 @@ def test_require_secure_policy(self): _require_secure('http://idp.example.com/x', insecure=False) _require_secure('http://idp.example.com/x', insecure=True) + def test_post_form_attaches_status_to_non_json_error(self): + # The device-flow poll loop and the silent refresh classify a non-JSON + # token-endpoint failure (4xx terminal vs 5xx/429 transient) by the HTTP + # status, so post_form must attach it to the raised OidcError. M1/M2. + from questdb.auth._http import post_form + with _raw_response_server(403, 'text/plain', b'forbidden') as raw: + with self.assertRaises(OidcError) as cm: + post_form(raw + '/token', {'grant_type': 'x'}) + self.assertEqual(cm.exception.status, 403) + # A non-JSON 5xx likewise carries its status (classified as transient). + with _raw_response_server(503, 'text/html', b'

bad gw

') as raw: + with self.assertRaises(OidcError) as cm: + post_form(raw + '/token', {'grant_type': 'x'}) + self.assertEqual(cm.exception.status, 503) + def test_insecure_does_not_downgrade_idp(self): # insecure=True must NOT permit plaintext to a non-loopback IdP: the # device code / refresh token must never traverse the network in clear. From dcd44bc3040b371b7150cd5fc87bbf8875a09181 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 22 Jun 2026 11:38:14 +0100 Subject: [PATCH 036/104] fix: keep the device-flow prompt visible on a non-UTF-8 terminal TerminalRenderer._write swallowed every write error, so on a stream that can't encode the prompt's emoji -- a legacy code-page Windows console, an ascii PYTHONIOENCODING, or a redirected stderr -- the UnicodeEncodeError discarded the whole prompt, including the verification URL and user code. The sign-in then polled invisibly and looked like a silent hang. Catch UnicodeEncodeError and retry with the stream's own encoding using errors='replace', so only the un-encodable glyphs degrade while the ASCII URL and code still reach the user. The "never raises" contract is kept via the outer swallow. Add a regression test driving the renderer at an ascii-only stream. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_render.py | 13 ++++++++++++- test/test_auth.py | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 283153b7..9263b556 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -205,7 +205,18 @@ def __init__(self, stream: Optional[TextIO] = None, qr: bool = False): def _write(self, text: str) -> None: try: - self._stream.write(text) + try: + self._stream.write(text) + except UnicodeEncodeError: + # The stream's encoding can't represent some characters (e.g. + # the emoji on a legacy code-page Windows console, an ``ascii`` + # PYTHONIOENCODING, or a redirected stderr). Degrade only those + # characters instead of letting the whole prompt — including the + # verification URL and user code — vanish, which would make the + # sign-in look like a silent hang. + enc = getattr(self._stream, 'encoding', None) or 'ascii' + self._stream.write( + text.encode(enc, 'replace').decode(enc, 'replace')) self._stream.flush() except Exception: pass diff --git a/test/test_auth.py b/test/test_auth.py index b9a5588b..1fc5cf0f 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2413,6 +2413,44 @@ def test_terminal_prompt_strips_control_chars(self): self.assertNotIn('\x1b', out) self.assertNotIn('\x07', out) + def test_terminal_prompt_survives_unencodable_stream(self): + # On a stream whose encoding can't represent the prompt's emoji (a + # legacy code-page Windows console, an `ascii` PYTHONIOENCODING, or a + # redirected stderr), the decorative glyphs must degrade but the + # verification URL and user code must STILL reach the user — not vanish + # into a silent hang. M3. + from questdb.auth._render import TerminalRenderer + + class _AsciiStream: + encoding = 'ascii' + + def __init__(self): + self.parts = [] + + def write(self, s): + s.encode(self.encoding) # raises UnicodeEncodeError, like a TTY + self.parts.append(s) + + def flush(self): + pass + + stream = _AsciiStream() + r = TerminalRenderer(stream=stream) + r.on_prompt({ + 'user_code': 'WDJB-MJHT', + 'verification_uri': 'https://idp.example.com/device', + }) + r.on_success('alice@example.com', 3600) + r.on_failure('access denied') + out = ''.join(stream.parts) + # The essential content survived (only the un-encodable glyphs were + # replaced); nothing was blackholed and no exception escaped. + self.assertIn('https://idp.example.com/device', out) + self.assertIn('WDJB-MJHT', out) + self.assertIn('alice@example.com', out) + self.assertIn('access denied', out) + out.encode('ascii') # the whole transcript is ascii-encodable + def test_strip_control_removes_bidi_and_zero_width(self): # Beyond C0/C1, untrusted device-response fields must have Unicode # bidi-override / zero-width / line-separator characters stripped before From 86cf83f25cebcbce2f5cf550c7daf57bcdfa4ae0 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 22 Jun 2026 11:52:01 +0100 Subject: [PATCH 037/104] fix: close issuer-path pin bypass via encoded/backslash/;params traversal _endpoint_path_under_issuer scanned the path for dot segments after a single percent-decode but did the containment check on the raw string, so three traversals slipped past the issuer-path pin and could steer the device code / refresh token to a different realm on a path-based IdP (Keycloak host/realms/{realm}): * double-encoded dots /realms/prod/%252e%252e/EVIL/token * backslash separator /realms/prod/..\EVIL/token * last-segment ;params /realms/prod/token;..%2f..%2fEVIL (urllib splits ;params off .path, so its dots were never scanned, and a server that unescapes twice or folds a backslash to '/' resolves the others to a different realm.) Compare fully-decoded segments instead of the raw string: a new _decode_path_segments unquotes until stable, folds backslash to '/', and splits on '/'; fold the last segment's ;params back in, reject any '.'/'..' segment, then do a segment-wise prefix check. Legitimate non-traversal escapes (/some%20path, ;jsessionid=abc) are still accepted. Extend the unit and end-to-end (resolve_config) tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 56 +++++++++++++++++++++++++++------- test/test_auth.py | 16 +++++++++- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index de95b58f..eeb839c2 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -176,6 +176,29 @@ def _settings_channel_is_plaintext(questdb_url: str) -> bool: parts.hostname) +def _decode_path_segments(path: str) -> list: + """ + Fully percent-decode a URL path and split it into ``/`` segments. + + Decoding is repeated until stable so a double/triple-encoded dot segment + (``%252e%252e`` -> ``%2e%2e`` -> ``..``), or an encoded slash (``%2f``) that + splits a segment, is unmasked — a server or reverse proxy may unescape more + than once before it normalizes. A backslash is treated as a separator, since + some proxies fold ``\\`` to ``/`` before routing. The returned segments are + what the containment check compares, never the raw string urllib puts on the + wire, so an encoding the server later undoes can't smuggle a ``..`` past the + scan. The loop is bounded (a real path needs 0-1 passes; more layers than a + server would itself decode can't resolve to a traversal anyway). + """ + decoded = path + for _ in range(10): # bounded; each pass peels one percent-encoding layer + nxt = urllib.parse.unquote(decoded) + if nxt == decoded: + break + decoded = nxt + return decoded.replace('\\', '/').split('/') + + def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: """ True if ``endpoint``'s path is the issuer's path or a sub-path of it. @@ -187,22 +210,33 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: IdP (Keycloak issuers are ``https://host/realms/{realm}``), which an origin-only check can't catch. - A ``.`` / ``..`` path segment is rejected outright: urllib puts the dotted - path on the wire verbatim, but the IdP (or a reverse proxy in front of it) - normalizes it, so ``/realms/prod/../attacker/token`` would satisfy a naive - prefix test yet resolve server-side to a *different* realm — defeating the - very isolation this check exists to provide. Percent-encoded dot segments - (``%2e``) are decoded before the segment scan, since a server may unescape - before normalizing; a legitimate endpoint path never contains dot segments. + The comparison is done on the fully *decoded* path segments, never the raw + string urllib sends. A ``.`` / ``..`` segment is rejected outright: urllib + puts the dotted path on the wire verbatim, but the IdP (or a reverse proxy + in front of it) normalizes it, so ``/realms/prod/../attacker/token`` would + satisfy a naive prefix test yet resolve server-side to a *different* realm — + defeating the very isolation this check exists to provide. Encoded dot + segments are unmasked first — including double-encoded (``%252e``) and + encoded slashes (``%2f``) a server may unescape more than once — a backslash + is treated as a separator, and the last segment's ``;params`` (which urllib + splits off ``.path``) is folded back in, so none of those can smuggle a + traversal past the segment scan. A legitimate endpoint path never contains + dot segments. """ base = (safe_urlparse(issuer)[0].path or '').rstrip('/') if not base: return True - ep = safe_urlparse(endpoint)[0].path or '' - decoded_segments = urllib.parse.unquote(ep).split('/') - if '.' in decoded_segments or '..' in decoded_segments: + base_segs = _decode_path_segments(base) + eparts = safe_urlparse(endpoint)[0] + # urllib splits the last segment's ;params off .path; fold it back so a + # traversal hidden there (…/token;..%2f..%2fEVIL) can't slip past the scan. + ep_path = eparts.path or '' + if eparts.params: + ep_path = f'{ep_path};{eparts.params}' + ep_segs = _decode_path_segments(ep_path) + if '.' in ep_segs or '..' in ep_segs: return False - return ep == base or ep.startswith(base + '/') + return ep_segs[:len(base_segs)] == base_segs def validate_endpoint_origins( diff --git a/test/test_auth.py b/test/test_auth.py index 1fc5cf0f..25c2ace3 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1255,7 +1255,10 @@ def test_issuer_path_scope_rejects_dot_segment_traversal(self): # _endpoint_path_under_issuer. kc = 'https://idp.example.com/realms' for ep in (kc + '/prod/../EVIL/protocol/openid-connect', - kc + '/prod/%2e%2e/EVIL/protocol/openid-connect'): + kc + '/prod/%2e%2e/EVIL/protocol/openid-connect', + # double-encoded: a server that unescapes twice resolves the + # '..' the old single-decode check missed (M4). + kc + '/prod/%252e%252e/EVIL/protocol/openid-connect'): evil = { 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', 'acl.oidc.token.endpoint': ep + '/token', @@ -2058,6 +2061,17 @@ def test_endpoint_path_under_issuer(self): self.assertFalse(under(iss + '/../EVIL/protocol/token', iss)) self.assertFalse(under(iss + '/%2e%2e/EVIL/token', iss)) self.assertFalse(under(iss + '/./token', iss)) + # Encodings/escapes the old decode-once-then-compare-raw check let + # through (M4): a server that unescapes more than once, folds a + # backslash to '/', or normalizes the last segment's ;params would + # resolve these to a DIFFERENT realm, so they must be rejected too. + self.assertFalse(under(iss + '/%252e%252e/EVIL/token', iss)) # 2x-enc + self.assertFalse(under(iss + '/..\\EVIL/token', iss)) # backslash + self.assertFalse(under(iss + '/token;..%2f..%2fEVIL', iss)) # ;params + # A legitimate sub-path with a (non-traversal) percent-escape or matrix + # param is still accepted — only dot traversal is rejected. + self.assertTrue(under(iss + '/some%20path/token', iss)) + self.assertTrue(under(iss + '/token;jsessionid=abc', iss)) class TestCacheKey(unittest.TestCase): From 0001e4d21938698d65fe1f2c1c517b897e5b7058 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 22 Jun 2026 12:26:27 +0100 Subject: [PATCH 038/104] fix: tighten device-auth, host/port handling, and refresh audience Four small robustness/ergonomics fixes surfaced in review: * _request_device_code: a 200 response missing device_code/user_code now raises a clear "non-conformant 200 response" error instead of the self-contradictory "Device authorization request failed (HTTP 200)". * _resolve_endpoint: a non-string acl.oidc.host (a JSON number/list from a buggy or hostile /settings) is dropped via _str_setting instead of being interpolated raw into the netloc (https://12345:9000/...); a non-numeric port is likewise dropped. * QuestDB.sender(): coerce the port kwarg to int (before the extension import, so it fails fast) so a string like "9000;tls_verify=unsafe_off" can't smuggle ILP conf parameters into the addr= string -- the same injection _require_host() already blocks for the host. * _refresh: re-send the audience on refresh (mirroring the device-auth request) so an IdP that scopes `aud` per request keeps it on the rotated token instead of minting one QuestDB rejects after a silent refresh; omitted when unconfigured. Each fix lands with a regression test. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 18 ++++++++++ src/questdb/auth/_discovery.py | 16 +++++++-- src/questdb/auth/_questdb.py | 18 ++++++++-- test/test_auth.py | 65 ++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 5 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 376ca27d..33d8d397 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -578,6 +578,13 @@ def _refresh(self, tokens: TokenSet) -> TokenSet: 'refresh_token': tokens.refresh_token, 'client_id': self.config.client_id, 'scope': self.config.scope, + # Re-send the audience on refresh too, mirroring the + # device-authorization request: some IdPs (e.g. Auth0) need + # it to keep the rotated access token's `aud`, and would + # otherwise mint a token QuestDB rejects only AFTER a silent + # refresh. IdPs that don't use it ignore the param; post_form + # drops it entirely when audience is None (not configured). + 'audience': self.config.audience, }) except OidcNetworkError: # Already transient (socket drop / DNS / per-request timeout): @@ -649,6 +656,17 @@ def _request_device_code(self) -> Dict[str, Any]: if status == 200 and body.get('device_code') and body.get('user_code'): return body error = body.get('error') + if status == 200: + # 200 but the success guard above failed: the response is missing + # device_code/user_code. That is a non-conformant body, not an + # HTTP-level failure — say so plainly rather than the contradictory + # "Device authorization request failed (HTTP 200)". + raise OidcDeviceFlowError( + 'The IdP returned a 200 device-authorization response that is ' + 'missing the required "device_code"/"user_code" fields; cannot ' + 'start the device flow.', + error=error, + error_description=body.get('error_description')) if status in (400, 404, 405) or error in ( 'invalid_client', 'unauthorized_client', 'unsupported_grant_type'): diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index eeb839c2..01010b5e 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -309,7 +309,12 @@ def _resolve_endpoint(value: Optional[str], cfg: Dict[str, Any]) -> Optional[str if value.startswith('http://') or value.startswith('https://'): return value if value.startswith('/'): - host = cfg.get(_K_HOST) + # _str_setting drops a non-string acl.oidc.host (a JSON number/list from + # a buggy or hostile /settings) so it can't be interpolated raw into the + # netloc — e.g. https://12345:9000/path — and instead reads as absent, + # mirroring how endpoint values above are coerced. (safe_urlparse would + # otherwise reject the bogus URL only incidentally, downstream.) + host = _str_setting(cfg.get(_K_HOST)) if not host: # A path-only endpoint with no acl.oidc.host to resolve it against # can't be turned into a URL. Treat it as absent (return None) so @@ -320,8 +325,15 @@ def _resolve_endpoint(value: Optional[str], cfg: Dict[str, Any]) -> Optional[str return None tls = _as_bool(cfg.get(_K_TLS_ENABLED), default=True) scheme = 'https' if tls else 'http' + # A usable port is an int or a digit string; anything else (a JSON + # list/object, a bool, or a non-numeric string) would corrupt the + # netloc, so drop it and resolve host-only. port = cfg.get(_K_PORT) - netloc = f'{host}:{port}' if port else str(host) + if isinstance(port, bool) or not ( + isinstance(port, int) + or (isinstance(port, str) and port.isdigit())): + port = None + netloc = f'{host}:{port}' if port else host return f'{scheme}://{netloc}{value}' return value diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py index a2186442..76a6979b 100644 --- a/src/questdb/auth/_questdb.py +++ b/src/questdb/auth/_questdb.py @@ -326,6 +326,21 @@ def sender(self, *, port: Optional[int] = None, The token is captured at creation time; create a new sender to pick up a refreshed token. """ + scheme = 'https' if self._parts.scheme == 'https' else 'http' + resolved_port = port or self._port or ( + 443 if scheme == 'https' else 9000) + # Coerce to int (before the heavy import, so bad input fails fast) so a + # stray non-integer port kwarg can't smuggle ILP conf parameters — e.g. + # "9000;tls_verify=unsafe_off" — into the addr= string via _ilp_addr, + # the same injection _require_host() blocks for the host. The + # URL-derived self._port is already an int. + try: + resolved_port = int(resolved_port) + except (TypeError, ValueError): + raise OidcConfigError( + f'Invalid port {resolved_port!r} for QuestDB.sender(); expected ' + 'an integer.') + try: from questdb.ingress import Sender except ImportError as e: @@ -334,9 +349,6 @@ def sender(self, *, port: Optional[int] = None, 'QuestDB.sender(). Install the full client wheel ' '(`pip install questdb`).') from e - scheme = 'https' if self._parts.scheme == 'https' else 'http' - resolved_port = port or self._port or ( - 443 if scheme == 'https' else 9000) conf = (f'{scheme}::addr=' f'{self._ilp_addr(self._require_host(), resolved_port)};') # Forward the private CA bundle (explicit ca_bundle=, else the diff --git a/test/test_auth.py b/test/test_auth.py index 25c2ace3..1a3afb96 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -183,6 +183,7 @@ def __init__(self): self.device_requests = 0 self.token_requests = [] self.refresh_requests = 0 + self.refresh_forms = [] self.exec_requests = [] @@ -269,6 +270,7 @@ def do_POST(self): grant = form.get('grant_type') if grant == 'refresh_token': self.state.refresh_requests += 1 + self.state.refresh_forms.append(form) status, body = self.state.refresh_response or ( 200, self._default_token_body()) self._send_json(status, body) @@ -430,6 +432,21 @@ def test_non_json_4xx_during_poll_is_terminal(self): self.assertNotIsInstance(cm.exception, OidcTimeoutError) self.assertLessEqual(len(self._clock.sleeps), 1) + def test_device_200_without_codes_is_rejected_clearly(self): + # A 200 device-authorization response missing device_code/user_code is + # a non-conformant body, not an HTTP failure: the error must say so + # plainly (NOT the self-contradictory "failed (HTTP 200)") and the flow + # must never start polling. + self.state.device_status = 200 + self.state.device_response = {'verification_uri': 'https://idp/device'} + auth = self.make_auth() + with self.assertRaises(OidcDeviceFlowError) as cm: + auth.token() + msg = str(cm.exception) + self.assertNotIn('HTTP 200', msg) + self.assertIn('device_code', msg) + self.assertEqual(self.state.token_requests, []) # never polled + def test_timeout_when_never_authorized(self): self.state.device_response = { 'device_code': 'DEV-CODE', 'user_code': 'X', @@ -867,6 +884,27 @@ def test_refresh_transient_429_kept_for_retry(self): self.assertEqual(self.state.device_requests, 0) self.assertEqual(auth._tokens.refresh_token, 'REFRESH-1') + def test_refresh_includes_audience_when_configured(self): + # The audience is re-sent on refresh (mirroring the device-auth + # request), so an IdP that scopes `aud` per request keeps it on the + # rotated token instead of minting one QuestDB rejects after a silent + # refresh. When no audience is configured the param is omitted. + auth = self.make_auth(audience='questdb-api') + self._seed_expired(auth) + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual( + self.state.refresh_forms[-1].get('audience'), 'questdb-api') + + # Without an audience, the refresh form carries no audience key. + _MEMORY_STORE.clear() + _MEMORY_GENERATION.clear() + self.state.refresh_forms.clear() + auth2 = self.make_auth() # no audience + self._seed_expired(auth2) + auth2.token() + self.assertNotIn('audience', self.state.refresh_forms[-1]) + def test_refresh_transient_5xx_non_interactive_does_not_hard_fail(self): # The worst case: in a non-interactive context (papermill / cron / CI) a # transient refresh error must surface as a retryable OidcNetworkError, @@ -1825,6 +1863,17 @@ def test_sender_missing_extension_raises(self): with self.assertRaises(ImportError): self._qdb().sender() + def test_sender_rejects_non_integer_port(self): + # A non-integer port kwarg must be rejected before it can be + # interpolated into the addr= conf string, where ";tls_verify= + # unsafe_off" would silently disable TLS verification — the same + # injection _require_host() blocks for the host. The coercion runs + # before the extension import, so this fails cleanly even without it. + qdb = self._qdb('https://db.example.com:9000') + for bad in ('9000;tls_verify=unsafe_off', 'notaport', ['9000']): + with self.assertRaises(OidcConfigError): + qdb.sender(port=bad) + class TestConfigHelpers(unittest.TestCase): def test_as_bool_variants(self): @@ -1937,6 +1986,22 @@ def test_resolve_endpoint_relative_path_without_host_is_none(self): self.assertIsNone( # port present but host missing -> still unresolved _resolve_endpoint('/as/token.oauth2', {'acl.oidc.port': 443})) + def test_resolve_endpoint_ignores_non_string_host(self): + # A non-string acl.oidc.host (a JSON number/list from a buggy or hostile + # /settings) must not be interpolated raw into the netloc (e.g. + # https://12345:9000/path); treat it as absent so a path-only endpoint + # reads as unresolvable, mirroring how endpoint values are coerced. + from questdb.auth._discovery import _resolve_endpoint + for bad_host in (12345, ['idp'], {'h': 'idp'}, True): + self.assertIsNone( + _resolve_endpoint('/as/token', {'acl.oidc.host': bad_host})) + # A non-numeric port is dropped rather than corrupting the netloc. + self.assertEqual( + _resolve_endpoint('/as/token', { + 'acl.oidc.host': 'idp', 'acl.oidc.tls.enabled': True, + 'acl.oidc.port': ['x']}), + 'https://idp/as/token') + def test_settings_config_nesting(self): from questdb.auth._discovery import settings_config self.assertEqual(settings_config({'config': {'a': 1}}), {'a': 1}) From 99c05ace3d3fa6034a2642810a1173afceb98172 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 22 Jun 2026 12:42:47 +0100 Subject: [PATCH 039/104] docs: condense the questdb.auth comments and docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new module's comments and docstrings were thorough but verbose (inline threat-model essays). Tighten them across all eight files — each long rationale becomes a one-to-two-line "risk + defense" statement — without losing any security/concurrency "why". Comments and docstrings only: code and every non-docstring string (exception messages, rendered prompt text, regexes, dict keys) are unchanged, verified by an AST-equivalence check against the prior revision (docstrings stripped) and the full test suite (152 pass). Net -177 lines of commentary. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/__init__.py | 20 +- src/questdb/auth/_cache.py | 49 ++--- src/questdb/auth/_device.py | 391 +++++++++++++++------------------ src/questdb/auth/_discovery.py | 291 ++++++++++-------------- src/questdb/auth/_errors.py | 37 ++-- src/questdb/auth/_http.py | 111 ++++------ src/questdb/auth/_questdb.py | 102 ++++----- src/questdb/auth/_render.py | 102 ++++----- 8 files changed, 463 insertions(+), 640 deletions(-) diff --git a/src/questdb/auth/__init__.py b/src/questdb/auth/__init__.py index cfacb6e8..a06acaf5 100644 --- a/src/questdb/auth/__init__.py +++ b/src/questdb/auth/__init__.py @@ -25,16 +25,12 @@ """ OIDC authentication helper for QuestDB (Jupyter-first). -Runs the OAuth 2.0 Device Authorization Grant (RFC 8628) entirely client-side, -obtains a token, and presents it to QuestDB over the auth paths it already -supports (HTTP ``Bearer`` / PG-wire ``_sso``). Designed for data scientists on -local **and remote** kernels (JupyterHub, SageMaker, Colab, VS Code-remote), -where the kernel has no browser: you authorize in any browser (laptop or -phone), the kernel only makes outbound calls to the IdP. +Runs the OAuth 2.0 Device Authorization Grant (RFC 8628) client-side and +presents the token to QuestDB (HTTP ``Bearer`` / PG-wire ``_sso``). Works on +browserless local and remote kernels (JupyterHub, SageMaker, Colab, +VS Code-remote): authorize in any browser, the kernel only calls the IdP. -Two ways to use it, depending on your needs: - -* **Just the token** — works with anything (PG-wire, HTTP, your own tooling):: +* **Just the token** — works with anything; no optional dependencies:: from questdb.auth import OidcDeviceAuth @@ -52,10 +48,8 @@ with qdb.sender() as sender: # ingestion (ILP/HTTP) ... -Only ``token()`` / ``headers()`` are needed for the bring-your-own-client path, -and they require no optional dependencies. ``pandas`` (for ``sql()``), -``sqlalchemy`` / ``psycopg`` (adapters), ``qrcode`` and ``IPython`` are imported -lazily, only when used. +Optional deps (``pandas``, ``sqlalchemy``/``psycopg``, ``qrcode``, ``IPython``) +are imported lazily, only when used. """ from ._device import OidcDeviceAuth diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index b3e0ff95..b8373fb6 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -39,14 +39,13 @@ @dataclass(frozen=True) class TokenSet: """ - A set of tokens obtained from the IdP, plus their expiry. + IdP tokens plus their expiry. - Immutable (``frozen``): the lock-free fast path in + ``frozen`` because the lock-free fast path in :class:`~questdb.auth._device.OidcDeviceAuth` reads a published ``TokenSet`` - without holding a lock, which is only safe because its fields never change - after construction. Derive a modified copy with :func:`dataclasses.replace` - rather than mutating in place. The three secret fields are kept out of - ``repr`` so a token can't leak into a log line or traceback. + without a lock, which is safe only if its fields never change; use + :func:`dataclasses.replace` for a modified copy. The secret fields are + excluded from ``repr`` so a token can't leak into a log or traceback. """ access_token: Optional[str] = field(default=None, repr=False) @@ -62,9 +61,8 @@ def is_valid(self, now: float, skew: float = DEFAULT_SKEW_SECONDS) -> bool: """True if the token is present and not within ``skew`` of expiry.""" if self.expires_at <= 0: return False - # Never let the early-refresh skew exceed half the token's own - # lifetime, so a short-lived (< 2*skew) token isn't reported expired - # the instant it is issued (which would refresh on every call). + # Cap skew at half the token lifetime, so a short-lived (< 2*skew) + # token isn't reported expired the instant it's issued. if self.issued_at: lifetime = self.expires_at - self.issued_at if lifetime > 0: @@ -85,14 +83,13 @@ def clear(self, key: str) -> None: # pragma: no cover raise NotImplementedError -# Module-global so that re-running a notebook cell (which constructs a fresh -# ``OidcDeviceAuth``) reuses the already-acquired token instead of re-prompting. +# Module-global so a re-run notebook cell (fresh ``OidcDeviceAuth``) reuses the +# acquired token instead of re-prompting. _MEMORY_STORE: Dict[str, TokenSet] = {} -# Per-key counter bumped on every clear(). store_if_current() uses it to drop a -# write from an acquisition that began before a concurrent clear() — including a -# clear() on a *different* OidcDeviceAuth that shares this process-global store, -# whose per-instance lock does not serialize against this one — so clear() can't -# be silently undone by an in-flight sign-in / refresh. +# Per-key counter bumped on every clear(); store_if_current() uses it to drop a +# write from an acquisition that began before a concurrent clear() — even a +# clear() on a different OidcDeviceAuth sharing this store, whose per-instance +# lock doesn't serialize against this one — so clear() can't be silently undone. _MEMORY_GENERATION: Dict[str, int] = {} _MEMORY_LOCK = threading.Lock() @@ -101,14 +98,12 @@ class MemoryCache(TokenCache): """ Process-global, in-memory cache (the default). - Safest backend: nothing is written to disk. Tokens survive for the life - of the Python process, so re-running cells is silent, but a kernel - restart re-prompts once. + Safest backend: nothing hits disk. Tokens live for the life of the process, + so re-running cells is silent; a kernel restart re-prompts once. """ def load(self, key: str) -> Optional[TokenSet]: - # Return a copy so callers can't mutate the cached entry in place - # (the live token is refreshed/rotated independently). + # Return a copy so callers can't mutate the cached entry in place. with _MEMORY_LOCK: tokens = _MEMORY_STORE.get(key) return replace(tokens) if tokens is not None else None @@ -126,9 +121,8 @@ def generation(self, key: str) -> int: """ Current clear()-generation for ``key``. - Captured before an acquisition's IdP round-trip and handed back to - :meth:`store_if_current`, which drops the write if a ``clear()`` bumped - the counter meanwhile (see :meth:`store_if_current`). + Capture before an IdP round-trip and pass to :meth:`store_if_current`, + which drops the write if a ``clear()`` bumped the counter meanwhile. """ with _MEMORY_LOCK: return _MEMORY_GENERATION.get(key, 0) @@ -138,11 +132,10 @@ def store_if_current( """ Store ``tokens`` only if no :meth:`clear` happened since ``generation``. - If a concurrent ``clear()`` — on this or any other - :class:`~questdb.auth.OidcDeviceAuth` sharing this process-global store — + If a concurrent ``clear()`` (on any OidcDeviceAuth sharing this store) bumped the counter after ``generation`` was captured, the write is - dropped (returns ``False``) so the just-cleared entry is not resurrected - with a now-stale token. Returns ``True`` when the token was stored. + dropped (``False``) so the cleared entry isn't resurrected with a stale + token; returns ``True`` when stored. """ with _MEMORY_LOCK: if _MEMORY_GENERATION.get(key, 0) != generation: diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 33d8d397..a3db901a 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -62,11 +62,10 @@ # A non-positive expires_in is non-conformant; treat it as "unknown". _DEFAULT_EXPIRES_IN = 3600 -# Bounds for the device-authorization response's timing fields (RFC 8628). The -# device code is short-lived, so the IdP-supplied values are clamped: a hostile -# or buggy response must not be able to time the flow out before its first poll, -# nor pin the polling thread — which holds the acquisition lock — in one -# enormous sleep, nor keep the loop (and the lock) alive indefinitely. +# Clamp the device-authorization timing fields (RFC 8628): a hostile/buggy +# response must not time the flow out before its first poll, pin the polling +# thread (which holds the acquisition lock) in one huge sleep, or keep the loop +# (and lock) alive indefinitely. _DEFAULT_DEVICE_CODE_LIFETIME = 600 # expires_in fallback (absent/invalid/<=0) _MAX_DEVICE_CODE_LIFETIME = 1800 # cap on how long we keep polling _MAX_POLL_INTERVAL = 60 # cap on the poll interval (incl. slow_down) @@ -86,8 +85,8 @@ def _decode_jwt_claims(token: Optional[str]) -> Dict[str, Any]: """ Best-effort decode of a JWT payload **without signature verification**. - Used only to show a friendly identity in the sign-in message. QuestDB - performs the real validation. Returns ``{}`` for opaque/invalid tokens. + Used only to show a friendly identity in the sign-in message; QuestDB does + the real validation. Returns ``{}`` for opaque/invalid tokens. """ if not token or token.count('.') < 2: return {} @@ -98,10 +97,9 @@ def _decode_jwt_claims(token: Optional[str]) -> Dict[str, Any]: claims = json.loads(raw) return claims if isinstance(claims, dict) else {} except (ValueError, binascii.Error, UnicodeDecodeError, RecursionError): - # RecursionError: a deeply-nested JSON payload exhausts the decoder's - # stack; it is not a ValueError, so list it explicitly so a hostile or - # buggy token response can't crash token()/refresh with a raw exception - # here (mirrors the guards in _http.get_json / post_form / QuestDB.sql). + # RecursionError (deeply-nested JSON exhausts the decoder stack) isn't a + # ValueError, so list it explicitly: a hostile token must not crash + # token()/refresh here. return {} @@ -115,14 +113,12 @@ def _identity_from_claims(claims: Dict[str, Any]) -> Optional[str]: def _http_status_is_terminal_4xx(status: Optional[int]) -> bool: """ - True for a client-error HTTP status that is a definitive rejection. - - A non-JSON response body carrying such a status (e.g. an HTML/plain ``403`` - from a WAF or reverse proxy in front of the IdP, or a non-conformant IdP) is - never a RFC-conformant ``authorization_pending`` / ``slow_down`` — those are - always JSON — so the device-flow poll must fail fast rather than keep - retrying to a misleading "code expired". ``429`` is excluded: it is a - rate-limit, handled as transient with back-off. + True for a 4xx that is a definitive rejection. + + A non-JSON body with such a status (e.g. an HTML ``403`` from a WAF/proxy or + non-conformant IdP) is never an ``authorization_pending`` / ``slow_down`` + (those are always JSON), so the poll must fail fast rather than retry to a + misleading "code expired". ``429`` is excluded — it's a transient rate-limit. """ return status is not None and 400 <= status < 500 and status != 429 @@ -136,28 +132,23 @@ class OidcDeviceAuth: """ Acquire and refresh an OIDC token via the device authorization grant. - The token is presented to QuestDB over the auth paths it already - supports: HTTP ``Authorization: Bearer`` or PG-wire ``_sso`` (token as - password). The flow runs entirely client-side; QuestDB is never in the - token-acquisition path. - - Most users only ever call :meth:`token` (or :meth:`headers`). The first - call runs the interactive device flow; subsequent calls return the cached - token and refresh it silently (synchronously, on the first call made after - it nears expiry — there is no background thread). Acquisition is - serialized so concurrent callers don't double-prompt, while a valid cached - token is returned without blocking on another thread's in-progress - sign-in. - - **Concurrency note.** The serialization lock is held for the whole of an - interactive sign-in (up to the device-code lifetime, ~30 min). A caller - that already holds a *valid* cached token never blocks, but a caller whose - token is missing or expired blocks behind whoever is signing in; if that - sign-in is abandoned, each waiter then re-prompts in turn. When several - threads share one auth object (e.g. a SQLAlchemy / psycopg connection - pool), sign in once up front — :func:`questdb.auth.connect` does this for - you with ``eager=True`` (the default), so the interactive flow runs a - single time on the main thread before the pool opens connections. + The token is presented to QuestDB over the auth paths it already supports: + HTTP ``Authorization: Bearer`` or PG-wire ``_sso`` (token as password). The + flow runs entirely client-side; QuestDB is never in the acquisition path. + + Most users only call :meth:`token` (or :meth:`headers`). The first call runs + the interactive device flow; later calls return the cached token, refreshing + it silently and synchronously once it nears expiry (no background thread). + Acquisition is serialized so concurrent callers don't double-prompt, while a + valid cached token is returned without blocking on another's sign-in. + + **Concurrency note.** The lock is held for a whole interactive sign-in (up + to the device-code lifetime, ~30 min): a caller with a *valid* cached token + never blocks, but one whose token is missing/expired waits behind the + signer. So when threads share an auth object (e.g. a SQLAlchemy/psycopg + pool), sign in once up front — :func:`questdb.auth.connect` does this via + ``eager=True`` (the default), running the flow once on the main thread before + the pool opens connections. .. code-block:: python @@ -221,43 +212,38 @@ def __init__( audience=audience, issuer=issuer) - # Enforce the credential-endpoint co-location / issuer pin on every - # construction path (not just discovery), so the documented guarantee - # holds for the explicit constructor too. + # Enforce the credential-endpoint co-location / issuer pin here too (not + # just on the discovery path), so the guarantee holds for this + # constructor as well. validate_endpoint_origins( self.config.token_endpoint, self.config.device_authorization_endpoint, self.config.issuer) - # `insecure` permits plaintext http only to QuestDB (e.g. a local dev - # server). The IdP is always held to https — or loopback http — by - # _idp_post, so the device code / refresh token are never sent in - # cleartext over the network even when this is set. + # `insecure` permits plaintext http only to QuestDB (e.g. local dev). + # _idp_post always holds the IdP to https (or loopback http), so the + # device code / refresh token are never sent in cleartext even when set. self.insecure = insecure self.open_browser = open_browser - # Kept so adapters that build their own transport (QuestDB.sender's ILP - # Sender) can forward the same private CA the urllib _ctx uses, instead - # of falling back to the default trust roots. See QuestDB.sender. + # Kept so adapters with their own transport (QuestDB.sender's ILP Sender) + # can forward the same private CA as _ctx rather than the default roots. self._ca_bundle = ca_bundle self._interactive = interactive self._default_interval = default_interval - # Per-request network timeout for every IdP call (device-code request, - # each poll, refresh). It bounds how long a single network leg can pin - # the acquisition lock if the IdP stalls: lower it to reduce lock-hold - # (and connection-pool starvation) during an IdP outage; raise it for a - # slow IdP. The total interactive-poll duration is separately capped by + # Per-request network timeout for every IdP call (device-code, each poll, + # refresh). Bounds how long one network leg pins the acquisition lock if + # the IdP stalls; the total poll duration is separately capped by # _MAX_DEVICE_CODE_LIFETIME. self._timeout = timeout self._cache = make_cache(cache) self._ctx = build_ssl_context(ca_bundle) self._renderer = renderer if renderer is not None else make_renderer(qr=qr) - # Serializes token *acquisition* (a silent refresh or the interactive - # sign-in) only. Concurrent callers are possible via the threaded - # SQLAlchemy/psycopg adapters: without this, several connections - # opening as the token expires would run overlapping refreshes, and - # with refresh-token rotation all but one would fail and force a - # spurious re-prompt. It is NOT held on the fast path, so a caller with - # a valid cached token never blocks behind another thread's sign-in. + # Serializes token *acquisition* (silent refresh or interactive sign-in) + # only. Without it, threaded SQLAlchemy/psycopg connections opening as + # the token expires would run overlapping refreshes — and with + # refresh-token rotation all but one would fail and re-prompt. NOT held + # on the fast path, so a valid cached token never blocks behind a + # sign-in. self._lock = threading.Lock() self._tokens: Optional[TokenSet] = None clock = _clock or _SYSTEM_CLOCK @@ -296,8 +282,8 @@ def from_questdb( Reads ``{url}/settings`` for the OIDC client id, scope, endpoints and groups mode, falling back to the IdP ``.well-known`` document for the - device-authorization endpoint when QuestDB does not advertise it. - Any explicit keyword overrides discovery. + device-authorization endpoint when QuestDB doesn't advertise it. Any + explicit keyword overrides discovery. """ _validate_flow(flow) ctx = build_ssl_context(ca_bundle) @@ -340,7 +326,7 @@ def token(self) -> str: Return a valid token for QuestDB, acquiring or refreshing as needed. Returns the ``id_token`` when the server expects groups encoded in the - token (``acl.oidc.groups.encoded.in.token=true``), otherwise the + token (``acl.oidc.groups.encoded.in.token=true``), else the ``access_token`` — mirroring QuestDB's own selection logic. """ return self._select(self._obtain_tokens()) @@ -354,18 +340,17 @@ def cache_key(self) -> str: """ Identifies the token's security context for caching. - Two sessions share a cached token only when they would accept the same - one: same IdP token endpoint (**path included**, so multi-tenant realms - sharing a host don't collide), client id, scope *set* (order-insensitive), + Two sessions share a cached token only when they'd accept the same one: + same IdP token endpoint (**path included**, so multi-tenant realms on one + host don't collide), client id, scope *set* (order-insensitive), audience, and token-kind mode (``groups_in_token`` — id_token vs - access_token). The QuestDB URL is deliberately excluded — the same IdP - token is valid against any QuestDB that trusts it. - - ``groups_in_token`` is part of the key because it selects which token - kind :meth:`_select` returns; without it two sessions that differ only - in that mode would collide on one entry and repeatedly evict each - other's token (the gate self-corrects, but at the cost of avoidable - refreshes / re-prompts). + access_token). The QuestDB URL is excluded — the same IdP token is valid + against any QuestDB that trusts it. + + ``groups_in_token`` is keyed because it selects the token kind + :meth:`_select` returns; otherwise two sessions differing only in that + mode would collide and repeatedly evict each other's token (self- + correcting, but at the cost of avoidable refreshes / re-prompts). """ c = self.config scope = ' '.join(sorted(c.scope.split())) if c.scope else '' @@ -380,11 +365,10 @@ def cache_key(self) -> str: def clear(self) -> None: """Forget the cached token (forces a fresh sign-in next time).""" # self._lock serializes against THIS instance's acquisition; the shared - # MemoryCache additionally bumps a per-key generation here, so an - # in-flight acquisition on ANOTHER OidcDeviceAuth that shares the - # process-global store can't repopulate the entry after this clear (its - # _store sees the bumped generation and drops the write). This resets the - # local / process cache only — it does not revoke the token at the IdP. + # MemoryCache also bumps a per-key generation, so an in-flight acquire on + # ANOTHER instance sharing the process-global store can't repopulate the + # entry (its _store sees the bumped generation and drops the write). + # Resets the local/process cache only — does not revoke at the IdP. with self._lock: self._tokens = None self._cache.clear(self.cache_key) @@ -406,10 +390,10 @@ def _select(self, tokens: TokenSet) -> str: def _has_required_token(self, tokens: TokenSet) -> bool: """ - True if ``tokens`` carries the kind :meth:`_select` will return — the - ``id_token`` when groups are encoded in the token, else the - ``access_token``. The cache gate and the post-refresh check share this - predicate so they can't disagree with ``_select``. + True if ``tokens`` carries the kind :meth:`_select` will return (the + ``id_token`` in groups mode, else the ``access_token``). The cache gate + and post-refresh check share this predicate so they can't disagree with + ``_select``. """ if self.config.groups_in_token: return bool(tokens.id_token) @@ -417,11 +401,10 @@ def _has_required_token(self, tokens: TokenSet) -> bool: def _missing_required_token_error(self) -> OidcDeviceFlowError: """ - Build the terminal error for a *completed* grant whose token response - omits the kind :meth:`_select` needs (the ``id_token`` in groups mode, - else the ``access_token``). Mirrors :meth:`_select`'s diagnostics, but - is an :class:`OidcDeviceFlowError` — a flow failure — so the device-flow - poll can raise it without first caching an unusable response. + Terminal error for a *completed* grant whose response omits the kind + :meth:`_select` needs. Mirrors :meth:`_select`'s diagnostics but as an + :class:`OidcDeviceFlowError`, so the poll can raise it without first + caching an unusable response. """ if self.config.groups_in_token: return OidcDeviceFlowError( @@ -434,29 +417,26 @@ def _missing_required_token_error(self) -> OidcDeviceFlowError: 'access_token.') def _obtain_tokens(self) -> TokenSet: - # Fast path: return a valid token without taking the lock, so a caller - # with a usable token never blocks behind another thread's in-progress - # refresh or interactive sign-in. This path is READ-ONLY: it never - # writes self._tokens (M4). Every write to that field happens under the - # lock (the promotion below, plus _store and clear), so the lock-free - # reader can't race a concurrent write / lose an update / resurrect a - # just-cleared token. + # Fast path: return a valid token without the lock, so a caller with a + # usable token never blocks behind another thread's refresh/sign-in. + # READ-ONLY — never writes self._tokens; every write to that field is + # under the lock (the promotion below, _store, clear), so this lock-free + # reader can't race a write or resurrect a just-cleared token. tokens = self._valid_cached() if tokens is not None: return tokens - # Slow path: serialize acquisition so concurrent callers don't run - # overlapping refreshes or double-prompt; the loser re-checks and - # reuses the winner's freshly acquired token. + # Slow path: serialize acquisition so concurrent callers don't overlap + # refreshes or double-prompt; the loser re-checks and reuses the + # winner's token. with self._lock: - # Capture the cache generation before reading or acquiring, so a - # clear() that races this acquisition — including one on another - # OidcDeviceAuth that shares the process-global MemoryCache (whose - # per-instance lock does not serialize against ours) — invalidates - # the store below instead of resurrecting the just-cleared entry. + # Capture the generation before reading/acquiring, so a racing + # clear() — including on another instance sharing the process-global + # MemoryCache (whose per-instance lock doesn't serialize against + # ours) — invalidates the store below instead of resurrecting the + # cleared entry. generation = self._cache_generation() - # Promote a cached token into the field under the lock (even an - # expired one, so _acquire can reuse its refresh_token for a silent - # refresh). Done here, not on the lock-free fast path, so every + # Promote a cached token under the lock (even expired, so _acquire + # can reuse its refresh_token). Here, not on the fast path, so every # write to self._tokens stays serialized. if self._tokens is None: cached = self._cache.load(self.cache_key) @@ -468,10 +448,9 @@ def _obtain_tokens(self) -> TokenSet: return self._acquire(generation) def _valid_cached(self) -> Optional[TokenSet]: - # Read-only: reads the published field, falling back to a read of the - # shared cache backend. It never writes self._tokens — that write is - # done only under the lock (in _obtain_tokens' slow path / _store / - # clear) — so it is safe to call on the lock-free fast path. + # Read-only: reads the published field, falling back to the shared cache + # backend. Never writes self._tokens (that's lock-only), so it's safe on + # the lock-free fast path. tokens = self._tokens if tokens is None: tokens = self._cache.load(self.cache_key) @@ -481,31 +460,26 @@ def _valid_cached(self) -> Optional[TokenSet]: return None def _acquire(self, generation: int) -> TokenSet: - # Called while holding self._lock. Try a silent refresh, else run the - # interactive device flow. `generation` was captured before the cache - # read in _obtain_tokens; _store drops its write if a concurrent clear() - # has bumped it since (see _store / _cache_generation). + # Holds self._lock. Try a silent refresh, else run the device flow. + # `generation` was captured before the cache read in _obtain_tokens; + # _store drops its write if a concurrent clear() bumped it since. tokens = self._tokens if tokens is not None and tokens.refresh_token: try: refreshed = self._refresh(tokens) except OidcNetworkError: - # Transient connectivity failure: the refresh token is still - # valid, so re-authenticating won't help (the interactive flow - # needs the same network) and would needlessly re-prompt. - # Surface it — the cached token + refresh_token are kept, so a - # later call retries the refresh. + # Transient: the refresh token is still valid, so the interactive + # flow (same network) wouldn't help and would needlessly + # re-prompt. Surface it; the cached token is kept for a retry. raise except OidcError: - # The refresh token was rejected (expired/revoked) or the IdP - # returned an unusable response: fall through to a fresh - # interactive sign-in. + # Refresh token rejected (expired/revoked) or unusable response: + # fall through to a fresh interactive sign-in. pass else: - # Only accept a refresh that actually yields the token kind we - # need. Some IdPs don't re-issue the id_token on refresh; such - # a response is unusable, so fall through to the interactive - # flow rather than caching it and looping on every call. + # Accept only a refresh that yields the kind we need: some IdPs + # don't re-issue the id_token on refresh, so fall through rather + # than cache an unusable response and loop on every call. if self._has_required_token(refreshed): self._store(refreshed, generation) return refreshed @@ -515,13 +489,12 @@ def _acquire(self, generation: int) -> TokenSet: return fresh def _store(self, tokens: TokenSet, generation: int) -> None: - # self._tokens is this instance's own view, so always set it — the - # caller uses the token it just acquired. The shared-cache write is - # conditional: a clear() (here or on another instance sharing the - # process-global store) that bumped the generation since it was captured - # drops the write, so clear() is not silently undone. Backends without - # generation support (NullCache / a custom TokenCache) store - # unconditionally, exactly as before. + # self._tokens is this instance's own view, so always set it (the caller + # uses what it just acquired). The shared-cache write is conditional: a + # clear() (here or on another instance sharing the store) that bumped the + # generation drops the write, so clear() isn't silently undone. Backends + # without generation support (NullCache / custom TokenCache) store + # unconditionally. self._tokens = tokens store_if_current = getattr(self._cache, 'store_if_current', None) if store_if_current is not None: @@ -531,8 +504,8 @@ def _store(self, tokens: TokenSet, generation: int) -> None: def _cache_generation(self) -> int: # MemoryCache tracks a per-key clear()-generation for the cross-instance - # CAS in _store; other backends don't, so default to 0 (the store is - # then unconditional, matching the pre-existing behavior). + # CAS in _store; other backends don't, so default to 0 (unconditional + # store). generation = getattr(self._cache, 'generation', None) return generation(self.cache_key) if generation is not None else 0 @@ -540,12 +513,12 @@ def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: try: expires_in = int(body.get('expires_in', _DEFAULT_EXPIRES_IN)) except (TypeError, ValueError, OverflowError): - # OverflowError: a JSON Infinity (json.loads accepts it) → int(inf); - # it is not a ValueError, so list it to keep the typed contract. + # OverflowError: a JSON Infinity (json.loads accepts it) → int(inf) + # isn't a ValueError, so list it to keep the typed contract. expires_in = _DEFAULT_EXPIRES_IN if expires_in <= 0: - # A non-positive lifetime would mark a just-issued token as already - # expired, causing refresh/re-prompt churn. Treat it as unknown. + # A non-positive lifetime marks a just-issued token as expired, + # causing refresh/re-prompt churn. Treat it as unknown. expires_in = _DEFAULT_EXPIRES_IN claims = (_decode_jwt_claims(body.get('id_token')) or _decode_jwt_claims(body.get('access_token'))) @@ -561,11 +534,10 @@ def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: sub=claims.get('sub')) def _idp_post(self, url: str, form: Dict[str, Any]): - # IdP POSTs carry the device code / refresh token, so they are always - # required to be https (loopback http is fine for local dev); the - # user's `insecure` flag — which is about the QuestDB link — never - # downgrades them. The timeout bounds how long this leg can hold the - # acquisition lock if the IdP stalls. + # IdP POSTs carry the device code / refresh token, so always https + # (loopback http is fine for local dev); the user's `insecure` flag (the + # QuestDB link) never downgrades them. The timeout bounds how long this + # leg can hold the acquisition lock if the IdP stalls. return post_form( url, form, ctx=self._ctx, insecure=False, timeout=self._timeout) @@ -578,42 +550,38 @@ def _refresh(self, tokens: TokenSet) -> TokenSet: 'refresh_token': tokens.refresh_token, 'client_id': self.config.client_id, 'scope': self.config.scope, - # Re-send the audience on refresh too, mirroring the - # device-authorization request: some IdPs (e.g. Auth0) need - # it to keep the rotated access token's `aud`, and would - # otherwise mint a token QuestDB rejects only AFTER a silent - # refresh. IdPs that don't use it ignore the param; post_form - # drops it entirely when audience is None (not configured). + # Re-send the audience (mirroring the device-authorization + # request): some IdPs (e.g. Auth0) need it to keep the + # rotated token's `aud`, else they mint one QuestDB rejects + # only after a silent refresh. Others ignore it; post_form + # drops it when audience is None. 'audience': self.config.audience, }) except OidcNetworkError: - # Already transient (socket drop / DNS / per-request timeout): - # propagate so _acquire keeps the still-valid refresh token and - # retries later instead of re-prompting. + # Already transient (socket drop / DNS / timeout): propagate so + # _acquire keeps the still-valid refresh token and retries later. raise except OidcError as e: - # Non-JSON HTTP error body (e.g. an HTML 5xx from a proxy in front - # of the IdP). A 5xx / 429 is a transient hiccup — re-raise as a - # network error so _acquire keeps the refresh token; a 4xx is a - # genuine rejection, so let it fall through (as an OidcError) to a - # fresh interactive sign-in. + # Non-JSON HTTP error body (e.g. an HTML 5xx from a proxy). 5xx/429 + # is transient → re-raise as a network error so _acquire keeps the + # refresh token; a 4xx is a genuine rejection, so let it fall through + # to a fresh interactive sign-in. if _http_status_is_transient(getattr(e, 'status', None)): raise OidcNetworkError(str(e)) from e raise if status == 200: refreshed = self._tokenset_from_response(body) - # Many IdPs do not rotate the refresh token; keep the old one. - # TokenSet is frozen, so derive a copy rather than mutating. + # Many IdPs don't rotate the refresh token; keep the old one. + # TokenSet is frozen, so derive a copy. if not refreshed.refresh_token: refreshed = replace( refreshed, refresh_token=tokens.refresh_token) return refreshed - # A transient IdP error (5xx / 429) during a silent refresh must not - # tear down the session: the refresh token is still valid, so surface it - # as a network error and let _acquire keep it and retry later — matching - # the poll loop, which also treats 5xx/429 as transient. Only a genuine - # rejection (an expired/revoked refresh token, a 4xx invalid_grant) - # falls through to a fresh interactive sign-in. + # A transient 5xx/429 during a silent refresh must not tear down the + # session: the refresh token is still valid, so surface it as a network + # error for _acquire to retry — matching the poll loop. Only a genuine + # rejection (expired/revoked token, 4xx invalid_grant) falls through to a + # fresh sign-in. if _http_status_is_transient(status): raise OidcNetworkError( f'Token refresh hit a transient IdP error (HTTP {status}); ' @@ -657,10 +625,9 @@ def _request_device_code(self) -> Dict[str, Any]: return body error = body.get('error') if status == 200: - # 200 but the success guard above failed: the response is missing - # device_code/user_code. That is a non-conformant body, not an - # HTTP-level failure — say so plainly rather than the contradictory - # "Device authorization request failed (HTTP 200)". + # 200 but the guard above failed: device_code/user_code missing. + # A non-conformant body, not an HTTP failure — say so plainly rather + # than a contradictory "failed (HTTP 200)". raise OidcDeviceFlowError( 'The IdP returned a 200 device-authorization response that is ' 'missing the required "device_code"/"user_code" fields; cannot ' @@ -690,18 +657,16 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: interval = int(resp.get('interval', self._default_interval)) except (TypeError, ValueError, OverflowError): interval = self._default_interval - # At least 1s (RFC 8628 floor), and capped so a hostile/huge value can't - # pin the polling thread (which holds the acquisition lock) in one - # enormous sleep. + # At least 1s (RFC 8628 floor), capped so a hostile value can't pin the + # polling thread (which holds the lock) in one enormous sleep. interval = min(_MAX_POLL_INTERVAL, max(1, interval)) try: expires_in = int(resp.get('expires_in', _DEFAULT_DEVICE_CODE_LIFETIME)) except (TypeError, ValueError, OverflowError): expires_in = _DEFAULT_DEVICE_CODE_LIFETIME - # A non-positive lifetime would time the flow out before the first poll - # (the user has already been shown the code); treat it as unknown. Cap - # the upper end so a hostile expires_in can't keep the loop — and the - # lock — alive indefinitely. + # A non-positive lifetime would time out before the first poll (the code + # is already shown); treat it as unknown. Cap the upper end so a hostile + # value can't keep the loop — and the lock — alive indefinitely. if expires_in <= 0: expires_in = _DEFAULT_DEVICE_CODE_LIFETIME expires_in = min(expires_in, _MAX_DEVICE_CODE_LIFETIME) @@ -717,8 +682,7 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: 'Run the sign-in again.', error='expired_token') self._renderer.on_waiting(remaining) - # Never sleep past the deadline (remaining > 0 here): a clamped - # interval still shouldn't overshoot a short-lived code. + # Never sleep past the deadline (remaining > 0 here). self._sleep(min(interval, remaining)) try: @@ -730,11 +694,10 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: 'client_id': self.config.client_id, }) except OidcError as e: - # A non-JSON 4xx is a terminal rejection (e.g. an HTML/plain - # error page from a WAF or reverse proxy in front of the IdP, or - # a non-conformant IdP): a conformant OAuth error is JSON, so it - # can never be authorization_pending / slow_down. Fail fast - # instead of polling on to a misleading "code expired". + # A non-JSON 4xx is a terminal rejection (e.g. an HTML error page + # from a WAF/proxy, or a non-conformant IdP): a conformant OAuth + # error is JSON, so it can never be authorization_pending / + # slow_down. Fail fast instead of polling on to "code expired". if _http_status_is_terminal_4xx(getattr(e, 'status', None)): self._renderer.on_failure( 'Sign-in failed: the identity provider rejected the ' @@ -742,42 +705,36 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: raise OidcDeviceFlowError( f'Device flow failed: the IdP rejected the token ' f'request ({e}).') from e - # Otherwise transient, not a terminal OAuth decision: a dropped - # connection / DNS blip / per-request timeout (OidcNetworkError), - # or a non-JSON 5xx/429 such as an HTML 502/503/504 from a proxy - # in front of the IdP (a bare OidcError from post_form). The user - # may already have authorized in the browser, and RFC 8628 §3.4 - # expects polling to continue until the device code expires, so - # poll again instead of discarding the in-progress sign-in. The - # deadline check at the top of the loop bounds the total wait; a - # genuine JSON rejection arrives as a JSON error body (below). + # Otherwise transient: a dropped connection / DNS blip / timeout + # (OidcNetworkError) or a non-JSON 5xx/429 from a proxy (bare + # OidcError). The user may already have authorized, and RFC 8628 + # §3.4 expects polling to continue until the code expires, so + # poll again rather than discard the sign-in (the deadline bounds + # the total wait; a genuine JSON rejection arrives below). if getattr(e, 'status', None) == 429: interval = min(_MAX_POLL_INTERVAL, interval + 5) continue if status == 200: - # A 200 is the RFC 6749 §5.1 token response: the grant - # completed. Accept it only if it actually carries the kind - # _select will hand to QuestDB (the id_token in groups mode, - # else the access_token), using the same predicate as the cache - # gate and the post-refresh check so the three can't disagree. + # The RFC 6749 §5.1 token response: the grant completed. Accept + # it only if it carries the kind _select hands to QuestDB, using + # the same predicate as the cache gate and post-refresh check so + # the three can't disagree. tokens = self._tokenset_from_response(body) if self._has_required_token(tokens): return tokens - # The grant completed but the required kind is absent: a stable - # misconfiguration, not a transient poll state. Raise a clear - # terminal error here instead of caching an unusable token and - # silently re-running the whole interactive flow on every later - # token() call. + # Grant completed but the required kind is absent: a stable + # misconfiguration, not a transient poll state. Raise a terminal + # error rather than cache an unusable token and silently re-run + # the whole flow on every later token() call. self._renderer.on_failure( 'Sign-in failed: the identity provider did not return the ' 'token this server requires.') raise self._missing_required_token_error() - # A 5xx or 429 that did carry a JSON body is also transient (a - # server-side error or a rate-limit) rather than a terminal OAuth - # rejection: back off on a rate-limit and keep polling until the - # deadline, matching the connection-failure handling above. + # A 5xx/429 with a JSON body is also transient (server error or + # rate-limit), not a terminal rejection: back off on 429 and keep + # polling until the deadline, as above. if status >= 500 or status == 429: if status == 429: interval = min(_MAX_POLL_INTERVAL, interval + 5) @@ -812,12 +769,12 @@ def _is_interactive(self) -> bool: return detect_interactive() def _maybe_open_browser(self, resp: Dict[str, Any]) -> None: - # Never auto-open on a (possibly remote) notebook kernel; only do so - # for an explicitly opted-in local terminal session. + # Never auto-open on a (possibly remote) notebook kernel; only on an + # opted-in local terminal. if not self.open_browser or in_ipython_kernel(): return - # Only open an http(s) URL — never a javascript:/data: scheme from a - # malicious or MITM'd device response. + # Only http(s) — never a javascript:/data: scheme from a malicious or + # MITM'd device response. target = _safe_link_url( resp.get('verification_uri_complete') or resp.get('verification_uri') @@ -841,9 +798,9 @@ def _validate_flow(flow: str) -> None: def _normalize_url(url: str) -> str: - # Full URL with scheme/host lower-cased and the default port dropped, but - # the path kept (it distinguishes multi-tenant realms). Used for the cache - # key so trivial spelling differences don't cause a spurious re-prompt. + # Full URL with scheme/host lower-cased and default port dropped, but path + # kept (it distinguishes multi-tenant realms). Used for the cache key so + # trivial spelling differences don't cause a spurious re-prompt. parts, port = safe_urlparse(url) scheme = (parts.scheme or '').lower() host = (parts.hostname or '').lower() diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 01010b5e..68fe7301 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -25,13 +25,12 @@ """ OIDC configuration discovery. -Resolution order, mirroring the design doc: +Resolution order: -1. ``GET {questdb_url}/settings`` (public, no auth) -> the QuestDB-authoritative +1. ``GET {questdb_url}/settings`` (public) -> QuestDB-authoritative ``acl.oidc.*`` values (client id, scope, endpoints, groups mode). -2. If the device-authorization endpoint is not advertised by QuestDB (today's - servers), fall back to the IdP discovery document - (``{issuer}/.well-known/openid-configuration``). +2. If QuestDB doesn't advertise the device-authorization endpoint, fall back to + the IdP discovery document (``{issuer}/.well-known/openid-configuration``). """ from __future__ import annotations @@ -92,12 +91,10 @@ def _str_setting(value: Any) -> Optional[str]: """ A ``/settings`` value as a non-empty string, else ``None``. - ``/settings`` is server-controlled (and tamperable over a plaintext insecure - channel). A non-string ``acl.oidc.*`` value — a JSON list/number from a buggy - or hostile server — must not reach ``scope.split()`` or the cache-key join as - a raw object, where it would escape the package's typed-error contract with a - bare ``AttributeError`` / ``TypeError``. Mirrors :func:`_resolve_endpoint`, - which already drops a non-string endpoint. + Drops a non-string ``acl.oidc.*`` value (a JSON list/number from a buggy or + hostile server) so it can't reach ``scope.split()`` / the cache-key join as a + raw object and escape the typed-error contract with an ``AttributeError`` / + ``TypeError``. Mirrors :func:`_resolve_endpoint`. """ return value if isinstance(value, str) and value else None @@ -106,27 +103,20 @@ def settings_config(settings: Any) -> Dict[str, Any]: """ Return the trusted config map from a ``/settings`` response. - Modern QuestDB nests the server-authoritative values under a top-level - ``"config"`` object, alongside a **user-writable** ``"preferences"`` sibling - (the web console persists UI preferences there via ``PUT /settings``). - Discovery must read only ``"config"`` and never the top level, so a user who - can write a preference cannot smuggle an ``acl.oidc.*`` key — e.g. a - redirected ``token.endpoint`` that points the device code / refresh token at - an attacker — into the resolved OIDC configuration. - - A genuinely flat, legacy ``/settings`` response (no ``"config"`` / - ``"preferences"`` split) is still tolerated at the top level. + Modern QuestDB nests server-authoritative values under ``"config"``, + alongside a **user-writable** ``"preferences"`` sibling (written via + ``PUT /settings``). Read only ``"config"`` so a user who can write a + preference can't smuggle an ``acl.oidc.*`` key (e.g. a redirected + ``token.endpoint``) into the resolved config. A genuinely flat legacy + response (no ``config`` / ``preferences`` split) is still tolerated. """ if not isinstance(settings, dict): return {} cfg = settings.get('config') if isinstance(cfg, dict): return cfg - # A structured response carries the user-writable "preferences" sibling - # (and normally the "config" object). If either marker is present, the top - # level is NOT trusted config: read "config" or nothing — so user-writable - # preferences can never be mistaken for server-authoritative config, even - # when "config" is absent or malformed. + # Either marker present => structured response: read "config" or nothing, + # never the user-writable top level — even when "config" is absent/malformed. if 'config' in settings or 'preferences' in settings: return {} # Legacy flat response: no config/preferences split; tolerate top-level keys. @@ -166,10 +156,9 @@ def _origin_str(url: str) -> str: def _settings_channel_is_plaintext(questdb_url: str) -> bool: """ True if QuestDB ``/settings`` was fetched over plaintext http to a - non-loopback host — a channel a network MITM can tamper (only reachable - with ``insecure=True``; ``_require_secure`` rejects it otherwise). IdP - endpoints advertised by such an unauthenticated ``/settings`` response must - not be trusted to route credentials without an out-of-band pin. + non-loopback host — a MITM-tamperable channel (only reachable with + ``insecure=True``). Endpoints advertised over it must not route credentials + without an out-of-band pin. """ parts, _ = safe_urlparse(questdb_url) return (parts.scheme or '').lower() == 'http' and not _is_loopback( @@ -180,15 +169,12 @@ def _decode_path_segments(path: str) -> list: """ Fully percent-decode a URL path and split it into ``/`` segments. - Decoding is repeated until stable so a double/triple-encoded dot segment - (``%252e%252e`` -> ``%2e%2e`` -> ``..``), or an encoded slash (``%2f``) that - splits a segment, is unmasked — a server or reverse proxy may unescape more - than once before it normalizes. A backslash is treated as a separator, since - some proxies fold ``\\`` to ``/`` before routing. The returned segments are - what the containment check compares, never the raw string urllib puts on the - wire, so an encoding the server later undoes can't smuggle a ``..`` past the - scan. The loop is bounded (a real path needs 0-1 passes; more layers than a - server would itself decode can't resolve to a traversal anyway). + Decoding repeats until stable so a multiply-encoded dot segment + (``%252e%252e`` -> ``..``) or encoded slash (``%2f``) — which a server/proxy + may unescape more than once before normalizing — is unmasked. Backslash is a + separator (some proxies fold ``\\`` to ``/``). The containment check compares + these decoded segments, not the raw wire string, so an encoding the server + later undoes can't hide a ``..``. Loop bounded: a real path needs 0-1 passes. """ decoded = path for _ in range(10): # bounded; each pass peels one percent-encoding layer @@ -204,32 +190,26 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: True if ``endpoint``'s path is the issuer's path or a sub-path of it. Segment-aware, so ``/realms/prod`` does not match ``/realms/production``. A - root issuer (no path, e.g. ``https://idp.example.com``) constrains the - origin only and matches any path. Used to keep a tampered ``/settings`` from - redirecting credentials to a different tenant on a path-based multi-tenant - IdP (Keycloak issuers are ``https://host/realms/{realm}``), which an - origin-only check can't catch. - - The comparison is done on the fully *decoded* path segments, never the raw - string urllib sends. A ``.`` / ``..`` segment is rejected outright: urllib - puts the dotted path on the wire verbatim, but the IdP (or a reverse proxy - in front of it) normalizes it, so ``/realms/prod/../attacker/token`` would - satisfy a naive prefix test yet resolve server-side to a *different* realm — - defeating the very isolation this check exists to provide. Encoded dot - segments are unmasked first — including double-encoded (``%252e``) and - encoded slashes (``%2f``) a server may unescape more than once — a backslash - is treated as a separator, and the last segment's ``;params`` (which urllib - splits off ``.path``) is folded back in, so none of those can smuggle a - traversal past the segment scan. A legitimate endpoint path never contains - dot segments. + root issuer (no path) constrains the origin only and matches any path. Stops + a tampered ``/settings`` from redirecting credentials to a different tenant + on a path-based multi-tenant IdP (Keycloak issuers are + ``https://host/realms/{realm}``), which an origin-only check can't catch. + + Compared on fully *decoded* path segments, not the raw wire string. A ``.`` / + ``..`` segment is rejected outright: the server normalizes it, so + ``/realms/prod/../attacker/token`` passes a naive prefix test yet resolves to + a *different* realm. ``_decode_path_segments`` unmasks encoded dot segments, + and the last segment's ``;params`` (which urllib splits off ``.path``) is + folded back, so neither can hide a traversal. Legitimate paths have no dot + segments. """ base = (safe_urlparse(issuer)[0].path or '').rstrip('/') if not base: return True base_segs = _decode_path_segments(base) eparts = safe_urlparse(endpoint)[0] - # urllib splits the last segment's ;params off .path; fold it back so a - # traversal hidden there (…/token;..%2f..%2fEVIL) can't slip past the scan. + # Fold the last segment's ;params back into the path so a traversal hidden + # there (…/token;..%2f..%2fEVIL) can't slip past the scan. ep_path = eparts.path or '' if eparts.params: ep_path = f'{ep_path};{eparts.params}' @@ -246,28 +226,24 @@ def validate_endpoint_origins( """ Reject an OIDC configuration that would send credentials off-origin. - The device code and the long-lived refresh token are POSTed to the device- - authorization and token endpoints. These come from QuestDB ``/settings`` - (or the IdP ``.well-known``), which the client trusts; this check limits a - tampered or MITM'd configuration from redirecting those credentials to an - attacker-controlled host: - - * the two credential endpoints must share a single origin (they are always - co-located on the authorization server per RFC 8628); and - * when the ``issuer`` is known independently (passed explicitly or resolved - from the IdP ``.well-known``), both endpoints must share its **origin**. - - This is an origin-level check: it does **not**, on its own, isolate - path-based multi-tenant realms (e.g. Keycloak issuers - ``https://host/realms/{realm}``, where every realm shares one origin). That - path-scoping is enforced separately in :func:`resolve_config`, and only for - endpoints advertised by the (untrusted) QuestDB ``/settings`` — endpoints - from IdP discovery (the issuer's own ``.well-known``) and caller-explicit - endpoints are authoritative and are not path-restricted (some IdPs, e.g. - Azure AD, legitimately place endpoints outside the issuer path). - - Pass ``issuer=`` to pin the IdP explicitly when QuestDB advertises the - endpoints directly (so a compromised server cannot redirect the token POST). + The device code and long-lived refresh token are POSTed to the device- + authorization and token endpoints. This limits a tampered or MITM'd config + from steering those credentials to an attacker host: + + * the two credential endpoints must share a single origin (always co-located + on the authorization server per RFC 8628); and + * when ``issuer`` is known independently (explicit or from the IdP + ``.well-known``), both endpoints must share its **origin**. + + Origin-level only: it does **not** isolate path-based multi-tenant realms + (e.g. Keycloak ``https://host/realms/{realm}``, one origin per realm). That + path-scoping lives in :func:`resolve_config`, and only for endpoints from the + untrusted QuestDB ``/settings``; endpoints from IdP discovery or the caller + are authoritative and not path-restricted (some IdPs, e.g. Azure AD, + legitimately place endpoints outside the issuer path). + + Pass ``issuer=`` to pin the IdP when QuestDB advertises the endpoints + directly, so a compromised server cannot redirect the token POST. """ if _normalized_origin(token_endpoint) != _normalized_origin( device_authorization_endpoint): @@ -301,33 +277,25 @@ def _resolve_endpoint(value: Optional[str], cfg: Dict[str, Any]) -> Optional[str if not value: return None if not isinstance(value, str): - # A non-string endpoint from /settings (e.g. a JSON number) is - # malformed; treat it as absent so resolution falls through to a clear - # OidcConfigError (or the IdP-discovery fallback) instead of an - # AttributeError from .startswith() escaping the typed-error contract. + # Non-string endpoint (e.g. a JSON number): treat as absent so resolution + # yields a clear OidcConfigError instead of an AttributeError from + # .startswith() escaping the typed-error contract. return None if value.startswith('http://') or value.startswith('https://'): return value if value.startswith('/'): - # _str_setting drops a non-string acl.oidc.host (a JSON number/list from - # a buggy or hostile /settings) so it can't be interpolated raw into the - # netloc — e.g. https://12345:9000/path — and instead reads as absent, - # mirroring how endpoint values above are coerced. (safe_urlparse would - # otherwise reject the bogus URL only incidentally, downstream.) + # _str_setting drops a non-string acl.oidc.host so it can't be + # interpolated raw into the netloc (e.g. https://12345:9000/path). host = _str_setting(cfg.get(_K_HOST)) if not host: - # A path-only endpoint with no acl.oidc.host to resolve it against - # can't be turned into a URL. Treat it as absent (return None) so - # resolution fails with the clear "could not resolve the ... - # endpoint" error rather than passing a scheme-less "/path" - # downstream, where it surfaces as a confusing "insecure/malformed - # URL" instead. + # Path-only endpoint with no host to resolve against: treat as absent + # for the clear "could not resolve" error, rather than passing a + # scheme-less "/path" on to a confusing "malformed URL" downstream. return None tls = _as_bool(cfg.get(_K_TLS_ENABLED), default=True) scheme = 'https' if tls else 'http' - # A usable port is an int or a digit string; anything else (a JSON - # list/object, a bool, or a non-numeric string) would corrupt the - # netloc, so drop it and resolve host-only. + # A usable port is an int or digit string; anything else would corrupt + # the netloc, so drop it and resolve host-only. port = cfg.get(_K_PORT) if isinstance(port, bool) or not ( isinstance(port, int) @@ -352,12 +320,11 @@ def discover_device_endpoint_from_idp( """ Fetch the IdP ``.well-known/openid-configuration`` and return it. - The discovery URL is taken from ``discovery_url``, else built from - ``issuer``. One of the two is required: the discovery origin is **never** - derived from a QuestDB-advertised endpoint, because that would let a - tampered ``/settings`` choose where the device code and refresh token are - sent (the resolved issuer and endpoints would then all share the attacker's - origin and pass the co-location / issuer-pin checks trivially). + The discovery URL comes from ``discovery_url``, else built from ``issuer``; + one is required. The discovery origin is **never** derived from a + QuestDB-advertised endpoint — that would let a tampered ``/settings`` choose + where credentials are sent, with the co-location / issuer-pin checks passing + trivially because every value would share the attacker's origin. """ url = discovery_url or (well_known_url(issuer) if issuer else None) if not url: @@ -366,14 +333,10 @@ def discover_device_endpoint_from_idp( 'or discovery_url was given. Pass issuer=... (or ' 'device_authorization_endpoint=... to skip discovery).') doc = get_json(url, ctx=ctx, insecure=insecure, timeout=timeout) - # get_json guarantees valid JSON, not a JSON *object*. A discovery document - # that is valid-JSON-but-not-a-dict (a list/null/number/string from a - # captive portal, a misconfigured proxy, or a hostile IdP) must not reach - # resolve_config's doc.get(...) calls as a raw object, where it would escape - # the package's typed-error contract with a bare AttributeError. Coerce it - # to empty so resolution fails with the clear "could not resolve the ... - # endpoint" OidcConfigError instead — mirroring settings_config, which - # applies the same guard to the QuestDB /settings response. + # get_json guarantees valid JSON, not a JSON *object*. Coerce a non-dict + # document (from a captive portal, bad proxy, or hostile IdP) to empty so + # resolve_config's doc.get(...) yields a clear "could not resolve" error + # rather than an AttributeError. Mirrors settings_config. return doc if isinstance(doc, dict) else {} @@ -409,10 +372,8 @@ def resolve_config( f'QuestDB at {questdb_url} reports OIDC is disabled ' f'({_K_ENABLED}=false). Nothing to authenticate against.') - # _str_setting drops a non-string /settings value (e.g. a JSON list) so it - # can't reach scope.split() / the cache-key join as a raw object and escape - # the typed-error contract; a non-string client.id thus reads as absent and - # surfaces the clear "Missing client_id" error below. + # _str_setting drops a non-string /settings value so a non-string client.id + # reads as absent and hits the clear "Missing client_id" error below. client_id = client_id or _str_setting(cfg.get(_K_CLIENT_ID)) if not client_id: raise OidcConfigError( @@ -426,9 +387,9 @@ def resolve_config( if audience is None: audience = _str_setting(cfg.get(_K_AUDIENCE)) - # Track which credential endpoints the caller supplied directly. Those are - # trusted; endpoints learned from /settings are only as trustworthy as the - # channel that delivered them (see the insecure-channel guard below). + # Track caller-supplied credential endpoints: those are trusted, whereas + # /settings endpoints are only as trustworthy as the channel that delivered + # them (see the insecure-channel guard below). explicit_token_endpoint = token_endpoint is not None explicit_device_endpoint = device_authorization_endpoint is not None @@ -441,18 +402,13 @@ def resolve_config( device_authorization_endpoint or _resolve_endpoint(cfg.get(_K_DEVICE_ENDPOINT), cfg)) - # When QuestDB itself was reached over plaintext http to a non-loopback host - # (only possible with insecure=True), its /settings response can be tampered - # in transit. Any IdP credential endpoint it advertises would then route the - # device code and long-lived refresh token to an attacker origin. The - # missing-endpoint discovery path below already demands an out-of-band pin, - # but when a tampered /settings advertises BOTH endpoints at one attacker - # origin that path is skipped, the co-location check passes trivially (they - # share that origin) and the issuer-pin check is vacuous (no issuer) — so - # nothing else catches it. Require the same out-of-band pin (issuer= / - # discovery_url=) before trusting /settings-supplied endpoints over such a - # channel. Endpoints the caller passed explicitly, and endpoints from an - # authenticated (https / loopback) /settings, are unaffected. + # Over a plaintext-http /settings channel (insecure=True, non-loopback), a + # tampered response can advertise BOTH credential endpoints at one attacker + # origin: the discovery path below is skipped, co-location passes trivially + # (shared origin) and the issuer-pin check is vacuous (no issuer), so nothing + # else catches it. Demand the same out-of-band pin (issuer= / discovery_url=) + # before trusting /settings endpoints here. Caller-explicit endpoints and + # those from an authenticated (https / loopback) /settings are unaffected. settings_supplied_credentials = ( (token_endpoint and not explicit_token_endpoint) or (device_authorization_endpoint and not explicit_device_endpoint)) @@ -469,16 +425,13 @@ def resolve_config( 'device_authorization_endpoint=...), or connect to QuestDB over ' 'https so /settings is authenticated.') - # When the credential endpoints came from QuestDB /settings (not the - # caller) and an issuer is pinned out-of-band, require each to sit under the - # issuer's PATH, not merely its origin. Path-based IdPs put every tenant on - # one origin (Keycloak issuers are https://host/realms/{realm}), so the - # origin check alone (validate_endpoint_origins) can't stop a tampered - # /settings from steering the device code / refresh token to a different - # realm on the same host. The issuer is out-of-band, so the server can't - # forge it. Caller-explicit endpoints, and endpoints from IdP discovery (the - # issuer's own .well-known), are authoritative and skip this — some IdPs - # (e.g. Azure AD) legitimately place endpoints outside the issuer path. + # For /settings endpoints with an out-of-band issuer, require each under the + # issuer's PATH, not just its origin: path-based IdPs share one origin per + # tenant (Keycloak https://host/realms/{realm}), so the origin check alone + # can't stop a tampered /settings steering credentials to a different realm. + # The out-of-band issuer can't be forged. Caller-explicit endpoints and those + # from IdP discovery are authoritative and skip this — some IdPs (e.g. Azure + # AD) legitimately place endpoints outside the issuer path. if issuer: for label, url, from_settings in ( ('token endpoint', token_endpoint, @@ -497,18 +450,13 @@ def resolve_config( 'device_authorization_endpoint=...).') # Fall back to IdP discovery when QuestDB doesn't advertise the device - # endpoint (and/or the token endpoint). This contacts the IdP, so it is - # held to https/loopback (insecure=False) regardless of the QuestDB flag. + # (and/or token) endpoint. This contacts the IdP, so it is held to + # https/loopback (insecure=False) regardless of the QuestDB flag. if not device_authorization_endpoint or not token_endpoint: - # Require a caller-supplied trust anchor before contacting the IdP for - # discovery. Without issuer= / discovery_url=, the discovery target - # would have to be guessed from the token endpoint that /settings - # supplied; a tampered or MITM'd /settings (reachable in cleartext when - # QuestDB is http:// with insecure=True) could then steer discovery — - # and so the device-code and refresh-token POSTs — to an attacker - # origin, with the co-location and issuer-pin checks passing trivially - # because every value shares that one origin. issuer= is out-of-band, - # so the server cannot forge it. + # Require an out-of-band trust anchor first. Otherwise the discovery + # target would be guessed from the /settings token endpoint, so a + # tampered /settings could steer discovery (and the credential POSTs) to + # an attacker origin with co-location / issuer-pin passing trivially. if not issuer and not discovery_url: raise OidcConfigError( 'QuestDB did not advertise the OIDC device-authorization ' @@ -524,12 +472,10 @@ def resolve_config( doc = discover_device_endpoint_from_idp( issuer=issuer, discovery_url=discovery_url, ctx=ctx, insecure=False, timeout=timeout) - # The IdP discovery document is untrusted too: coerce its values the - # same way as /settings values. A non-string endpoint / issuer (a JSON - # number/list from a buggy or hostile IdP) must read as absent — the - # clear "could not resolve" OidcConfigError below, or no issuer pin — - # rather than reach safe_urlparse / the cache-key join as a raw object - # and escape the typed-error contract with a bare AttributeError. + # The discovery document is untrusted too: coerce its values like + # /settings. A non-string endpoint / issuer reads as absent (clear + # "could not resolve" below, or no issuer pin) instead of reaching + # safe_urlparse / the cache-key join as a raw object. device_authorization_endpoint = ( device_authorization_endpoint or _str_setting(doc.get('device_authorization_endpoint'))) @@ -538,20 +484,13 @@ def resolve_config( authorization_endpoint = ( authorization_endpoint or _str_setting(doc.get('authorization_endpoint'))) - # OIDC Discovery §4.3 / RFC 8414 §3: when the IdP is pinned ONLY by - # discovery_url (no out-of-band issuer=), the document's self-declared - # issuer would otherwise be the trust anchor validate_endpoint_origins - # (in OidcDeviceAuth.__init__) checks the endpoints against — but that - # issuer comes from the same (possibly hostile, confused, or - # multi-tenant) document, so the check is vacuous, and an absent or - # non-string issuer makes it vacuous too (a document declaring attacker - # endpoints all on one origin would pass co-location trivially). Anchor - # instead to the caller-pinned discovery_url itself: require the - # credential endpoints to live on its origin, so a document can't - # redirect the device-code / refresh-token POSTs to an attacker origin. - # Origin-level, matching validate_endpoint_origins; pass issuer= and the - # endpoints explicitly if your IdP serves discovery and tokens from - # different origins. + # OIDC Discovery §4.3 / RFC 8414 §3: when pinned ONLY by discovery_url, + # the document's self-declared issuer (the anchor + # validate_endpoint_origins would use) comes from that same untrusted + # document, so it's a vacuous check. Anchor to the caller-pinned + # discovery_url instead: require the credential endpoints on its origin + # so the document can't redirect the POSTs off it. Origin-level; pass + # issuer= and explicit endpoints if discovery and tokens differ in origin. if discovery_url and not issuer: discovery_origin = _normalized_origin(discovery_url) for label, url in ( @@ -582,9 +521,9 @@ def resolve_config( 'device grant, or pass device_authorization_endpoint=... ' 'explicitly.') - # Note: the credential-endpoint origin check (validate_endpoint_origins) - # is enforced centrally in OidcDeviceAuth.__init__, which every path - # (including the explicit constructor) goes through. + # The credential-endpoint origin check (validate_endpoint_origins) is + # enforced centrally in OidcDeviceAuth.__init__, which every path goes + # through. return OidcConfig( client_id=client_id, diff --git a/src/questdb/auth/_errors.py b/src/questdb/auth/_errors.py index b4b7f55f..17b83e9e 100644 --- a/src/questdb/auth/_errors.py +++ b/src/questdb/auth/_errors.py @@ -34,21 +34,17 @@ class OidcError(Exception): def __init__(self, *args, status: Optional[int] = None): super().__init__(*args) - # HTTP status that produced this error, when it originated from a - # non-JSON HTTP response (else None). Lets the device-flow poll loop and - # the silent refresh tell a terminal 4xx rejection (e.g. a WAF/proxy - # error page) from a transient 5xx/429/network blip even when the body - # was not a conformant JSON OAuth error. + # HTTP status behind a non-JSON HTTP response (else None), so the poll + # loop and silent refresh can tell a terminal 4xx (e.g. a WAF error + # page) from a transient 5xx/429/network blip. self.status = status class OidcConfigError(OidcError): """ - The OIDC configuration could not be resolved or is inconsistent. - - Raised, for example, when QuestDB does not advertise OIDC, when the - IdP device-authorization endpoint cannot be discovered, or when a - required argument is missing. + The OIDC configuration could not be resolved or is inconsistent (e.g. + QuestDB does not advertise OIDC, the IdP device-authorization endpoint + cannot be discovered, or a required argument is missing). """ @@ -58,21 +54,16 @@ class OidcNetworkError(OidcError): class OidcInteractionRequired(OidcError): """ - Interactive sign-in is required but the process is not interactive. - - This is raised instead of hanging forever when the device flow is - started from a context with no human to authorize it (e.g. a - ``papermill`` run, a cron job or CI). Use a QuestDB service-account - REST token or the OAuth2 client-credentials grant in those contexts. + Interactive sign-in is required, but raised instead of hanging in a + non-interactive context (``papermill``, cron, CI). Use a QuestDB + service-account REST token or the OAuth2 client-credentials grant there. """ class OidcDeviceFlowError(OidcError): """ - The OAuth 2.0 device authorization grant failed. - - The original IdP ``error``/``error_description`` are preserved on the - exception when available. + The OAuth 2.0 device authorization grant failed; the IdP + ``error``/``error_description`` are preserved when available. """ def __init__( @@ -92,9 +83,7 @@ class OidcTimeoutError(OidcDeviceFlowError): class OidcAuthError(OidcError): """ - QuestDB rejected the token we presented. - - Typically a ``401``/``403`` from the server. The message includes hints - about the most common causes (scope / ``groups.encoded.in.token`` / + QuestDB rejected the token (typically a ``401``/``403`` from the server); + the message hints at common causes (scope / ``groups.encoded.in.token`` / ``audience`` mismatches). """ diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index f43afeca..a07444ed 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -23,17 +23,16 @@ ################################################################################ """ -A tiny HTTP helper built on the standard library. +A tiny stdlib-only HTTP helper. -OIDC device flow implementation deliberately avoids a hard dependency on ``requests``/``httpx`` -so that ``OidcDeviceAuth.token()`` / ``headers()`` work out of the box with no -extra installs. Only the device flow, discovery and the REST adapter use this -module; the heavier adapters (SQLAlchemy / psycopg / ingestion ``Sender``) bring -their own transports. +Avoids a hard dependency on ``requests``/``httpx`` so ``OidcDeviceAuth.token()`` +/ ``headers()`` work with no extra installs. Only the device flow, discovery and +the REST adapter use this; heavier adapters (SQLAlchemy / psycopg / ingestion +``Sender``) bring their own transports. -Standard proxy environment variables (``HTTPS_PROXY`` / ``HTTP_PROXY`` / -``NO_PROXY``) are honoured automatically by ``urllib``. A custom CA bundle can be -supplied explicitly or via ``REQUESTS_CA_BUNDLE`` / ``SSL_CERT_FILE``. +``urllib`` honours the standard proxy env vars (``HTTPS_PROXY`` / ``HTTP_PROXY`` +/ ``NO_PROXY``); a custom CA bundle can come from ``REQUESTS_CA_BUNDLE`` / +``SSL_CERT_FILE``. """ from __future__ import annotations @@ -56,9 +55,8 @@ def build_ssl_context(ca_bundle: Optional[str] = None) -> ssl.SSLContext: """ - Build an SSL context, honouring an explicit CA bundle or the - ``REQUESTS_CA_BUNDLE`` / ``SSL_CERT_FILE`` environment variables - (useful behind a corporate TLS-intercepting proxy). + Build an SSL context from an explicit CA bundle or the ``REQUESTS_CA_BUNDLE`` + / ``SSL_CERT_FILE`` env vars (useful behind a TLS-intercepting proxy). """ ca = ( ca_bundle @@ -66,10 +64,8 @@ def build_ssl_context(ca_bundle: Optional[str] = None) -> ssl.SSLContext: or os.environ.get('SSL_CERT_FILE')) if not ca: return ssl.create_default_context() - # A missing / unreadable / invalid bundle makes the stdlib raise a raw - # FileNotFoundError or ssl.SSLError; map it to the package's typed error so - # a mistyped ca_bundle path (or env var) fails clearly instead of leaking a - # bare stdlib exception. + # Map the raw FileNotFoundError / ssl.SSLError from a missing/invalid bundle + # to a typed error so a mistyped path fails clearly. try: if os.path.isdir(ca): return ssl.create_default_context(capath=ca) @@ -104,13 +100,12 @@ def ok(self) -> bool: def safe_urlparse(url: str) -> tuple: """ - ``urllib.parse.urlparse(url)`` paired with its port, but with a typed error. + ``urlparse(url)`` paired with its port, but with a typed error. - Both ``urlparse`` itself (e.g. ``https://[::1`` — a malformed IPv6 literal) - and ``ParseResult.port`` (e.g. ``https://idp:notaport`` — a non-integer - port) raise a bare ``ValueError``; re-raise it as :class:`OidcConfigError` - so a malformed endpoint URL stays within the package's error contract - instead of escaping as a raw ``ValueError``. Returns ``(parts, port)``. + Both ``urlparse`` (malformed IPv6 literal) and ``ParseResult.port`` + (non-integer port) raise a bare ``ValueError``; re-raise as + :class:`OidcConfigError` to keep a malformed URL within the error contract. + Returns ``(parts, port)``. """ try: parts = urllib.parse.urlparse(url) @@ -121,8 +116,7 @@ def safe_urlparse(url: str) -> tuple: def _is_loopback(host: Optional[str]) -> bool: - # Traffic to a loopback address never leaves the host, so plaintext http - # carries no network interception risk and is always permitted. + # Loopback traffic never leaves the host, so plaintext http is safe here. if not host: return False if host.lower() == 'localhost': @@ -134,8 +128,7 @@ def _is_loopback(host: Optional[str]) -> bool: def _require_secure(url: str, insecure: bool) -> None: - # safe_urlparse maps a malformed URL (bad IPv6 literal / non-integer port) - # to OidcConfigError instead of letting a bare ValueError escape. + # safe_urlparse maps a malformed URL to OidcConfigError, not a bare ValueError. parts, _ = safe_urlparse(url) scheme = parts.scheme.lower() if scheme == 'https': @@ -154,19 +147,16 @@ def _require_secure(url: str, insecure: bool) -> None: class _NoRedirect(urllib.request.HTTPRedirectHandler): """Refuse to follow HTTP redirects. - The discovery / device / token / ``/settings`` / ``/exec`` endpoints never - legitimately redirect. Auto-following a ``30x`` is unsafe here because only - the *original* URL is vetted: ``_require_secure`` and - ``validate_endpoint_origins`` never see the redirect target. urllib also - does not strip the ``Authorization`` header on a cross-origin redirect, so a - single ``302`` from ``/exec`` would re-send ``Authorization: Bearer - `` to an attacker-chosen host — including a downgrade to plaintext - ``http`` — leaking the QuestDB token off-origin. - - Returning ``None`` makes urllib stop following and surface the ``30x`` as an - ``HTTPError`` (which :func:`request` turns into a non-2xx - :class:`HttpResponse`), so callers see a clean failure instead of a - silently-followed redirect. + These endpoints never legitimately redirect, and auto-following is unsafe: + only the *original* URL is vetted (``_require_secure`` / + ``validate_endpoint_origins`` never see the target), and urllib does not + strip ``Authorization`` on a cross-origin redirect — so one ``302`` from + ``/exec`` could re-send the bearer token to an attacker host, even + downgrading to plaintext ``http``. + + Returning ``None`` surfaces the ``30x`` as an ``HTTPError`` (which + :func:`request` turns into a non-2xx :class:`HttpResponse`), giving callers a + clean failure. """ def redirect_request(self, *args, **kwargs): @@ -174,9 +164,8 @@ def redirect_request(self, *args, **kwargs): def _opener(ctx: Optional[ssl.SSLContext]) -> urllib.request.OpenerDirector: - # build_opener keeps the default ProxyHandler (which reads *_PROXY env - # vars), while letting us pin our own TLS context and forbid redirects - # (the credential/token endpoints never legitimately redirect). + # build_opener keeps the default ProxyHandler (reads *_PROXY env vars) while + # letting us pin our own TLS context and forbid redirects. handlers: list = [_NoRedirect()] if ctx is not None: handlers.append(urllib.request.HTTPSHandler(context=ctx)) @@ -196,11 +185,11 @@ def request( """ Perform a single HTTP request. - ``form`` is form-url-encoded into the body (``application/x-www-form- - urlencoded``). HTTP error statuses (``4xx``/``5xx``) are returned as an - :class:`HttpResponse` rather than raised, so callers can inspect OAuth - error bodies (e.g. ``authorization_pending``). Only genuine network - failures raise (:class:`OidcNetworkError`). + ``form`` is encoded into the body as ``application/x-www-form-urlencoded``. + HTTP error statuses (``4xx``/``5xx``) are returned as an + :class:`HttpResponse`, not raised, so callers can inspect OAuth error bodies + (e.g. ``authorization_pending``); only genuine network failures raise + (:class:`OidcNetworkError`). """ _require_secure(url, insecure) body: Optional[bytes] = data @@ -221,10 +210,9 @@ def request( resp.read(), resp.headers) except urllib.error.HTTPError as e: - # 4xx/5xx still carry a (possibly JSON) body we want to inspect. - # Map a mid-body read failure to a network error (rather than letting a - # bare OSError escape) and close the error response so its socket isn't - # leaked (the poll loop drives many 400s during a long sign-in). + # 4xx/5xx still carry a (possibly JSON) body to inspect. Map a mid-body + # read failure to a network error, and close the response so its socket + # isn't leaked (the poll loop drives many 400s during a long sign-in). try: body = e.read() except (TimeoutError, OSError) as read_err: @@ -236,9 +224,8 @@ def request( except urllib.error.URLError as e: raise OidcNetworkError(f'Failed to reach {url}: {e.reason}') from e except http.client.InvalidURL as e: - # A malformed URL (e.g. a non-integer port) can't be turned into a - # request; surface it as a config error rather than letting a raw - # http.client exception escape the package's typed-error contract. + # A malformed URL (e.g. non-integer port) can't become a request; + # surface it as a config error, not a raw http.client exception. raise OidcConfigError(f'Malformed URL {url!r}: {e}') from e except (TimeoutError, OSError, http.client.HTTPException) as e: raise OidcNetworkError(f'Failed to reach {url}: {e}') from e @@ -261,8 +248,8 @@ def get_json( try: return resp.json() except (ValueError, UnicodeDecodeError, RecursionError) as e: - # RecursionError: deeply-nested JSON exhausts the decoder's stack; it is - # not a ValueError, so catch it explicitly to keep the typed contract. + # RecursionError (deeply-nested JSON) isn't a ValueError, so catch it + # explicitly to keep the typed contract. raise OidcError(f'Invalid JSON from {url}: {e}') from e @@ -278,7 +265,7 @@ def post_form( POST a form-url-encoded body and parse the JSON response. Returns ``(status, parsed_json)``. Used for the device-authorization and - token endpoints, which return JSON bodies on both success and error. + token endpoints, which return JSON on both success and error. """ resp = request( 'POST', url, form=form, headers=headers, timeout=timeout, ctx=ctx, @@ -286,16 +273,14 @@ def post_form( try: parsed = resp.json() except (ValueError, UnicodeDecodeError, RecursionError): - # RecursionError: deeply-nested JSON exhausts the decoder's stack; not a - # ValueError, so catch it explicitly to keep the typed contract. + # RecursionError (deeply-nested JSON) isn't a ValueError, so catch it + # explicitly to keep the typed contract. if resp.ok: raise OidcError( f'Expected JSON from {url}, got: {resp.text()[:200]}', status=resp.status) - # Non-JSON error body: surface the status + text. Attach the HTTP status - # so callers (the device-flow poll loop / silent refresh) can tell a - # terminal 4xx rejection from a transient 5xx/429 even though the body - # was not a conformant JSON OAuth error. + # Non-JSON error body: attach the HTTP status so callers (poll loop / + # silent refresh) can tell a terminal 4xx from a transient 5xx/429. raise OidcError( f'HTTP {resp.status} from {url}: {resp.text()[:200]}', status=resp.status) diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py index 76a6979b..2cb7da05 100644 --- a/src/questdb/auth/_questdb.py +++ b/src/questdb/auth/_questdb.py @@ -38,12 +38,10 @@ _DEFAULT_PG_PORT = 8812 _DEFAULT_DATABASE = 'qdb' -# A hostname or IP literal never contains the ILP conf-string delimiters (';' -# separates parameters, '=' separates key from value) nor whitespace/control -# characters. Reject them in the resolved host so a crafted or tampered URL -# can't smuggle extra conf parameters — e.g. ';tls_verify=unsafe_off;', which -# silently disables TLS certificate verification — into the 'addr=host:port;' -# string sender() hands to Sender.from_conf. Note ':' is intentionally allowed +# Reject ILP conf-string delimiters (';', '=') and whitespace/control chars in +# the host: a real hostname/IP never has them, so their presence means a +# tampered URL trying to inject conf params like ';tls_verify=unsafe_off;' +# (which disables TLS verification) into the addr= string. ':' is allowed # (IPv6 literals contain it; _ilp_addr brackets them). _ILLEGAL_HOST_CHARS = re.compile(r'[\x00-\x20\x7f;=]') @@ -71,12 +69,10 @@ def _import_pandas(): def _exec_json_to_df(data: Dict[str, Any], pandas): columns = data.get('columns') or [] - # /exec returns a list of {"name", "type"} column descriptors. A malformed - # response — a non-list, entries that aren't objects, or a non-string name — - # must surface as a clean OidcError, not a raw AttributeError from .get(), - # nor a TypeError from `name in df.columns` below when a name is - # non-hashable (a JSON list/object), escaping the package's typed-error - # contract. A real QuestDB column name is always a string. + # /exec returns a list of {"name", "type"} descriptors. Validate the shape + # (a real column name is always a string) so a malformed response raises a + # clean OidcError rather than a raw AttributeError from .get() or a + # TypeError from `name in df.columns` on a non-hashable name. if not isinstance(columns, list) or not all( isinstance(c, dict) and isinstance(c.get('name'), (str, type(None))) @@ -91,8 +87,8 @@ def _exec_json_to_df(data: Dict[str, Any], pandas): try: df = pandas.DataFrame(dataset, columns=names or None) except (ValueError, TypeError) as e: - # TypeError too: a hostile/malformed dataset shape can make the pandas - # constructor raise it (not only ValueError); keep it within OidcError. + # A malformed dataset shape can make the pandas constructor raise + # ValueError or TypeError; keep both within OidcError. raise OidcError( f'Unexpected shape in QuestDB /exec response: {e}') from e for col in columns: @@ -124,10 +120,9 @@ class QuestDB: """ A thin, authenticated QuestDB session built on an :class:`OidcDeviceAuth`. - Provides a one-call DataFrame query over REST plus adapters that feed the - same auto-refreshed token into your existing tools (SQLAlchemy / psycopg / - the ingestion ``Sender``). You can also just take :meth:`token` / - :meth:`headers` and wire them up yourself. + Offers a one-call DataFrame query over REST plus adapters that feed the + same auto-refreshed token into SQLAlchemy / psycopg / the ingestion + ``Sender``, or take :meth:`token` / :meth:`headers` and wire it up yourself. """ def __init__( @@ -141,12 +136,11 @@ def __init__( self._insecure = insecure self._ctx = auth._ctx # Same private CA bundle the auth/REST transport uses, so sender() can - # forward it to the ILP Sender (which has its own TLS stack). getattr - # keeps test doubles that only set _ctx working. + # forward it to the ILP Sender's own TLS stack. getattr keeps test + # doubles that only set _ctx working. self._ca_bundle = getattr(auth, '_ca_bundle', None) # safe_urlparse validates the port up-front, raising OidcConfigError - # (not a bare ValueError) for a malformed one, so the adapters that read - # the port stay within the package's typed-error contract. + # (not a bare ValueError) for a malformed one. self._parts, self._port = safe_urlparse(self.url) # -- token access ------------------------------------------------------- @@ -168,7 +162,7 @@ def sql(self, query: str, *, limit: Optional[str] = None, :class:`pandas.DataFrame`. Uses ``Authorization: Bearer`` (no token-length limit, unlike PG-wire), - which makes it the recommended path for large groups-encoded JWTs. + so it's the recommended path for large groups-encoded JWTs. :param query: The SQL query to run. :param limit: Optional QuestDB ``limit`` (e.g. ``"1,1000"``). @@ -195,17 +189,15 @@ def sql(self, query: str, *, limit: Optional[str] = None, try: data = resp.json() except (ValueError, UnicodeDecodeError, RecursionError): - # A 2xx body that isn't JSON (e.g. an HTML error/login page from a - # reverse proxy or captive portal), or deeply-nested JSON that - # exhausts the decoder's stack (RecursionError, not a ValueError), - # must surface as a clean OidcError, not a raw decoder exception. - # Mirrors the error path and post_form(). + # A 2xx body that isn't JSON (e.g. an HTML page from a proxy/captive + # portal) or deeply-nested JSON that exhausts the decoder's stack + # (RecursionError) surfaces as a clean OidcError. Mirrors post_form(). raise OidcError( 'QuestDB returned a non-JSON success response from /exec: ' f'{resp.text()[:300]}') if not isinstance(data, dict): - # Valid JSON but not an object (e.g. a bare list) would make - # _exec_json_to_df fail with AttributeError on .get(); reject it. + # Valid JSON but not an object (e.g. a bare list) would break + # _exec_json_to_df's .get(); reject it. raise OidcError( 'QuestDB /exec returned JSON that is not an object ' f'(got {type(data).__name__}); cannot build a DataFrame.') @@ -216,13 +208,13 @@ def sql(self, query: str, *, limit: Optional[str] = None, def _require_host(self, host: Optional[str] = None) -> str: """ Resolve the PG-wire / ILP host: an explicit ``host`` override, else the - host from the QuestDB URL. Raises when neither yields one (e.g. a URL - with no authority such as ``"localhost"`` or ``"questdb:9000"``) instead - of passing a bare ``None`` down to the driver. + host from the QuestDB URL. Raises (rather than passing a bare ``None`` + to the driver) when neither yields one, e.g. a URL with no authority + such as ``"localhost"`` or ``"questdb:9000"``. - The returned host is *unbracketed* — psycopg and SQLAlchemy take the - address and port as separate arguments. :meth:`_ilp_addr` adds the - brackets an IPv6 literal needs in the ILP ``addr=host:port`` form. + The returned host is *unbracketed* — psycopg and SQLAlchemy take address + and port separately. :meth:`_ilp_addr` adds the brackets an IPv6 literal + needs in the ILP ``addr=host:port`` form. """ resolved = host or self._parts.hostname if not resolved: @@ -242,8 +234,8 @@ def _require_host(self, host: Optional[str] = None) -> str: @staticmethod def _ilp_addr(host: str, port: int) -> str: - # Bracket an IPv6 literal so the ILP conf parser reads host:port - # unambiguously; hostnames and IPv4 addresses never contain ':'. + # Bracket an IPv6 literal (it contains ':', unlike hostnames/IPv4) so + # the ILP conf parser reads host:port unambiguously. bracketed = f'[{host}]' if ':' in host else host return f'{bracketed}:{port}' @@ -258,8 +250,8 @@ def sqlalchemy_engine( """ Build a SQLAlchemy ``Engine`` for QuestDB's PG-wire endpoint. - Connects as user ``_sso`` and injects a **fresh** token as the password - for every new connection (via a ``do_connect`` listener), so pooled + Connects as user ``_sso``, injecting a **fresh** token as the password + on every new connection (via a ``do_connect`` listener) so pooled connections always authenticate with a valid token. Requires ``acl.oidc.pg.token.as.password.enabled=true`` on the server. """ @@ -305,8 +297,8 @@ def psycopg( Open a raw psycopg (v3) or psycopg2 connection to QuestDB's PG-wire endpoint, authenticating as ``_sso`` with the current token. - The token is captured at connect time; open a new connection to pick up - a refreshed token. + The token is captured at connect time; reconnect to pick up a refreshed + token. """ mod = _pg_module() return mod.connect( @@ -320,8 +312,8 @@ def psycopg( def sender(self, *, port: Optional[int] = None, **sender_kwargs) -> 'questdb.ingress.Sender': """ - Build a :class:`questdb.ingress.Sender` (ILP-over-HTTP) configured with - the current bearer token, for ingestion. + Build a :class:`questdb.ingress.Sender` (ILP-over-HTTP) for ingestion, + configured with the current bearer token. The token is captured at creation time; create a new sender to pick up a refreshed token. @@ -330,8 +322,8 @@ def sender(self, *, port: Optional[int] = None, resolved_port = port or self._port or ( 443 if scheme == 'https' else 9000) # Coerce to int (before the heavy import, so bad input fails fast) so a - # stray non-integer port kwarg can't smuggle ILP conf parameters — e.g. - # "9000;tls_verify=unsafe_off" — into the addr= string via _ilp_addr, + # stray non-integer port can't inject conf params like + # "9000;tls_verify=unsafe_off" into the addr= string via _ilp_addr — # the same injection _require_host() blocks for the host. The # URL-derived self._port is already an int. try: @@ -353,11 +345,10 @@ def sender(self, *, port: Optional[int] = None, f'{self._ilp_addr(self._require_host(), resolved_port)};') # Forward the private CA bundle (explicit ca_bundle=, else the # REQUESTS_CA_BUNDLE / SSL_CERT_FILE env vars — same precedence as - # build_ssl_context) to the Sender's own TLS stack as tls_roots, so an - # https Sender against a private-CA QuestDB trusts the same roots the - # REST/IdP paths do. Only a PEM file works here (tls_roots is a file; - # the Sender has no capath equivalent), and only over https. The caller - # can still override via tls_roots=/tls_ca= in **sender_kwargs. + # build_ssl_context) to the Sender as tls_roots, so an https Sender + # against a private-CA QuestDB trusts the same roots the REST/IdP paths + # do. Only a PEM file works (tls_roots takes a file, no capath), only + # over https, and the caller can still override via tls_roots=/tls_ca=. if (scheme == 'https' and 'tls_roots' not in sender_kwargs and 'tls_ca' not in sender_kwargs): @@ -395,17 +386,16 @@ def connect( :param url: The QuestDB HTTP(S) base URL, e.g. ``"https://questdb.example.com:9000"``. :param flow: ``"auto"`` (default), ``"device"`` or ``"loopback"``. Today - ``"auto"`` always resolves to the device flow (works on local and - remote kernels); ``"loopback"`` is reserved for a future release. + ``"auto"`` resolves to the device flow (works on local and remote + kernels); ``"loopback"`` is reserved for a future release. :param cache: Token cache backend: ``"memory"`` (default) or ``None``. :param insecure: Allow plaintext ``http://`` URLs (development only). :param eager: If ``True`` (default), sign in immediately; otherwise defer until the first call that needs a token. :param opts: Forwarded to :meth:`OidcDeviceAuth.from_questdb` (e.g. ``client_id``, ``scope``, ``audience``, ``issuer``, ``open_browser``, - ``qr``, ``ca_bundle``, ``timeout`` — the per-request IdP network - timeout, which also bounds how long a stalled IdP can hold the - token-acquisition lock). + ``qr``, ``ca_bundle``, ``timeout`` — the per-request IdP network timeout, + which also bounds how long a stalled IdP can hold the token lock). """ auth = OidcDeviceAuth.from_questdb( url, flow=flow, cache=cache, insecure=insecure, **opts) diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 9263b556..85dc29c0 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -25,10 +25,9 @@ """ Presentation of the device-flow prompt. -Renders a clickable link + user code in Jupyter (via ``IPython.display``) and -falls back to plain text on a terminal. Nothing here is required for -``token()`` / ``headers()`` to work; ``IPython`` and ``qrcode`` are imported -lazily and only when actually used. +Renders a clickable link + user code in Jupyter (via ``IPython.display``), +falling back to plain text on a terminal. Not required for ``token()`` / +``headers()``; ``IPython`` and ``qrcode`` are imported lazily. """ from __future__ import annotations @@ -49,8 +48,8 @@ def in_ipython_kernel() -> bool: ip = get_ipython() if ip is None: return False - # ZMQInteractiveShell == notebook / qtconsole / lab; TerminalInteractive - # Shell == ipython in a terminal (still interactive). + # ZMQInteractiveShell == notebook/qtconsole/lab; TerminalInteractiveShell + # == ipython in a terminal. return ip.__class__.__name__ in ( 'ZMQInteractiveShell', 'TerminalInteractiveShell') @@ -59,9 +58,8 @@ def detect_interactive() -> bool: """ Best-effort detection of whether a human can complete the sign-in. - Interactive when attached to a TTY or running in an interactive IPython - shell. This guards against hanging forever in a non-interactive context - (papermill / cron / CI). + Interactive when attached to a TTY or an interactive IPython shell; guards + against hanging forever in a non-interactive context (papermill/cron/CI). """ if in_ipython_kernel(): return True @@ -74,17 +72,14 @@ def detect_interactive() -> bool: def _verification_uri(resp: Dict[str, Any]) -> str: # RFC 8628 uses ``verification_uri``; some IdPs (older Google) use - # ``verification_url``. The device response is untrusted: coerce to str so a - # non-string value (e.g. a JSON number) can't crash the renderer - # (``re.sub`` / ``html.escape``) with a raw TypeError before the prompt is - # even shown — matching the defensive ``str(user_code)`` at the call sites. + # ``verification_url``. Coerce to str: the device response is untrusted, and + # a non-string (e.g. a JSON number) would crash the renderer. uri = resp.get('verification_uri') or resp.get('verification_url') or '' return uri if isinstance(uri, str) else '' def _verification_uri_complete(resp: Dict[str, Any]) -> Optional[str]: - # Coerce to str / None for the same untrusted-input reason as - # _verification_uri (a non-string would crash the renderer / _safe_link_url). + # Coerce to str/None for the same untrusted-input reason as _verification_uri. uri = (resp.get('verification_uri_complete') or resp.get('verification_url_complete')) return uri if isinstance(uri, str) else None @@ -94,15 +89,13 @@ def _safe_link_url(url: Optional[str]) -> Optional[str]: """ Return ``url`` only if it uses an ``http(s)`` scheme, else ``None``. - The verification URL comes from the IdP's device-authorization response, - which is untrusted input. Embedding it in an HTML ``href`` without a scheme - allowlist would let a malicious/MITM'd response inject a ``javascript:`` or - ``data:`` URL that executes in the notebook DOM when clicked - (``html.escape`` guards markup, not the URL scheme). + The verification URL is untrusted (from the IdP's device-authorization + response); the scheme allowlist blocks a ``javascript:`` / ``data:`` href + from executing in the notebook DOM (``html.escape`` guards markup, not the + scheme). """ if not url or not isinstance(url, str): - # A non-string (e.g. a JSON number from an untrusted device response) - # has no scheme to vet and would make urlparse raise; treat it as unsafe. + # A non-string has no scheme to vet and would make urlparse raise. return None try: scheme = urllib.parse.urlparse(url).scheme.lower() @@ -116,9 +109,8 @@ def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: Render ``url`` as a clickable link, or as inert escaped text if its scheme is not ``http(s)``. - The visible label defaults to the URL itself. When the URL is rejected, the - (escaped) URL is shown as plain text so the user can still see/copy it, but - it is never turned into a clickable/executable link. + The label defaults to the URL itself. A rejected URL is shown as escaped + plain text (still visible/copyable) but never made clickable. """ safe = _safe_link_url(url) label = html.escape(text if text is not None else (url or '')) @@ -128,14 +120,10 @@ def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: f'rel="noopener noreferrer">{label}') -# C0/C1 control chars (incl. ESC, which drives ANSI escape sequences), the -# Unicode bidi controls, the zero-width / invisible-format chars, the -# line/paragraph separators and the interlinear-annotation controls. All can -# spoof a prompt: U+202E (RIGHT-TO-LEFT OVERRIDE) reverses displayed text to -# disguise a URL's host; U+2028/U+2029 inject fake lines; zero-width / invisible -# chars hide or join content. Stripped from untrusted device-response fields on -# BOTH the terminal and the Jupyter path (html.escape neutralizes markup, not -# these). Covers the dangerous Unicode Cc/Cf code points for our inputs. +# Strips C0/C1/ESC, bidi overrides, zero-width and line/paragraph separators +# — all can spoof the prompt (e.g. U+202E reverses a URL's host). Applied to +# untrusted device-response fields on both paths; html.escape would not catch +# these. _CONTROL_CHARS = re.compile( r'[\x00-\x1f\x7f-\x9f\u00ad\u061c\u115f\u180e\u200b-\u200f' r'\u2028-\u202e\u2060-\u2064\u2066-\u2069\ufeff\ufff9-\ufffb]') @@ -143,17 +131,12 @@ def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: def _strip_control(text: Optional[str]) -> str: """ - Strip control / format characters from an untrusted string before it is - written to a terminal. - - The verification URL, user code and IdP error strings come from the device- - authorization response (untrusted). Writing them verbatim to a TTY would let - a hostile or MITM'd response inject ANSI escape sequences (C0/C1 control - chars — cursor moves, screen clears) or Unicode bidi overrides / zero-width - / line separators to spoof the prompt or hide the real sign-in URL (e.g. - U+202E visually reverses the displayed host). Needed on BOTH paths: the - plain-text terminal path (raw bytes to the TTY) and the Jupyter path — - ``html.escape`` neutralizes markup, not bidi/zero-width spoofing. + Strip control / format characters from an untrusted string before display. + + The verification URL, user code and IdP error strings are untrusted; raw + ANSI escapes or bidi/zero-width/line-separator chars could spoof the prompt + or hide the real sign-in URL. Needed on both paths — ``html.escape`` does + not catch bidi/zero-width spoofing. """ if not text: return '' @@ -208,12 +191,10 @@ def _write(self, text: str) -> None: try: self._stream.write(text) except UnicodeEncodeError: - # The stream's encoding can't represent some characters (e.g. - # the emoji on a legacy code-page Windows console, an ``ascii`` - # PYTHONIOENCODING, or a redirected stderr). Degrade only those - # characters instead of letting the whole prompt — including the - # verification URL and user code — vanish, which would make the - # sign-in look like a silent hang. + # The stream's encoding can't represent some chars (e.g. the + # emoji on a legacy Windows console or ascii PYTHONIOENCODING). + # Degrade only those, so the URL/code don't vanish and look like + # a silent hang. enc = getattr(self._stream, 'encoding', None) or 'ascii' self._stream.write( text.encode(enc, 'replace').decode(enc, 'replace')) @@ -272,16 +253,12 @@ def _panel(self, body: str) -> str: def _prompt_head(self): """Header + sanitized verification link and user code. - Shared by :meth:`on_prompt` and :meth:`_render_with_status` so the - sanitization can't be applied to one path and forgotten on the other. - ``verification_uri`` / ``user_code`` / ``verification_uri_complete`` are - untrusted device-response fields: strip control / bidi / zero-width - chars (which ``html.escape`` does NOT remove) before rendering, so a - hostile or MITM'd response can't inject a U+202E bidi override or - zero-width chars to visually spoof the prompt in the notebook DOM. - ``_render_link`` additionally html-escapes and scheme-vets the URL. - Returns ``(body, uri, complete)`` — the sanitized URLs are handed back - so the QR target isn't re-derived (and re-sanitized). + Shared by :meth:`on_prompt` and :meth:`_render_with_status` so + sanitization is applied on both paths, never forgotten on one. The + untrusted device-response fields are stripped of control/bidi/zero-width + chars (which ``html.escape`` does NOT remove) before rendering; + ``_render_link`` also html-escapes and scheme-vets the URL. Returns + ``(body, uri, complete)`` so the QR target isn't re-derived. """ resp = self._resp uri = _strip_control(_verification_uri(resp)) @@ -327,8 +304,7 @@ def on_waiting(self, seconds_left: float) -> None: color='#888') def on_success(self, identity: Optional[str], expires_in: float) -> None: - # identity is derived from the (untrusted) JWT claims — strip control / - # bidi chars before html-escaping, as for the other rendered fields. + # identity comes from untrusted JWT claims: strip then html-escape. who = html.escape(_strip_control(identity)) if identity else '' mins = max(1, int(round(expires_in / 60))) suffix = f' as {who}' if who else '' @@ -337,7 +313,7 @@ def on_success(self, identity: Optional[str], expires_in: float) -> None: color='#2e7d32') def on_failure(self, message: str) -> None: - # message may interpolate the IdP's (untrusted) error_description. + # message may interpolate the IdP's untrusted error_description. self._render_with_status( '❌ ' + html.escape(_strip_control(message)), color='#c62828') From 73e3c5fa372467b3ad59bd213bc88aeb20917830 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 10:11:04 +0100 Subject: [PATCH 040/104] fix: default groups_in_token to False to match the QuestDB server The OIDC token-kind default assumed groups were encoded in the token (send the id_token). The QuestDB server default for acl.oidc.groups.encoded.in.token is False, and the Java client's builder defaults to false; align the Python client with both. Flip the default in all three places it was set: - the OidcDeviceAuth(...) constructor signature - the OidcConfig dataclass field - the discovery fallback when /settings omits the key (the behaviourally significant one: from_questdb against a server that doesn't advertise the key now sends the access_token, not the id_token) Document the default in docs/auth.rst and add two regression tests (bare constructor and discovery-with-key-absent both resolve to the access_token). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/auth.rst | 5 ++++- src/questdb/auth/_device.py | 2 +- src/questdb/auth/_discovery.py | 4 ++-- test/test_auth.py | 31 +++++++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/docs/auth.rst b/docs/auth.rst index fbef0b3e..150ce917 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -135,7 +135,10 @@ The helper mirrors QuestDB's own selection logic ``false`` ``access_token`` ============================================ ================= -When sending the ``id_token`` the ``openid`` scope is requested automatically. +When neither the server's ``/settings`` nor an explicit ``groups_in_token=`` +specifies it, the helper defaults to ``False`` (send the ``access_token``), +mirroring the QuestDB server default. When sending the ``id_token`` the +``openid`` scope is requested automatically. Token lifecycle (cache + refresh) --------------------------------- diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index a3db901a..d1df02a5 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -179,7 +179,7 @@ def __init__( token_endpoint: str, *, scope: str = 'openid', - groups_in_token: bool = True, + groups_in_token: bool = False, audience: Optional[str] = None, issuer: Optional[str] = None, cache: Any = 'memory', diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 68fe7301..3ddab02d 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -65,7 +65,7 @@ class OidcConfig: token_endpoint: str device_authorization_endpoint: str scope: str = 'openid' - groups_in_token: bool = True + groups_in_token: bool = False audience: Optional[str] = None issuer: Optional[str] = None authorization_endpoint: Optional[str] = None @@ -383,7 +383,7 @@ def resolve_config( if scope is None: scope = _str_setting(cfg.get(_K_SCOPE)) or 'openid' if groups_in_token is None: - groups_in_token = _as_bool(cfg.get(_K_GROUPS_IN_TOKEN), default=True) + groups_in_token = _as_bool(cfg.get(_K_GROUPS_IN_TOKEN), default=False) if audience is None: audience = _str_setting(cfg.get(_K_AUDIENCE)) diff --git a/test/test_auth.py b/test/test_auth.py index 1a3afb96..a215388a 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -356,6 +356,20 @@ def test_access_token_when_groups_not_in_token(self): auth = self.make_auth(groups_in_token=False) self.assertEqual(auth.token(), ACCESS_TOKEN) + def test_constructor_defaults_to_access_token(self): + # The bare constructor default is groups_in_token=False (send the + # access_token), matching the QuestDB server default. + auth = OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint=self.base + '/device', + token_endpoint=self.base + '/token', + insecure=True, + interactive=True, + renderer=Renderer(), + _clock=FakeClock()) + self.assertFalse(auth.config.groups_in_token) + self.assertEqual(auth.token(), ACCESS_TOKEN) + def test_headers(self): auth = self.make_auth() self.assertEqual(auth.headers(), @@ -938,6 +952,23 @@ def test_from_questdb_reads_settings(self): self.base + '/device') self.assertEqual(auth.token(), ID_TOKEN) + def test_groups_mode_defaults_to_access_token_when_unset(self): + # /settings omits acl.oidc.groups.encoded.in.token: the helper mirrors + # the QuestDB server default (groups NOT encoded in the token) and sends + # the access_token rather than the id_token. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.scope': 'openid', + 'acl.oidc.token.endpoint': self.base + '/token', + 'acl.oidc.device.authorization.endpoint': self.base + '/device', + }} + auth = OidcDeviceAuth.from_questdb( + self.base, insecure=True, interactive=True, renderer=Renderer(), + _clock=FakeClock()) + self.assertFalse(auth.config.groups_in_token) + self.assertEqual(auth.token(), ACCESS_TOKEN) + def test_user_writable_preferences_cannot_override_config(self): # A user-writable "preferences" sibling in /settings must never override # the trusted "config" object during discovery: end-to-end, the resolved From b23e5db08f680d6d2854cb5c1cdfd12c8c3cafe3 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 11:36:42 +0100 Subject: [PATCH 041/104] refactor(auth): replace the QuestDB session with PG-wire adapter functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The questdb.auth module shipped a batteries-included data layer: connect() returned a QuestDB session with sql()->DataFrame (REST /exec), SQLAlchemy, psycopg and ingestion-Sender adapters. Trim it to a focused auth helper — get a token, plus two conveniences that wire it into PG-wire. - Remove connect(), the QuestDB session class, sql()/DataFrame and sender(); delete _questdb.py. - Add _adapters.py with two free functions, sqlalchemy_engine(auth, url) and psycopg_connect(auth, url), that inject the auto-refreshed token as the _sso password (SQLAlchemy refreshes per pooled connection via do_connect). For REST and ingestion, callers take headers()/token() and wire it up themselves. - Drop OidcAuthError: its only raise site was the deleted sql() adapter, so it was exported but never raised. - Update the tests, docs (auth/api/installation), the CHANGELOG and the runnable example to the token-first shape. Public surface: connect/QuestDB -> sqlalchemy_engine/psycopg_connect; 15 -> 14 exported names. Net -516 lines. Auth suite: 133 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.rst | 20 +- docs/api.rst | 14 +- docs/auth.rst | 109 ++++----- docs/installation.rst | 6 +- examples/oidc_device_auth.py | 80 ++++--- src/questdb/auth/__init__.py | 34 ++- src/questdb/auth/_adapters.py | 187 +++++++++++++++ src/questdb/auth/_device.py | 5 +- src/questdb/auth/_errors.py | 8 - src/questdb/auth/_questdb.py | 405 -------------------------------- test/test_auth.py | 428 +++++----------------------------- 11 files changed, 390 insertions(+), 906 deletions(-) create mode 100644 src/questdb/auth/_adapters.py delete mode 100644 src/questdb/auth/_questdb.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 672455bb..d56e18b3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,15 +25,14 @@ QuestDB over the auth paths it already supports (HTTP ``Bearer`` / PG-wire .. code-block:: python - from questdb.auth import OidcDeviceAuth, connect + from questdb.auth import OidcDeviceAuth, sqlalchemy_engine - # Just the token (use it with PG-wire, HTTP, or any client): + # Sign in once and get a valid, auto-refreshed token: auth = OidcDeviceAuth.from_questdb("https://questdb.example.com:9000") - token = auth.token() + token = auth.token() # use it with PG-wire, HTTP, or any client - # Or the integrated session (query to a DataFrame, feed adapters): - qdb = connect("https://questdb.example.com:9000") - df = qdb.sql("SELECT * FROM trades LIMIT 10") + # Or wire it into PG-wire as the _sso password: + engine = sqlalchemy_engine(auth, "https://questdb.example.com:9000") Highlights: @@ -41,11 +40,12 @@ Highlights: fallback to the IdP ``.well-known`` document. * In-process token cache with silent refresh (tokens are never written to disk). -* Adapters for pandas (REST ``/exec``), SQLAlchemy, psycopg and the ingestion - ``Sender``. +* Convenience adapters (:func:`~questdb.auth.sqlalchemy_engine`, + :func:`~questdb.auth.psycopg_connect`) that wire the auto-refreshed token into + PG-wire as the ``_sso`` password. * ``token()`` / ``headers()`` require no dependencies beyond the standard - library; ``pandas`` / ``sqlalchemy`` / ``psycopg`` / ``qrcode`` / ``IPython`` - are imported lazily. + library; ``sqlalchemy`` / ``psycopg`` / ``qrcode`` / ``IPython`` are imported + lazily. See the :ref:`OIDC authentication guide ` for details. diff --git a/docs/api.rst b/docs/api.rst index 9ff4daf3..54d7a654 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -73,18 +73,15 @@ questdb.auth See the :ref:`oidc_auth` guide for an overview. -.. autofunction:: questdb.auth.connect - -.. autoclass:: questdb.auth.QuestDB - :members: - :undoc-members: - :show-inheritance: - .. autoclass:: questdb.auth.OidcDeviceAuth :members: :undoc-members: :show-inheritance: +.. autofunction:: questdb.auth.sqlalchemy_engine + +.. autofunction:: questdb.auth.psycopg_connect + .. autoclass:: questdb.auth.OidcConfig :members: :undoc-members: @@ -125,8 +122,5 @@ See the :ref:`oidc_auth` guide for an overview. .. autoexception:: questdb.auth.OidcTimeoutError :show-inheritance: -.. autoexception:: questdb.auth.OidcAuthError - :show-inheritance: - .. autoexception:: questdb.auth.OidcNetworkError :show-inheritance: diff --git a/docs/auth.rst b/docs/auth.rst index 150ce917..2405b86a 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -34,8 +34,9 @@ it with your own tooling. Just the token (PG-wire / HTTP / anything) ------------------------------------------ -If you connect to QuestDB yourself — over PG-wire, raw HTTP, or any other -client — you only need a valid token. This path has **no extra dependencies**. +You sign in once and get a valid, auto-refreshed token; present it to QuestDB +over PG-wire, raw HTTP, or any other client. This path has **no extra +dependencies**. .. code-block:: python @@ -47,47 +48,52 @@ client — you only need a valid token. This path has **no extra dependencies**. token = auth.token() # runs the device flow on first use, else cached headers = auth.headers() # {"Authorization": "Bearer "} - # Use the token however you like, e.g. PG-wire via psycopg: - import psycopg - conn = psycopg.connect( - host="questdb.example.com", port=8812, dbname="qdb", - user="_sso", password=token) +On first use you will see a sign-in prompt (rendered as a clickable link in +Jupyter, plain text on a terminal):: + + 🔐 Sign in to QuestDB + Open https://idp.example.com/device and enter code: WDJB-MJHT + (or open directly: https://idp.example.com/device?user_code=WDJB-MJHT) + ⏳ waiting for authorization… (4:51 left) + ✅ Signed in as alice@example.com — token cached, expires in 60 min + +Re-running is silent — the token is cached and refreshed silently on the next +use once it nears expiry. -The integrated session ----------------------- +PG-wire adapters +---------------- -The high-level :func:`questdb.auth.connect` returns a :class:`~questdb.auth.QuestDB` -session that signs you in and adapts the token into the common Python access -paths. +For PG-wire there are two convenience adapters that inject the auto-refreshed +token as the QuestDB ``_sso`` password (they require +``acl.oidc.pg.token.as.password.enabled=true`` on the server): .. code-block:: python - from questdb.auth import connect + from questdb.auth import OidcDeviceAuth, sqlalchemy_engine, psycopg_connect - qdb = connect("https://questdb.example.com:9000") # interactive sign-in - df = qdb.sql("SELECT * FROM trades WHERE ts > dateadd('h', -1, now())") + url = "https://questdb.example.com:9000" + auth = OidcDeviceAuth.from_questdb(url) + auth.token() # sign in once up front, before the pool opens connections - # Bring-your-own client, same auto-refreshed token: - engine = qdb.sqlalchemy_engine() # PG-wire, token as _sso - with qdb.psycopg() as conn: # raw psycopg - ... + # SQLAlchemy: a fresh token is injected as the password on every new + # (pooled) connection, so the engine keeps working as the token rotates. + engine = sqlalchemy_engine(auth, url) - from questdb.ingress import TimestampNanos # the compiled extension - with qdb.sender() as sender: # ingestion (ILP/HTTP) - sender.row("trades", columns={"price": 101.5}, - at=TimestampNanos.now()) + # Or a raw psycopg / psycopg2 connection: + conn = psycopg_connect(auth, url) -On first use you will see a sign-in prompt (rendered as a clickable link in -Jupyter, plain text on a terminal):: +For REST or ingestion, take ``auth.headers()`` / ``auth.token()`` and wire it +into your HTTP client or the ingestion :class:`~questdb.ingress.Sender` +yourself: - 🔐 Sign in to QuestDB - Open https://idp.example.com/device and enter code: WDJB-MJHT - (or open directly: https://idp.example.com/device?user_code=WDJB-MJHT) - ⏳ waiting for authorization… (4:51 left) - ✅ Signed in as alice@example.com — token cached, expires in 60 min +.. code-block:: python + + from questdb.ingress import Sender, TimestampNanos -Re-running any cell is silent — the token is cached and refreshed silently on -the next use once it nears expiry. + with Sender.from_conf("https::addr=questdb.example.com:9000;", + token=auth.token()) as sender: + sender.row("trades", columns={"price": 101.5}, + at=TimestampNanos.now()) How it works ============ @@ -96,8 +102,7 @@ Configuration discovery ------------------------ :meth:`OidcDeviceAuth.from_questdb ` -(and :func:`~questdb.auth.connect`) resolve the OIDC configuration in this -order: +resolves the OIDC configuration in this order: 1. ``GET {url}/settings`` (public, no auth) for the QuestDB-authoritative values: ``acl.oidc.client.id``, ``acl.oidc.scope``, ``acl.oidc.token.endpoint``, @@ -171,23 +176,26 @@ authorize the device. The helper detects this and raises Connection adapters =================== -* :meth:`QuestDB.sql ` — query over REST ``/exec`` to a - pandas DataFrame using ``Authorization: Bearer``. Recommended: there is no - token-length limit (a groups-encoded JWT can be several KB). -* :meth:`QuestDB.sqlalchemy_engine ` — - PG-wire engine that injects a fresh token as the ``_sso`` password for every - new connection. Requires ``acl.oidc.pg.token.as.password.enabled=true``. -* :meth:`QuestDB.psycopg ` — a raw psycopg / - psycopg2 connection. -* :meth:`QuestDB.sender ` — a - :class:`~questdb.ingress.Sender` for ingestion (ILP over HTTP). +Two helpers wire the auto-refreshed token into PG-wire as the ``_sso`` password +(both require ``acl.oidc.pg.token.as.password.enabled=true``): + +* :func:`~questdb.auth.sqlalchemy_engine` — a SQLAlchemy ``Engine`` that injects + a fresh token for every new connection, so a pool keeps working as the token + rotates. +* :func:`~questdb.auth.psycopg_connect` — a raw psycopg / psycopg2 connection + (token captured at connect time). + +For REST (``Authorization: Bearer``) and ingestion (the +:class:`~questdb.ingress.Sender`), take +:meth:`~questdb.auth.OidcDeviceAuth.headers` / +:meth:`~questdb.auth.OidcDeviceAuth.token` and wire the token in yourself. .. note:: QuestDB validates the token at **authentication** time, not per query. An already-open PG connection survives token expiry; only **new** connections - need a fresh token — which is why the PG-wire adapter supplies the token - per-connect. + need a fresh token — which is why :func:`~questdb.auth.sqlalchemy_engine` + supplies the token per-connect. .. _oidc_idp_requirements: @@ -234,12 +242,8 @@ Security notes refuses to guess the discovery origin from the server-supplied token endpoint. * Adapters avoid logging the token / PG DSN. Avoid logging them yourself. * Standard proxy / CA settings (``HTTPS_PROXY``, ``REQUESTS_CA_BUNDLE``, - ``SSL_CERT_FILE``) are honoured; you can also pass ``ca_bundle=``. The same - private CA is forwarded to the ingestion :meth:`~questdb.auth.QuestDB.sender` - (as the ILP ``tls_roots``) for an ``https`` QuestDB, so REST queries and ILP - ingestion trust the same roots. (Only a PEM **file** is forwarded this way; - for a CA *directory*, or to override, pass ``tls_roots=``/``tls_ca=`` to - ``sender()``.) + ``SSL_CERT_FILE``) are honoured for the IdP / discovery transport; you can + also pass ``ca_bundle=``. Dependencies =========== @@ -247,7 +251,6 @@ Dependencies ``token()`` / ``headers()`` need nothing beyond the standard library. The following are imported lazily, only when used: -* ``pandas`` — for :meth:`QuestDB.sql`; * ``sqlalchemy`` and ``psycopg`` / ``psycopg2`` — for the PG-wire adapters; * ``qrcode`` — to render a QR code for phone-based authorization (``qr=True``); * ``IPython`` — for the rich Jupyter prompt (falls back to plain text). diff --git a/docs/installation.rst b/docs/installation.rst index fccd7599..7fbc11c0 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -25,9 +25,9 @@ Without this option, you may still ingest data row-by-row. The :ref:`OIDC authentication helper ` (:mod:`questdb.auth`) needs no extra dependencies for ``token()`` / ``headers()``. Some of its conveniences -import the following lazily, only when used: ``pandas`` (for ``sql()``), -``sqlalchemy`` and ``psycopg`` / ``psycopg2`` (PG-wire adapters), ``qrcode`` -(QR-code prompt) and ``IPython`` (rich Jupyter prompt). +import the following lazily, only when used: ``sqlalchemy`` and ``psycopg`` / +``psycopg2`` (PG-wire adapters), ``qrcode`` (QR-code prompt) and ``IPython`` +(rich Jupyter prompt). PIP --- diff --git a/examples/oidc_device_auth.py b/examples/oidc_device_auth.py index 1691659f..eb5df9e2 100644 --- a/examples/oidc_device_auth.py +++ b/examples/oidc_device_auth.py @@ -12,56 +12,74 @@ import sys -from questdb.auth import connect, OidcDeviceAuth, OidcError +from questdb.auth import ( + OidcDeviceAuth, + OidcError, + psycopg_connect, + sqlalchemy_engine, +) QUESTDB_URL = 'https://questdb.example.com:9000' -def integrated(url: str = QUESTDB_URL): - """The high-level path: sign in, then query / ingest with one object.""" - # First call triggers the interactive device-flow sign-in; the token is - # cached, so re-running this is silent until it expires. - qdb = connect(url) - - # Query straight to a pandas DataFrame over REST (Authorization: Bearer). - df = qdb.sql("SELECT * FROM trades WHERE ts > dateadd('h', -1, now())") - print(df) - - # Feed the same auto-refreshed token into your existing tooling: - # engine = qdb.sqlalchemy_engine() # PG-wire, token as _sso password - # with qdb.psycopg() as conn: ... # raw psycopg - # - # questdb.ingress is the compiled extension; import it lazily (only the - # ingestion path needs it) so this module also loads for the pure-Python - # bring_your_own_client() path, which needs no extension. - from questdb.ingress import TimestampNanos - with qdb.sender() as sender: # ingestion (ILP over HTTP) - sender.row( - 'trades', - symbols={'symbol': 'ETH-USD', 'side': 'sell'}, - columns={'price': 2615.54, 'amount': 0.00044}, - at=TimestampNanos.now()) +def sign_in(url: str = QUESTDB_URL) -> OidcDeviceAuth: + """Discover config from QuestDB and sign in interactively (once).""" + auth = OidcDeviceAuth.from_questdb(url) + # The first token() triggers the interactive device-flow sign-in; the token + # is cached and refreshed silently, so re-running is silent until it + # expires. Sign in once up front, before any connection pool opens. + auth.token() + return auth + + +def pg_wire(url: str = QUESTDB_URL): + """Query over PG-wire, with the token wired in as the ``_sso`` password. + + Requires ``acl.oidc.pg.token.as.password.enabled=true`` on the server. + """ + auth = sign_in(url) + + # SQLAlchemy: a fresh token is injected as the password on every new + # (pooled) connection, so the engine keeps working as the token rotates. + from sqlalchemy import text + engine = sqlalchemy_engine(auth, url) + with engine.connect() as conn: + for row in conn.execute(text('SELECT * FROM trades LIMIT 10')): + print(row) + + # Or a raw psycopg / psycopg2 connection (token captured at connect time): + with psycopg_connect(auth, url) as conn: + with conn.cursor() as cur: + cur.execute('SELECT count() FROM trades') + print(cur.fetchone()) def bring_your_own_client(url: str = QUESTDB_URL): - """The low-level path: you just want the token (PG-wire / HTTP / anything).""" + """You just want the token (REST / ingestion / anything).""" auth = OidcDeviceAuth.from_questdb(url) token = auth.token() # valid, auto-refreshed id/access token headers = auth.headers() # {"Authorization": "Bearer "} print('Authorization header ready:', 'Authorization' in headers) - # e.g. hand the token to psycopg yourself over PG-wire: - # import psycopg - # conn = psycopg.connect(host='questdb.example.com', port=8812, - # dbname='qdb', user='_sso', password=token) + # Ingestion: hand the token to the ILP Sender. questdb.ingress is the + # compiled extension; import it lazily so this module loads without it. + from questdb.ingress import Sender, TimestampNanos + with Sender.from_conf( + 'https::addr=questdb.example.com:9000;', token=token) as sender: + sender.row( + 'trades', + symbols={'symbol': 'ETH-USD', 'side': 'sell'}, + columns={'price': 2615.54, 'amount': 0.00044}, + at=TimestampNanos.now()) + return token def main(): try: - integrated() + pg_wire() except OidcError as e: sys.stderr.write(f'OIDC sign-in failed: {e}\n') diff --git a/src/questdb/auth/__init__.py b/src/questdb/auth/__init__.py index a06acaf5..3623830f 100644 --- a/src/questdb/auth/__init__.py +++ b/src/questdb/auth/__init__.py @@ -30,26 +30,24 @@ browserless local and remote kernels (JupyterHub, SageMaker, Colab, VS Code-remote): authorize in any browser, the kernel only calls the IdP. -* **Just the token** — works with anything; no optional dependencies:: +**Get the token**, then present it however you like — no optional dependencies:: - from questdb.auth import OidcDeviceAuth + from questdb.auth import OidcDeviceAuth - auth = OidcDeviceAuth.from_questdb("https://questdb.example.com:9000") - token = auth.token() # device flow on first use - headers = auth.headers() # {"Authorization": "Bearer .."} + auth = OidcDeviceAuth.from_questdb("https://questdb.example.com:9000") + token = auth.token() # device flow on first use + headers = auth.headers() # {"Authorization": "Bearer .."} -* **The integrated session** — query to a DataFrame and feed adapters:: +For PG-wire there are two convenience adapters that wire the auto-refreshed +token in as the ``_sso`` password:: - from questdb.auth import connect + from questdb.auth import sqlalchemy_engine, psycopg_connect - qdb = connect("https://questdb.example.com:9000") - df = qdb.sql("SELECT * FROM trades LIMIT 10") - engine = qdb.sqlalchemy_engine() # PG-wire, token as _sso password - with qdb.sender() as sender: # ingestion (ILP/HTTP) - ... + engine = sqlalchemy_engine(auth, "https://questdb.example.com:9000") + conn = psycopg_connect(auth, "https://questdb.example.com:9000") -Optional deps (``pandas``, ``sqlalchemy``/``psycopg``, ``qrcode``, ``IPython``) -are imported lazily, only when used. +Optional deps (``sqlalchemy``/``psycopg``, ``qrcode``, ``IPython``) are imported +lazily, only when used. """ from ._device import OidcDeviceAuth @@ -62,14 +60,12 @@ OidcInteractionRequired, OidcDeviceFlowError, OidcTimeoutError, - OidcAuthError, ) -from ._questdb import QuestDB, connect +from ._adapters import sqlalchemy_engine, psycopg_connect __all__ = [ 'MemoryCache', 'NullCache', - 'OidcAuthError', 'OidcConfig', 'OidcConfigError', 'OidcDeviceAuth', @@ -78,8 +74,8 @@ 'OidcInteractionRequired', 'OidcNetworkError', 'OidcTimeoutError', - 'QuestDB', 'TokenCache', 'TokenSet', - 'connect', + 'psycopg_connect', + 'sqlalchemy_engine', ] diff --git a/src/questdb/auth/_adapters.py b/src/questdb/auth/_adapters.py new file mode 100644 index 00000000..a5008be8 --- /dev/null +++ b/src/questdb/auth/_adapters.py @@ -0,0 +1,187 @@ +################################################################################ +## ___ _ ____ ____ +## / _ \ _ _ ___ ___| |_| _ \| __ ) +## | | | | | | |/ _ \/ __| __| | | | _ \ +## | |_| | |_| | __/\__ \ |_| |_| | |_) | +## \__\_\\__,_|\___||___/\__|____/|____/ +## +## Copyright (c) 2014-2019 Appsicle +## Copyright (c) 2019-2024 QuestDB +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +""" +PG-wire connection adapters. + +Feed an :class:`OidcDeviceAuth` token into SQLAlchemy / psycopg as the QuestDB +``_sso`` password. These are thin conveniences over the token: for REST or the +ingestion ``Sender``, take :meth:`OidcDeviceAuth.headers` / :meth:`token` and +wire it up yourself. +""" + +from __future__ import annotations + +import re +from typing import Any, Optional + +from ._device import OidcDeviceAuth +from ._errors import OidcConfigError +from ._http import safe_urlparse + +_DEFAULT_PG_PORT = 8812 +_DEFAULT_DATABASE = 'qdb' + +# Reject connection-string delimiters (';', '=') and whitespace/control chars in +# the host: a real hostname/IP never has them, so their presence means a +# tampered URL trying to inject PG connection parameters (psycopg turns its +# kwargs into a libpq conninfo string). ':' is allowed — IPv6 literals contain +# it, and the PG drivers take host and port separately. +_ILLEGAL_HOST_CHARS = re.compile(r'[\x00-\x20\x7f;=]') + + +def _pg_module(): + try: + import psycopg # type: ignore # psycopg v3 + return psycopg + except ImportError: + pass + try: + import psycopg2 # type: ignore + return psycopg2 + except ImportError as e: + raise ImportError( + 'A PostgreSQL driver is required: install `psycopg` (v3) or ' + '`psycopg2-binary`.') from e + + +def _require_host(url: str, host: Optional[str] = None) -> str: + """ + Resolve the PG-wire host: an explicit ``host`` override, else the host from + the QuestDB ``url``. Raises (rather than passing a bare ``None`` to the + driver) when neither yields one, e.g. a URL with no authority such as + ``"localhost"`` or ``"questdb:9000"``. + + The returned host is *unbracketed* — psycopg and SQLAlchemy take address and + port separately. ``safe_urlparse`` validates the port up-front, raising + ``OidcConfigError`` (not a bare ``ValueError``) for a malformed one. + """ + parts, _ = safe_urlparse(url) + resolved = host or parts.hostname + if not resolved: + raise OidcConfigError( + f'The QuestDB URL {url!r} has no host. Use a URL with an explicit ' + 'host (e.g. "https://questdb.example.com:9000"), or pass host=... ' + 'to the adapter.') + if _ILLEGAL_HOST_CHARS.search(resolved): + raise OidcConfigError( + f'The QuestDB host {resolved!r} contains an illegal character ' + "(';', '=', whitespace or a control character). A hostname or IP " + 'address never does; this indicates a malformed or tampered URL. ' + '(Such a host could otherwise inject PG connection parameters.)') + return resolved + + +def sqlalchemy_engine( + auth: OidcDeviceAuth, + url: str, + *, + host: Optional[str] = None, + pg_port: int = _DEFAULT_PG_PORT, + database: str = _DEFAULT_DATABASE, + drivername: Optional[str] = None, + **engine_kwargs) -> 'sqlalchemy.engine.Engine': + """ + Build a SQLAlchemy ``Engine`` for QuestDB's PG-wire endpoint, authenticated + with ``auth``. + + Connects as user ``_sso``, injecting a **fresh** token as the password on + every new connection (via a ``do_connect`` listener) so pooled connections + always authenticate with a valid, auto-refreshed token. Requires + ``acl.oidc.pg.token.as.password.enabled=true`` on the server. + + Sign in once up front (``auth.token()``) before the pool opens connections, + so threads don't race the interactive prompt. + + :param auth: An :class:`OidcDeviceAuth`, e.g. from + :meth:`OidcDeviceAuth.from_questdb`. + :param url: The QuestDB base URL; the PG host is derived from it unless + ``host=`` is given. + :param pg_port: PG-wire port (default ``8812``). + :param database: Database name (default ``"qdb"``). + :param drivername: SQLAlchemy driver; defaults to ``postgresql+psycopg`` + (v3) or ``postgresql+psycopg2`` depending on what is installed. + :param engine_kwargs: Forwarded to ``create_engine``. + """ + try: + from sqlalchemy import create_engine, event + from sqlalchemy.engine import URL + except ImportError as e: + raise ImportError( + 'SQLAlchemy is required for questdb.auth.sqlalchemy_engine(); ' + 'install it with `pip install sqlalchemy`.') from e + + if drivername is None: + mod = _pg_module() + drivername = ( + 'postgresql+psycopg' + if mod.__name__ == 'psycopg' + else 'postgresql+psycopg2') + + engine = create_engine( + URL.create( + drivername=drivername, + username='_sso', + host=_require_host(url, host), + port=pg_port, + database=database), + **engine_kwargs) + + @event.listens_for(engine, 'do_connect') + def _provide_token(dialect, conn_rec, cargs, cparams): # noqa: ANN001 + cparams['password'] = auth.token() + + return engine + + +def psycopg_connect( + auth: OidcDeviceAuth, + url: str, + *, + host: Optional[str] = None, + pg_port: int = _DEFAULT_PG_PORT, + database: str = _DEFAULT_DATABASE, + **connect_kwargs) -> Any: + """ + Open a raw psycopg (v3) or psycopg2 connection to QuestDB's PG-wire + endpoint, authenticating as ``_sso`` with the current token. + + The token is captured at connect time; reconnect to pick up a refreshed + token. Requires ``acl.oidc.pg.token.as.password.enabled=true`` on the + server. + + :param auth: An :class:`OidcDeviceAuth`, e.g. from + :meth:`OidcDeviceAuth.from_questdb`. + :param url: The QuestDB base URL; the PG host is derived from it unless + ``host=`` is given. + :param connect_kwargs: Forwarded to the driver's ``connect()``. + """ + mod = _pg_module() + return mod.connect( + host=_require_host(url, host), + port=pg_port, + dbname=database, + user='_sso', + password=auth.token(), + **connect_kwargs) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index d1df02a5..91a6747e 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -146,9 +146,8 @@ class OidcDeviceAuth: to the device-code lifetime, ~30 min): a caller with a *valid* cached token never blocks, but one whose token is missing/expired waits behind the signer. So when threads share an auth object (e.g. a SQLAlchemy/psycopg - pool), sign in once up front — :func:`questdb.auth.connect` does this via - ``eager=True`` (the default), running the flow once on the main thread before - the pool opens connections. + pool), sign in once up front — call :meth:`token` once on the main thread + before the pool opens connections. .. code-block:: python diff --git a/src/questdb/auth/_errors.py b/src/questdb/auth/_errors.py index 17b83e9e..c4741cf4 100644 --- a/src/questdb/auth/_errors.py +++ b/src/questdb/auth/_errors.py @@ -79,11 +79,3 @@ def __init__( class OidcTimeoutError(OidcDeviceFlowError): """The user did not authorize the device in time (the code expired).""" - - -class OidcAuthError(OidcError): - """ - QuestDB rejected the token (typically a ``401``/``403`` from the server); - the message hints at common causes (scope / ``groups.encoded.in.token`` / - ``audience`` mismatches). - """ diff --git a/src/questdb/auth/_questdb.py b/src/questdb/auth/_questdb.py deleted file mode 100644 index 2cb7da05..00000000 --- a/src/questdb/auth/_questdb.py +++ /dev/null @@ -1,405 +0,0 @@ -################################################################################ -## ___ _ ____ ____ -## / _ \ _ _ ___ ___| |_| _ \| __ ) -## | | | | | | |/ _ \/ __| __| | | | _ \ -## | |_| | |_| | __/\__ \ |_| |_| | |_) | -## \__\_\\__,_|\___||___/\__|____/|____/ -## -## Copyright (c) 2014-2019 Appsicle -## Copyright (c) 2019-2024 QuestDB -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. -## -################################################################################ - -"""High-level QuestDB session: token, REST queries, and connection adapters.""" - -from __future__ import annotations - -import os -import re -import urllib.parse -from typing import Any, Dict, Optional - -from ._device import OidcDeviceAuth -from ._errors import OidcAuthError, OidcConfigError, OidcError -from ._http import request, safe_urlparse - -_DEFAULT_PG_PORT = 8812 -_DEFAULT_DATABASE = 'qdb' - -# Reject ILP conf-string delimiters (';', '=') and whitespace/control chars in -# the host: a real hostname/IP never has them, so their presence means a -# tampered URL trying to inject conf params like ';tls_verify=unsafe_off;' -# (which disables TLS verification) into the addr= string. ':' is allowed -# (IPv6 literals contain it; _ilp_addr brackets them). -_ILLEGAL_HOST_CHARS = re.compile(r'[\x00-\x20\x7f;=]') - -_AUTH_HINT = ( - 'QuestDB rejected the token (HTTP {status}). Common causes:\n' - " * scope / 'acl.oidc.groups.encoded.in.token' mismatch — the server may " - 'expect the id_token (groups in token) while an access_token was sent, or ' - 'vice-versa;\n' - " * the 'groups'/'sub' claim is missing — check the requested scope;\n" - " * 'aud' mismatch — the token's audience does not match " - "'acl.oidc.audience' (try passing audience=...).") - - -def _import_pandas(): - try: - import pandas # type: ignore - return pandas - except ImportError as e: - raise ImportError( - 'Missing optional dependency `pandas`, required for ' - 'QuestDB.sql(). Install it with `pip install questdb[dataframe]`. ' - 'See https://py-questdb-client.readthedocs.io/en/latest/' - 'installation.html') from e - - -def _exec_json_to_df(data: Dict[str, Any], pandas): - columns = data.get('columns') or [] - # /exec returns a list of {"name", "type"} descriptors. Validate the shape - # (a real column name is always a string) so a malformed response raises a - # clean OidcError rather than a raw AttributeError from .get() or a - # TypeError from `name in df.columns` on a non-hashable name. - if not isinstance(columns, list) or not all( - isinstance(c, dict) - and isinstance(c.get('name'), (str, type(None))) - for c in columns): - raise OidcError( - 'QuestDB /exec returned a malformed "columns" field; ' - 'cannot build a DataFrame.') - names = [c.get('name') for c in columns] - dataset = data.get('dataset') - if dataset is None: - dataset = data.get('data') or [] - try: - df = pandas.DataFrame(dataset, columns=names or None) - except (ValueError, TypeError) as e: - # A malformed dataset shape can make the pandas constructor raise - # ValueError or TypeError; keep both within OidcError. - raise OidcError( - f'Unexpected shape in QuestDB /exec response: {e}') from e - for col in columns: - name = col.get('name') - if col.get('type') in ('TIMESTAMP', 'DATE') and name in df.columns: - try: - df[name] = pandas.to_datetime(df[name], errors='coerce') - except Exception: - pass - return df - - -def _pg_module(): - try: - import psycopg # type: ignore # psycopg v3 - return psycopg - except ImportError: - pass - try: - import psycopg2 # type: ignore - return psycopg2 - except ImportError as e: - raise ImportError( - 'A PostgreSQL driver is required: install `psycopg` (v3) or ' - '`psycopg2-binary`.') from e - - -class QuestDB: - """ - A thin, authenticated QuestDB session built on an :class:`OidcDeviceAuth`. - - Offers a one-call DataFrame query over REST plus adapters that feed the - same auto-refreshed token into SQLAlchemy / psycopg / the ingestion - ``Sender``, or take :meth:`token` / :meth:`headers` and wire it up yourself. - """ - - def __init__( - self, - url: str, - auth: OidcDeviceAuth, - *, - insecure: bool = False): - self.url = url.rstrip('/') - self.auth = auth - self._insecure = insecure - self._ctx = auth._ctx - # Same private CA bundle the auth/REST transport uses, so sender() can - # forward it to the ILP Sender's own TLS stack. getattr keeps test - # doubles that only set _ctx working. - self._ca_bundle = getattr(auth, '_ca_bundle', None) - # safe_urlparse validates the port up-front, raising OidcConfigError - # (not a bare ValueError) for a malformed one. - self._parts, self._port = safe_urlparse(self.url) - - # -- token access ------------------------------------------------------- - - def token(self) -> str: - """Return a valid, auto-refreshed token (see :meth:`OidcDeviceAuth.token`).""" - return self.auth.token() - - def headers(self) -> Dict[str, str]: - """Return ``{"Authorization": "Bearer "}``.""" - return self.auth.headers() - - # -- REST query --------------------------------------------------------- - - def sql(self, query: str, *, limit: Optional[str] = None, - timeout: float = 60) -> 'pandas.DataFrame': - """ - Run a SQL query over QuestDB's REST ``/exec`` endpoint and return a - :class:`pandas.DataFrame`. - - Uses ``Authorization: Bearer`` (no token-length limit, unlike PG-wire), - so it's the recommended path for large groups-encoded JWTs. - - :param query: The SQL query to run. - :param limit: Optional QuestDB ``limit`` (e.g. ``"1,1000"``). - :param timeout: Request timeout in seconds. - """ - pandas = _import_pandas() - params = {'query': query} - if limit is not None: - params['limit'] = limit - url = f'{self.url}/exec?' + urllib.parse.urlencode(params) - resp = request( - 'GET', url, headers=self.headers(), ctx=self._ctx, - insecure=self._insecure, timeout=timeout) - if resp.status in (401, 403): - raise OidcAuthError(_AUTH_HINT.format(status=resp.status)) - if not resp.ok: - detail = resp.text()[:300] - try: - detail = resp.json().get('error', detail) - except Exception: - pass - raise OidcError( - f'QuestDB query failed (HTTP {resp.status}): {detail}') - try: - data = resp.json() - except (ValueError, UnicodeDecodeError, RecursionError): - # A 2xx body that isn't JSON (e.g. an HTML page from a proxy/captive - # portal) or deeply-nested JSON that exhausts the decoder's stack - # (RecursionError) surfaces as a clean OidcError. Mirrors post_form(). - raise OidcError( - 'QuestDB returned a non-JSON success response from /exec: ' - f'{resp.text()[:300]}') - if not isinstance(data, dict): - # Valid JSON but not an object (e.g. a bare list) would break - # _exec_json_to_df's .get(); reject it. - raise OidcError( - 'QuestDB /exec returned JSON that is not an object ' - f'(got {type(data).__name__}); cannot build a DataFrame.') - return _exec_json_to_df(data, pandas) - - # -- connection adapters ------------------------------------------------ - - def _require_host(self, host: Optional[str] = None) -> str: - """ - Resolve the PG-wire / ILP host: an explicit ``host`` override, else the - host from the QuestDB URL. Raises (rather than passing a bare ``None`` - to the driver) when neither yields one, e.g. a URL with no authority - such as ``"localhost"`` or ``"questdb:9000"``. - - The returned host is *unbracketed* — psycopg and SQLAlchemy take address - and port separately. :meth:`_ilp_addr` adds the brackets an IPv6 literal - needs in the ILP ``addr=host:port`` form. - """ - resolved = host or self._parts.hostname - if not resolved: - raise OidcConfigError( - f'The QuestDB URL {self.url!r} has no host. Use a URL with an ' - 'explicit host (e.g. "https://questdb.example.com:9000"), or ' - 'pass host=... to the adapter.') - if _ILLEGAL_HOST_CHARS.search(resolved): - raise OidcConfigError( - f'The QuestDB host {resolved!r} contains an illegal character ' - "(';', '=', whitespace or a control character). A hostname or " - 'IP address never does; this indicates a malformed or tampered ' - 'URL. (Such a host could otherwise inject ILP conf parameters ' - 'such as "tls_verify=unsafe_off" into the sender, silently ' - 'disabling TLS certificate verification.)') - return resolved - - @staticmethod - def _ilp_addr(host: str, port: int) -> str: - # Bracket an IPv6 literal (it contains ':', unlike hostnames/IPv4) so - # the ILP conf parser reads host:port unambiguously. - bracketed = f'[{host}]' if ':' in host else host - return f'{bracketed}:{port}' - - def sqlalchemy_engine( - self, - *, - host: Optional[str] = None, - pg_port: int = _DEFAULT_PG_PORT, - database: str = _DEFAULT_DATABASE, - drivername: Optional[str] = None, - **engine_kwargs) -> 'sqlalchemy.engine.Engine': - """ - Build a SQLAlchemy ``Engine`` for QuestDB's PG-wire endpoint. - - Connects as user ``_sso``, injecting a **fresh** token as the password - on every new connection (via a ``do_connect`` listener) so pooled - connections always authenticate with a valid token. Requires - ``acl.oidc.pg.token.as.password.enabled=true`` on the server. - """ - try: - from sqlalchemy import create_engine, event - from sqlalchemy.engine import URL - except ImportError as e: - raise ImportError( - 'SQLAlchemy is required for QuestDB.sqlalchemy_engine(); ' - 'install it with `pip install sqlalchemy`.') from e - - if drivername is None: - mod = _pg_module() - drivername = ( - 'postgresql+psycopg' - if mod.__name__ == 'psycopg' - else 'postgresql+psycopg2') - - url = URL.create( - drivername=drivername, - username='_sso', - host=self._require_host(host), - port=pg_port, - database=database) - engine = create_engine(url, **engine_kwargs) - - auth = self.auth - - @event.listens_for(engine, 'do_connect') - def _provide_token(dialect, conn_rec, cargs, cparams): # noqa: ANN001 - cparams['password'] = auth.token() - - return engine - - def psycopg( - self, - *, - host: Optional[str] = None, - pg_port: int = _DEFAULT_PG_PORT, - database: str = _DEFAULT_DATABASE, - **connect_kwargs) -> 'Any': - """ - Open a raw psycopg (v3) or psycopg2 connection to QuestDB's PG-wire - endpoint, authenticating as ``_sso`` with the current token. - - The token is captured at connect time; reconnect to pick up a refreshed - token. - """ - mod = _pg_module() - return mod.connect( - host=self._require_host(host), - port=pg_port, - dbname=database, - user='_sso', - password=self.auth.token(), - **connect_kwargs) - - def sender(self, *, port: Optional[int] = None, - **sender_kwargs) -> 'questdb.ingress.Sender': - """ - Build a :class:`questdb.ingress.Sender` (ILP-over-HTTP) for ingestion, - configured with the current bearer token. - - The token is captured at creation time; create a new sender to pick up - a refreshed token. - """ - scheme = 'https' if self._parts.scheme == 'https' else 'http' - resolved_port = port or self._port or ( - 443 if scheme == 'https' else 9000) - # Coerce to int (before the heavy import, so bad input fails fast) so a - # stray non-integer port can't inject conf params like - # "9000;tls_verify=unsafe_off" into the addr= string via _ilp_addr — - # the same injection _require_host() blocks for the host. The - # URL-derived self._port is already an int. - try: - resolved_port = int(resolved_port) - except (TypeError, ValueError): - raise OidcConfigError( - f'Invalid port {resolved_port!r} for QuestDB.sender(); expected ' - 'an integer.') - - try: - from questdb.ingress import Sender - except ImportError as e: - raise ImportError( - 'The compiled `questdb.ingress` module is required for ' - 'QuestDB.sender(). Install the full client wheel ' - '(`pip install questdb`).') from e - - conf = (f'{scheme}::addr=' - f'{self._ilp_addr(self._require_host(), resolved_port)};') - # Forward the private CA bundle (explicit ca_bundle=, else the - # REQUESTS_CA_BUNDLE / SSL_CERT_FILE env vars — same precedence as - # build_ssl_context) to the Sender as tls_roots, so an https Sender - # against a private-CA QuestDB trusts the same roots the REST/IdP paths - # do. Only a PEM file works (tls_roots takes a file, no capath), only - # over https, and the caller can still override via tls_roots=/tls_ca=. - if (scheme == 'https' - and 'tls_roots' not in sender_kwargs - and 'tls_ca' not in sender_kwargs): - ca = (self._ca_bundle - or os.environ.get('REQUESTS_CA_BUNDLE') - or os.environ.get('SSL_CERT_FILE')) - if ca and os.path.isfile(ca): - sender_kwargs['tls_roots'] = ca - return Sender.from_conf(conf, token=self.auth.token(), **sender_kwargs) - - -def connect( - url: str, - *, - flow: str = 'auto', - cache: Any = 'memory', - insecure: bool = False, - eager: bool = True, - **opts) -> QuestDB: - """ - High-level entry point: authenticate to QuestDB and return a - :class:`QuestDB` session. - - .. code-block:: python - - from questdb.auth import connect - - qdb = connect("https://questdb.example.com:9000") # signs in - df = qdb.sql("SELECT * FROM trades LIMIT 10") - - Configuration (OIDC client id, scope, endpoints, groups mode) is discovered - from ``{url}/settings`` and, as needed, the IdP ``.well-known`` document. - Re-running the same call reuses the cached token (no re-prompt). - - :param url: The QuestDB HTTP(S) base URL, e.g. - ``"https://questdb.example.com:9000"``. - :param flow: ``"auto"`` (default), ``"device"`` or ``"loopback"``. Today - ``"auto"`` resolves to the device flow (works on local and remote - kernels); ``"loopback"`` is reserved for a future release. - :param cache: Token cache backend: ``"memory"`` (default) or ``None``. - :param insecure: Allow plaintext ``http://`` URLs (development only). - :param eager: If ``True`` (default), sign in immediately; otherwise defer - until the first call that needs a token. - :param opts: Forwarded to :meth:`OidcDeviceAuth.from_questdb` (e.g. - ``client_id``, ``scope``, ``audience``, ``issuer``, ``open_browser``, - ``qr``, ``ca_bundle``, ``timeout`` — the per-request IdP network timeout, - which also bounds how long a stalled IdP can hold the token lock). - """ - auth = OidcDeviceAuth.from_questdb( - url, flow=flow, cache=cache, insecure=insecure, **opts) - qdb = QuestDB(url, auth, insecure=insecure) - if eager: - auth.token() - return qdb diff --git a/test/test_auth.py b/test/test_auth.py index a215388a..7218218f 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -53,25 +53,20 @@ from questdb.auth import ( # noqa: E402 OidcDeviceAuth, - QuestDB, - connect, OidcError, OidcConfigError, OidcDeviceFlowError, OidcTimeoutError, OidcInteractionRequired, - OidcAuthError, OidcNetworkError, TokenSet, + sqlalchemy_engine, + psycopg_connect, ) from questdb.auth._cache import ( # noqa: E402 MemoryCache, _MEMORY_GENERATION, _MEMORY_STORE) from questdb.auth._render import Renderer # noqa: E402 - -try: - import pandas as pd -except ImportError: - pd = None +from questdb.auth._adapters import _require_host # noqa: E402 _HAS_PG_DRIVER = ( importlib.util.find_spec('psycopg') is not None @@ -692,9 +687,9 @@ def fake_post_form(url, form, *, ctx=None, insecure=False, all(t == 3 for t in seen), f'IdP POSTs did not all use the configured timeout: {seen}') - def test_connect_lazy_defers_signin(self): - # eager=False must return a session WITHOUT running the device flow; the - # first token-needing call then triggers exactly one sign-in. See M4. + def test_from_questdb_defers_signin(self): + # from_questdb() must return WITHOUT running the device flow; the first + # token-needing call then triggers exactly one sign-in. self.state.settings = {'config': { 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', @@ -702,10 +697,11 @@ def test_connect_lazy_defers_signin(self): 'acl.oidc.groups.encoded.in.token': True, 'acl.oidc.token.endpoint': self.base + '/token', 'acl.oidc.device.authorization.endpoint': self.base + '/device'}} - qdb = connect(self.base, insecure=True, eager=False, - renderer=Renderer(), interactive=True, _clock=FakeClock()) + auth = OidcDeviceAuth.from_questdb( + self.base, insecure=True, renderer=Renderer(), + interactive=True, _clock=FakeClock()) self.assertEqual(self.state.device_requests, 0) # deferred - self.assertEqual(qdb.token(), ID_TOKEN) # first use signs in + self.assertEqual(auth.token(), ID_TOKEN) # first use signs in self.assertEqual(self.state.device_requests, 1) def test_open_browser_rejects_dangerous_scheme(self): @@ -1191,17 +1187,18 @@ def test_well_known_404_raises_oidc_error(self): OidcDeviceAuth.from_questdb(self.base, issuer=self.base, insecure=True) - def test_connect_forwards_default_interval(self): - # M5: connect(**opts) routes through from_questdb; default_interval must - # be accepted (it previously raised TypeError) and reach the auth. + def test_from_questdb_forwards_default_interval(self): + # from_questdb(**opts) must accept default_interval (it previously + # raised TypeError) and reach the auth. self.state.settings = {'config': { 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', 'acl.oidc.token.endpoint': self.base + '/token', 'acl.oidc.device.authorization.endpoint': self.base + '/device'}} - qdb = connect(self.base, insecure=True, eager=False, default_interval=9, - renderer=Renderer(), interactive=True, _clock=FakeClock()) - self.assertEqual(qdb.auth._default_interval, 9) + auth = OidcDeviceAuth.from_questdb( + self.base, insecure=True, default_interval=9, + renderer=Renderer(), interactive=True, _clock=FakeClock()) + self.assertEqual(auth._default_interval, 9) class TestInsecureSettingsGuard(unittest.TestCase): @@ -1338,154 +1335,6 @@ def test_issuer_path_scope_rejects_dot_segment_traversal(self): self.assertIn('issuer', str(cm.exception).lower()) -@unittest.skipIf(pd is None, 'pandas not installed') -class TestRestAdapter(AuthTestBase): - def _connected(self): - self.state.settings = {'config': { - 'acl.oidc.enabled': True, - 'acl.oidc.client.id': 'questdb', - 'acl.oidc.scope': 'openid groups', - 'acl.oidc.groups.encoded.in.token': True, - 'acl.oidc.token.endpoint': self.base + '/token', - 'acl.oidc.device.authorization.endpoint': self.base + '/device', - }} - self.state.expected_bearer = ID_TOKEN - return connect(self.base, insecure=True, renderer=Renderer(), - interactive=True, _clock=FakeClock()) - - def test_sql_returns_dataframe(self): - qdb = self._connected() - df = qdb.sql('SELECT * FROM trades') - self.assertEqual(list(df.columns), ['ts', 'price']) - self.assertEqual(len(df), 2) - self.assertEqual(df['price'].tolist(), [1.5, 2.5]) - # TIMESTAMP column coerced to datetime. - self.assertTrue(str(df['ts'].dtype).startswith('datetime64')) - - def test_sql_unauthorized_maps_to_auth_error(self): - qdb = self._connected() - self.state.expected_bearer = 'something-else' # force 401 - with self.assertRaises(OidcAuthError): - qdb.sql('SELECT 1') - - def test_connect_is_eager(self): - qdb = self._connected() - self.assertIsInstance(qdb, QuestDB) - # Sign-in already happened during connect(). - self.assertEqual(self.state.device_requests, 1) - - def test_sql_query_error_maps_to_oidc_error(self): - qdb = self._connected() - self.state.exec_status = 400 - self.state.exec_response = {'error': 'unexpected token', 'position': 5} - with self.assertRaises(OidcError) as cm: - qdb.sql('SELEKT 1') - self.assertIn('unexpected token', str(cm.exception)) - self.assertNotIsInstance(cm.exception, OidcAuthError) - - def test_sql_passes_limit(self): - qdb = self._connected() - qdb.sql('SELECT * FROM trades', limit='1,10') - self.assertTrue(any('limit=1' in p for p in self.state.exec_requests)) - - def test_sql_handles_empty_dataset(self): - qdb = self._connected() - self.state.exec_response = {'ddl': 'OK'} # no columns / dataset - df = qdb.sql('CREATE TABLE x (a INT)') - self.assertEqual(len(df), 0) - - def test_sql_malformed_shape_raises_oidc_error(self): - qdb = self._connected() - self.state.exec_response = { # rows shorter than the column list - 'columns': [{'name': 'a', 'type': 'LONG'}, - {'name': 'b', 'type': 'LONG'}], - 'dataset': [[1]]} - with self.assertRaises(OidcError): - qdb.sql('SELECT a, b FROM t') - - def test_sql_non_json_2xx_raises_oidc_error(self): - # A 2xx body that isn't JSON (e.g. an HTML page from a reverse proxy) - # must raise a clean OidcError, not a raw JSONDecodeError. See M3. - qdb = self._connected() - self.state.exec_raw = (200, 'text/html', b'proxy') - with self.assertRaises(OidcError) as cm: - qdb.sql('SELECT 1') - self.assertNotIsInstance(cm.exception, OidcAuthError) - - def test_sql_non_dict_json_raises_oidc_error(self): - # A valid-JSON-but-not-an-object 2xx body (e.g. a bare list) must raise - # OidcError, not AttributeError from .get(). See M3. - qdb = self._connected() - self.state.exec_response = ['not', 'an', 'object'] - with self.assertRaises(OidcError) as cm: - qdb.sql('SELECT 1') - self.assertNotIsInstance(cm.exception, OidcAuthError) - - def test_sql_non_dict_columns_raises_oidc_error(self): - # A /exec body whose "columns" entries aren't objects must raise a clean - # OidcError, not an AttributeError from .get() on the column. See M3. - qdb = self._connected() - self.state.exec_response = {'columns': [None], 'dataset': [[1]]} - with self.assertRaises(OidcError) as cm: - qdb.sql('SELECT 1') - self.assertNotIsInstance(cm.exception, OidcAuthError) - - def test_sql_non_string_column_name_raises_oidc_error(self): - # M2: a column descriptor with a non-hashable name (a JSON list/object) - # and a TIMESTAMP/DATE type must raise a clean OidcError, not a raw - # TypeError ("unhashable type") from `name in df.columns` during the - # timestamp coercion. - qdb = self._connected() - self.state.exec_response = { - 'columns': [{'name': ['evil'], 'type': 'TIMESTAMP'}, - {'name': 'b', 'type': 'LONG'}], - 'dataset': [['2021-01-01T00:00:00.000000Z', 2]]} - with self.assertRaises(OidcError) as cm: - qdb.sql('SELECT 1') - self.assertNotIsInstance(cm.exception, OidcAuthError) - - -class TestRestAdapterAuthErrors(AuthTestBase): - """QuestDB.sql maps 401/403 to OidcAuthError BEFORE it builds a DataFrame, - so the mapping is testable without a real pandas. Kept out of the - pandas-gated TestRestAdapter so this security-relevant mapping runs on EVERY - CI leg, not just the ones where pandas is installed. M5.""" - - def _connected(self): - self.state.settings = {'config': { - 'acl.oidc.enabled': True, - 'acl.oidc.client.id': 'questdb', - 'acl.oidc.scope': 'openid groups', - 'acl.oidc.groups.encoded.in.token': True, - 'acl.oidc.token.endpoint': self.base + '/token', - 'acl.oidc.device.authorization.endpoint': self.base + '/device', - }} - self.state.expected_bearer = ID_TOKEN - return connect(self.base, insecure=True, renderer=Renderer(), - interactive=True, _clock=FakeClock()) - - @staticmethod - def _stub_pandas(): - # sql() reaches the 401/403 check before it touches pandas, so a bare - # stub module is enough to exercise the mapping without the real - # (possibly absent) dependency. - return mock.patch.dict( - sys.modules, {'pandas': types.ModuleType('pandas')}) - - def test_sql_401_maps_to_auth_error_without_pandas(self): - qdb = self._connected() - self.state.expected_bearer = 'something-else' # force 401 - with self._stub_pandas(), self.assertRaises(OidcAuthError): - qdb.sql('SELECT 1') - - def test_sql_403_maps_to_auth_error_without_pandas(self): - qdb = self._connected() - self.state.exec_status = 403 # bearer matches; server forbids - self.state.exec_response = {'error': 'forbidden'} - with self._stub_pandas(), self.assertRaises(OidcAuthError): - qdb.sql('SELECT 1') - - class TestConcurrency(AuthTestBase): def test_valid_cached_token_does_not_block_during_signin(self): # A caller with a valid cached token must NOT block behind another @@ -1598,50 +1447,11 @@ def test_store_if_current_drops_write_after_concurrent_clear(self): class TestAdapters(unittest.TestCase): - """Connection adapters: tested via injected fake modules (the real - sqlalchemy / psycopg / questdb.ingress need not be installed).""" - - def _qdb(self, url='http://db.example.com:9000', token='TKN'): - return QuestDB(url, _FakeAuth(token), insecure=True) - - def test_sender_builds_conf_with_token(self): - qdb = self._qdb('http://db.example.com:9000', token='TKN') - captured = {} - - fake = types.ModuleType('questdb.ingress') - - class Sender: - @staticmethod - def from_conf(conf, *, token=None, **kw): - captured.update(conf=conf, token=token, kw=kw) - return 'SENDER' - - fake.Sender = Sender - with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): - sender = qdb.sender(auto_flush=False) - self.assertEqual(sender, 'SENDER') - self.assertEqual(captured['conf'], 'http::addr=db.example.com:9000;') - self.assertEqual(captured['token'], 'TKN') - self.assertEqual(captured['kw'], {'auto_flush': False}) - - def test_sender_https_defaults_to_443(self): - qdb = self._qdb('https://db.example.com') # no explicit port - captured = {} - fake = types.ModuleType('questdb.ingress') - - class Sender: - @staticmethod - def from_conf(conf, *, token=None, **kw): - captured['conf'] = conf - return 'S' - - fake.Sender = Sender - with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): - qdb.sender() - self.assertEqual(captured['conf'], 'https::addr=db.example.com:443;') + """PG-wire connection adapters: tested via injected fake modules (the real + sqlalchemy / psycopg need not be installed).""" - def test_psycopg_connects_as_sso_with_token(self): - qdb = self._qdb('http://db.example.com:9000', token='TKN') + def test_psycopg_connect_as_sso_with_token(self): + auth = _FakeAuth('TKN') captured = {} fake = types.ModuleType('psycopg') @@ -1651,7 +1461,8 @@ def connect(**kw): fake.connect = connect with mock.patch.dict(sys.modules, {'psycopg': fake}): - conn = qdb.psycopg(connect_timeout=3) + conn = psycopg_connect( + auth, 'http://db.example.com:9000', connect_timeout=3) self.assertEqual(conn, 'CONN') self.assertEqual(captured['user'], '_sso') self.assertEqual(captured['password'], 'TKN') @@ -1660,11 +1471,25 @@ def connect(**kw): self.assertEqual(captured['dbname'], 'qdb') self.assertEqual(captured['connect_timeout'], 3) # The token is fetched at connect time (fresh per connection). - self.assertEqual(qdb.auth.calls, 1) + self.assertEqual(auth.calls, 1) + + def test_psycopg_connect_uses_bare_ipv6_host(self): + # psycopg takes host and port separately, so the IPv6 host is passed + # WITHOUT brackets. + captured = {} + fake = types.ModuleType('psycopg') + + def connect(**kw): + captured.update(kw) + return 'CONN' + + fake.connect = connect + with mock.patch.dict(sys.modules, {'psycopg': fake}): + psycopg_connect(_FakeAuth(), 'http://[::1]:9000') + self.assertEqual(captured['host'], '::1') def test_sqlalchemy_engine_injects_fresh_token_per_connect(self): auth = _FakeAuth('TKN') - qdb = QuestDB('http://db.example.com:9000', auth, insecure=True) created = {} events = {} engine_obj = object() @@ -1702,7 +1527,8 @@ def create(**kw): 'sqlalchemy': fake_sa, 'sqlalchemy.engine': fake_eng, 'psycopg': fake_pg}): - engine = qdb.sqlalchemy_engine(pool_pre_ping=True) + engine = sqlalchemy_engine( + auth, 'http://db.example.com:9000', pool_pre_ping=True) self.assertIs(engine, engine_obj) self.assertEqual(created['drivername'], 'postgresql+psycopg') @@ -1721,190 +1547,64 @@ def create(**kw): self.assertEqual(cparams['password'], 'TKN') self.assertEqual(auth.calls - before, 2) - def test_sender_brackets_ipv6_addr(self): - # An IPv6 literal must be bracketed in the ILP addr=host:port conf, - # else "::1:9000" is ambiguous to the conf parser. See M5. - qdb = self._qdb('https://[::1]:9000') - captured = {} - fake = types.ModuleType('questdb.ingress') - - class Sender: - @staticmethod - def from_conf(conf, *, token=None, **kw): - captured['conf'] = conf - return 'S' - - fake.Sender = Sender - with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): - qdb.sender() - self.assertEqual(captured['conf'], 'https::addr=[::1]:9000;') - - def test_sender_forwards_ca_bundle_as_tls_roots(self): - # M2: an https Sender must inherit the private CA bundle (as tls_roots) - # so it trusts the same roots as the REST/IdP paths; http does not, and - # an explicit tls_roots= is never overridden. - import tempfile - - def captured_conf_kwargs(url, *, ca_bundle, **sender_kwargs): - auth = _FakeAuth('TKN') - auth._ca_bundle = ca_bundle - qdb = QuestDB(url, auth, insecure=True) - captured = {} - fake = types.ModuleType('questdb.ingress') - - class Sender: - @staticmethod - def from_conf(conf, *, token=None, **kw): - captured['kw'] = kw - return 'S' - - fake.Sender = Sender - with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): - qdb.sender(**sender_kwargs) - return captured['kw'] - - with tempfile.NamedTemporaryFile('w', suffix='.pem', delete=False) as f: - f.write('-----dummy-----') - ca = f.name - try: - # https + a real CA file -> forwarded as tls_roots. - self.assertEqual( - captured_conf_kwargs('https://db.example.com:9000', - ca_bundle=ca).get('tls_roots'), ca) - # http -> never forwarded (TLS roots are irrelevant). - self.assertNotIn( - 'tls_roots', - captured_conf_kwargs('http://db.example.com:9000', - ca_bundle=ca)) - # An explicit tls_roots= wins over the inherited bundle. - self.assertEqual( - captured_conf_kwargs('https://db.example.com:9000', - ca_bundle=ca, - tls_roots='/other/ca.pem').get('tls_roots'), - '/other/ca.pem') - finally: - os.unlink(ca) - - def test_psycopg_uses_bare_ipv6_host(self): - # psycopg takes host and port separately, so the IPv6 host is passed - # WITHOUT brackets (unlike the ILP addr= form). See M5. - qdb = self._qdb('http://[::1]:9000') - captured = {} - fake = types.ModuleType('psycopg') - - def connect(**kw): - captured.update(kw) - return 'CONN' - - fake.connect = connect - with mock.patch.dict(sys.modules, {'psycopg': fake}): - qdb.psycopg() - self.assertEqual(captured['host'], '::1') - def test_require_host_rejects_hostless_url(self): # A URL with no extractable host must raise, not pass None to a driver; - # an explicit host= override still resolves. See M5. + # an explicit host= override still resolves. for bad in ('localhost', 'questdb:9000'): with self.subTest(url=bad): with self.assertRaises(OidcConfigError): - QuestDB(bad, _FakeAuth(), insecure=True)._require_host() - self.assertEqual( - QuestDB('localhost', _FakeAuth())._require_host('h.example'), - 'h.example') + _require_host(bad) + self.assertEqual(_require_host('localhost', 'h.example'), 'h.example') - def test_malformed_port_url_raises_config_error(self): - # A QuestDB URL with a non-integer port must raise OidcConfigError at - # construction, not a bare ValueError when an adapter reads .port. M3. + def test_require_host_malformed_port_raises_config_error(self): + # A QuestDB URL with a non-integer port must raise OidcConfigError (via + # safe_urlparse), not a bare ValueError. with self.assertRaises(OidcConfigError): - QuestDB('https://questdb.example.com:notaport', _FakeAuth(), - insecure=True) - - def test_host_with_conf_metachars_rejected(self): - # C1: a host containing the ILP conf delimiters (';' / '=') or - # whitespace must be rejected, never spliced into the - # `addr=host:port;` conf string. Otherwise a crafted/tampered URL host - # injects extra conf params — e.g. `tls_verify=unsafe_off`, which - # silently disables the sender's TLS certificate verification, or - # `auto_flush=off` (data loss). urlparse() keeps ';'/'=' in .hostname. - for bad in ('https://realhost;tls_verify=unsafe_off;x=', - 'https://a=b'): + _require_host('https://questdb.example.com:notaport') + + def test_require_host_with_conf_metachars_rejected(self): + # A host containing connection-string delimiters (';' / '=') or + # whitespace must be rejected, never spliced into PG connection + # parameters (psycopg builds a libpq conninfo string from its kwargs). + # urlparse() keeps ';'/'=' in .hostname. + for bad in ('https://realhost;sslmode=disable;x=', 'https://a=b'): with self.subTest(url=bad): with self.assertRaises(OidcConfigError): - self._qdb(bad)._require_host() + _require_host(bad) # An explicit host= override goes through the same guard (incl. # whitespace, which is never valid in a host). - for bad_host in ('evil;tls_verify=unsafe_off', 'a=b', 'h ost'): + for bad_host in ('evil;sslmode=disable', 'a=b', 'h ost'): with self.subTest(host=bad_host): with self.assertRaises(OidcConfigError): - self._qdb()._require_host(bad_host) + _require_host('https://db.example.com:9000', bad_host) # A legitimate host (incl. an IPv6 literal, which contains ':') is # still accepted — the guard must not over-reject. - self.assertEqual(self._qdb()._require_host('::1'), '::1') self.assertEqual( - self._qdb()._require_host('questdb.example.com'), + _require_host('https://db.example.com:9000', '::1'), '::1') + self.assertEqual( + _require_host('https://db.example.com:9000', 'questdb.example.com'), 'questdb.example.com') - # The guard fires through the adapter (sender), before the conf string - # is built and handed to Sender.from_conf. - qdb = self._qdb('https://realhost;tls_verify=unsafe_off:9000') - fake = types.ModuleType('questdb.ingress') - fake.Sender = object() # import must succeed so we reach the guard - with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): - with self.assertRaises(OidcConfigError): - qdb.sender() - - def test_sender_hostless_url_raises(self): - # The guard propagates through an adapter (not just the helper): - # sender() on a host-less URL raises OidcConfigError. See M5. - qdb = self._qdb('questdb:9000') - fake = types.ModuleType('questdb.ingress') - fake.Sender = object() # import must succeed so we reach the guard - with mock.patch.dict(sys.modules, {'questdb.ingress': fake}): - with self.assertRaises(OidcConfigError): - qdb.sender() - - def test_sql_missing_pandas_raises(self): - qdb = self._qdb() - with mock.patch.dict(sys.modules, {'pandas': None}): - with self.assertRaises(ImportError): - qdb.sql('SELECT 1') @unittest.skipIf(importlib.util.find_spec('sqlalchemy') is not None, 'sqlalchemy installed') def test_sqlalchemy_engine_missing_dep_raises(self): with self.assertRaises(ImportError): - self._qdb().sqlalchemy_engine() + sqlalchemy_engine(_FakeAuth(), 'https://db.example.com:9000') @unittest.skipIf(_HAS_PG_DRIVER, 'a PostgreSQL driver is installed') def test_psycopg_missing_dep_raises(self): with self.assertRaises(ImportError): - self._qdb().psycopg() + psycopg_connect(_FakeAuth(), 'https://db.example.com:9000') @unittest.skipIf(_HAS_PG_DRIVER, 'a PostgreSQL driver is installed') def test_pg_module_missing_chains_cause(self): # The "no PG driver" ImportError chains the underlying import failure # (raise ... from e) so the traceback preserves the real cause. - from questdb.auth._questdb import _pg_module + from questdb.auth._adapters import _pg_module with self.assertRaises(ImportError) as cm: _pg_module() self.assertIsInstance(cm.exception.__cause__, ImportError) - @unittest.skipIf(importlib.util.find_spec('questdb.ingress') is not None, - 'questdb.ingress extension is built') - def test_sender_missing_extension_raises(self): - with self.assertRaises(ImportError): - self._qdb().sender() - - def test_sender_rejects_non_integer_port(self): - # A non-integer port kwarg must be rejected before it can be - # interpolated into the addr= conf string, where ";tls_verify= - # unsafe_off" would silently disable TLS verification — the same - # injection _require_host() blocks for the host. The coercion runs - # before the extension import, so this fails cleanly even without it. - qdb = self._qdb('https://db.example.com:9000') - for bad in ('9000;tls_verify=unsafe_off', 'notaport', ['9000']): - with self.assertRaises(OidcConfigError): - qdb.sender(port=bad) - class TestConfigHelpers(unittest.TestCase): def test_as_bool_variants(self): From c189141f46ac5490b7330ef087884398d301a7f6 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 11:42:06 +0100 Subject: [PATCH 042/104] refactor(auth): drop the unused `flow` parameter from from_questdb `flow` was validated but never used: it was not threaded into resolve_config or the constructor, so 'auto' and 'device' behaved identically and 'loopback' only raised "not implemented". The device flow is the only one built, so the knob was inert. Remove the `flow` parameter, `_validate_flow`, `_VALID_FLOWS`, and the loopback-rejection test. Re-add if/when an Authorization-Code + PKCE (loopback) flow lands. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 15 --------------- test/test_auth.py | 7 ------- 2 files changed, 22 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 91a6747e..beefef9c 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -57,8 +57,6 @@ DEVICE_CODE_GRANT = 'urn:ietf:params:oauth:grant-type:device_code' REFRESH_GRANT = 'refresh_token' -_VALID_FLOWS = ('auto', 'device', 'loopback') - # A non-positive expires_in is non-conformant; treat it as "unknown". _DEFAULT_EXPIRES_IN = 3600 @@ -265,7 +263,6 @@ def from_questdb( discovery_url: Optional[str] = None, token_endpoint: Optional[str] = None, device_authorization_endpoint: Optional[str] = None, - flow: str = 'auto', cache: Any = 'memory', insecure: bool = False, ca_bundle: Optional[str] = None, @@ -284,7 +281,6 @@ def from_questdb( device-authorization endpoint when QuestDB doesn't advertise it. Any explicit keyword overrides discovery. """ - _validate_flow(flow) ctx = build_ssl_context(ca_bundle) cfg = resolve_config( questdb_url=url, @@ -785,17 +781,6 @@ def _maybe_open_browser(self, resp: Dict[str, Any]) -> None: pass -def _validate_flow(flow: str) -> None: - if flow not in _VALID_FLOWS: - raise OidcConfigError( - f'Unknown flow {flow!r}; expected one of {_VALID_FLOWS}.') - if flow == 'loopback': - raise OidcConfigError( - "The 'loopback' (Authorization Code + PKCE) flow is not yet " - "implemented. Use flow='device' (works on local and remote " - 'kernels alike).') - - def _normalize_url(url: str) -> str: # Full URL with scheme/host lower-cased and default port dropped, but path # kept (it distinguishes multi-tenant realms). Used for the cache key so diff --git a/test/test_auth.py b/test/test_auth.py index 7218218f..e41ba386 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1126,13 +1126,6 @@ def test_malformed_endpoint_port_raises_config_error(self): with self.assertRaises(OidcConfigError): OidcDeviceAuth.from_questdb(self.base, insecure=True) - def test_loopback_flow_not_implemented(self): - # Reserved-but-unimplemented flow raises an OidcError subclass so it's - # caught by `except OidcError` like other config problems. - with self.assertRaises(OidcConfigError): - OidcDeviceAuth.from_questdb(self.base, flow='loopback', - insecure=True) - def test_endpoint_origin_mismatch_rejected(self): # /settings advertises the device endpoint on a different origin than # the token endpoint: refuse rather than POST credentials off-origin. From 806be9c644aa105ae5982b2833f8f65bee3d6b73 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 12:19:54 +0100 Subject: [PATCH 043/104] refactor(auth): only trust absolute /settings endpoint URLs (match Java) QuestDB-advertised OIDC endpoints were resolved by assembling a URL from acl.oidc.host / acl.oidc.port / acl.oidc.tls.enabled when token.endpoint or device.authorization.endpoint came back path-only. The Java client does not do this: it reads only the six acl.oidc.* keys (enabled, client.id, scope, token.endpoint, device.authorization.endpoint, groups.encoded.in.token) and trusts an endpoint only as a complete URL. Match it. _resolve_endpoint now accepts a /settings endpoint only as an absolute http(s) URL; a path-only (or non-string) value reads as absent, so resolution falls back to the IdP .well-known document (which requires an issuer / discovery_url pin). Drop the acl.oidc.host / port / tls.enabled keys and the netloc-assembly machinery. This narrows the trust surface: host/port/tls.enabled are server building blocks, not a credential-routing source. Update the unit tests and add a discovery-level test asserting path-only endpoints are not assembled. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 56 +++++++++-------------------- test/test_auth.py | 66 ++++++++++++++++------------------ 2 files changed, 47 insertions(+), 75 deletions(-) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 3ddab02d..c9324360 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -52,9 +52,6 @@ _K_DEVICE_ENDPOINT = 'acl.oidc.device.authorization.endpoint' # design §7 (new) _K_GROUPS_IN_TOKEN = 'acl.oidc.groups.encoded.in.token' _K_AUDIENCE = 'acl.oidc.audience' -_K_HOST = 'acl.oidc.host' -_K_PORT = 'acl.oidc.port' -_K_TLS_ENABLED = 'acl.oidc.tls.enabled' @dataclass @@ -267,43 +264,22 @@ def validate_endpoint_origins( 'issuer.') -def _resolve_endpoint(value: Optional[str], cfg: Dict[str, Any]) -> Optional[str]: +def _resolve_endpoint(value: Any) -> Optional[str]: """ - Turn a possibly-relative endpoint into a full URL. - - QuestDB usually exports fully-resolved URLs, but some deployments store - only the path (e.g. ``/as/token.oauth2``) alongside ``acl.oidc.host``. + A ``/settings`` endpoint, trusted only as a complete ``http(s)`` URL. + + Mirroring the Java client, a QuestDB-advertised endpoint is taken verbatim + and only as an absolute URL. A path-only (or otherwise non-absolute, or + non-string) value is treated as absent, so resolution falls back to the IdP + ``.well-known`` document — which requires an issuer / ``discovery_url`` pin. + We deliberately do **not** assemble a URL from ``acl.oidc.host`` / + ``acl.oidc.port`` / ``acl.oidc.tls.enabled``: those are server building + blocks, not a credential-routing source the client should trust. """ - if not value: - return None - if not isinstance(value, str): - # Non-string endpoint (e.g. a JSON number): treat as absent so resolution - # yields a clear OidcConfigError instead of an AttributeError from - # .startswith() escaping the typed-error contract. - return None - if value.startswith('http://') or value.startswith('https://'): + value = _str_setting(value) + if value and (value.startswith('https://') or value.startswith('http://')): return value - if value.startswith('/'): - # _str_setting drops a non-string acl.oidc.host so it can't be - # interpolated raw into the netloc (e.g. https://12345:9000/path). - host = _str_setting(cfg.get(_K_HOST)) - if not host: - # Path-only endpoint with no host to resolve against: treat as absent - # for the clear "could not resolve" error, rather than passing a - # scheme-less "/path" on to a confusing "malformed URL" downstream. - return None - tls = _as_bool(cfg.get(_K_TLS_ENABLED), default=True) - scheme = 'https' if tls else 'http' - # A usable port is an int or digit string; anything else would corrupt - # the netloc, so drop it and resolve host-only. - port = cfg.get(_K_PORT) - if isinstance(port, bool) or not ( - isinstance(port, int) - or (isinstance(port, str) and port.isdigit())): - port = None - netloc = f'{host}:{port}' if port else host - return f'{scheme}://{netloc}{value}' - return value + return None def well_known_url(issuer: str) -> str: @@ -394,13 +370,13 @@ def resolve_config( explicit_device_endpoint = device_authorization_endpoint is not None token_endpoint = ( - token_endpoint or _resolve_endpoint(cfg.get(_K_TOKEN_ENDPOINT), cfg)) + token_endpoint or _resolve_endpoint(cfg.get(_K_TOKEN_ENDPOINT))) authorization_endpoint = ( authorization_endpoint - or _resolve_endpoint(cfg.get(_K_AUTHORIZATION_ENDPOINT), cfg)) + or _resolve_endpoint(cfg.get(_K_AUTHORIZATION_ENDPOINT))) device_authorization_endpoint = ( device_authorization_endpoint - or _resolve_endpoint(cfg.get(_K_DEVICE_ENDPOINT), cfg)) + or _resolve_endpoint(cfg.get(_K_DEVICE_ENDPOINT))) # Over a plaintext-http /settings channel (insecure=True, non-loopback), a # tampered response can advertise BOTH credential endpoints at one attacker diff --git a/test/test_auth.py b/test/test_auth.py index e41ba386..cc7e9183 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -965,6 +965,25 @@ def test_groups_mode_defaults_to_access_token_when_unset(self): self.assertFalse(auth.config.groups_in_token) self.assertEqual(auth.token(), ACCESS_TOKEN) + def test_settings_path_only_endpoints_not_assembled(self): + # We do NOT assemble endpoint URLs from acl.oidc.host / port / + # tls.enabled (matching the Java client). A path-only endpoint reads as + # absent, so with no issuer to drive .well-known discovery, from_questdb + # fails with a clear OidcConfigError instead of building a URL from the + # host/port/tls building blocks. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.host': 'idp.example.com', + 'acl.oidc.port': 443, + 'acl.oidc.tls.enabled': True, + 'acl.oidc.token.endpoint': '/oauth/token', + 'acl.oidc.device.authorization.endpoint': '/oauth/device', + }} + with self.assertRaises(OidcConfigError): + OidcDeviceAuth.from_questdb( + self.base, insecure=True, renderer=Renderer()) + def test_user_writable_preferences_cannot_override_config(self): # A user-writable "preferences" sibling in /settings must never override # the trusted "config" object during discovery: end-to-end, the resolved @@ -1609,21 +1628,24 @@ def test_as_bool_variants(self): self.assertIsNone(_as_bool(None)) self.assertIs(_as_bool(None, default=True), True) - def test_resolve_endpoint_relative_path(self): + def test_resolve_endpoint_accepts_only_absolute_url(self): + # Matching the Java client, a /settings endpoint is trusted only as a + # complete http(s) URL. A path-only value is NOT assembled from + # acl.oidc.host / port / tls.enabled (no longer read) — it reads as + # absent so resolution falls back to .well-known discovery. from questdb.auth._discovery import _resolve_endpoint - cfg = {'acl.oidc.host': 'idp.example.com', - 'acl.oidc.tls.enabled': True, 'acl.oidc.port': 443} - self.assertEqual(_resolve_endpoint('/as/token.oauth2', cfg), - 'https://idp.example.com:443/as/token.oauth2') - self.assertEqual(_resolve_endpoint('https://idp/x', cfg), - 'https://idp/x') # absolute is kept verbatim + self.assertEqual(_resolve_endpoint('https://idp/x'), 'https://idp/x') + self.assertEqual(_resolve_endpoint('http://idp:9000/x'), + 'http://idp:9000/x') + self.assertIsNone(_resolve_endpoint('/as/token.oauth2')) + self.assertIsNone(_resolve_endpoint('')) def test_resolve_endpoint_ignores_non_string(self): # A non-string endpoint from /settings (e.g. a JSON number) must be # treated as absent, not raise AttributeError from .startswith(). M3. from questdb.auth._discovery import _resolve_endpoint - self.assertIsNone(_resolve_endpoint(8080, {})) - self.assertIsNone(_resolve_endpoint(True, {})) + self.assertIsNone(_resolve_endpoint(8080)) + self.assertIsNone(_resolve_endpoint(True)) def test_str_setting_ignores_non_string(self): # A non-empty string passes through; anything else (a JSON list / @@ -1700,32 +1722,6 @@ def from_discovery(well_known, **kw): self.assertIsNone(auth.config.issuer) self.assertTrue(auth.cache_key) - def test_resolve_endpoint_relative_path_without_host_is_none(self): - # A path-only endpoint with no acl.oidc.host can't be resolved; it must - # be treated as absent (None) so resolution fails with a clear "could - # not resolve the ... endpoint" error rather than a scheme-less "/path" - # that later surfaces as a confusing "insecure/malformed URL". - from questdb.auth._discovery import _resolve_endpoint - self.assertIsNone(_resolve_endpoint('/as/token.oauth2', {})) - self.assertIsNone( # port present but host missing -> still unresolved - _resolve_endpoint('/as/token.oauth2', {'acl.oidc.port': 443})) - - def test_resolve_endpoint_ignores_non_string_host(self): - # A non-string acl.oidc.host (a JSON number/list from a buggy or hostile - # /settings) must not be interpolated raw into the netloc (e.g. - # https://12345:9000/path); treat it as absent so a path-only endpoint - # reads as unresolvable, mirroring how endpoint values are coerced. - from questdb.auth._discovery import _resolve_endpoint - for bad_host in (12345, ['idp'], {'h': 'idp'}, True): - self.assertIsNone( - _resolve_endpoint('/as/token', {'acl.oidc.host': bad_host})) - # A non-numeric port is dropped rather than corrupting the netloc. - self.assertEqual( - _resolve_endpoint('/as/token', { - 'acl.oidc.host': 'idp', 'acl.oidc.tls.enabled': True, - 'acl.oidc.port': ['x']}), - 'https://idp/as/token') - def test_settings_config_nesting(self): from questdb.auth._discovery import settings_config self.assertEqual(settings_config({'config': {'a': 1}}), {'a': 1}) From 19f46d060db1aedac12dbe4887ad3549c233aeb2 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 12:28:35 +0100 Subject: [PATCH 044/104] feat(auth): open the verification URL in a browser by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit open_browser now defaults to True, so signing in always tries to open the device-verification URL when possible. The existing guards keep it safe: it is skipped on a (possibly remote) notebook kernel — where the prompt is already a clickable link and the kernel host is not the user's machine — only http(s) URLs are opened, and any failure is swallowed. Pass open_browser=False to disable. The test fixture stubs webbrowser.open so the device-flow suite never spawns a real browser, plus regression tests for the default, the end-to-end open on sign-in, and notebook-kernel suppression. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/auth.rst | 5 +++++ src/questdb/auth/_device.py | 9 ++++---- test/test_auth.py | 42 +++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/docs/auth.rst b/docs/auth.rst index 2405b86a..726cb177 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -57,6 +57,11 @@ Jupyter, plain text on a terminal):: ⏳ waiting for authorization… (4:51 left) ✅ Signed in as alice@example.com — token cached, expires in 60 min +On a local terminal the verification URL is also opened in your default browser +automatically (pass ``open_browser=False`` to disable); on a notebook kernel it +is not — the kernel host isn't your machine, so the clickable link above is +used instead. + Re-running is silent — the token is cached and refreshed silently on the next use once it nears expiry. diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index beefef9c..0a6f68d0 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -182,7 +182,7 @@ def __init__( cache: Any = 'memory', insecure: bool = False, ca_bundle: Optional[str] = None, - open_browser: bool = False, + open_browser: bool = True, interactive: Optional[bool] = None, qr: bool = False, renderer: Optional[Renderer] = None, @@ -266,7 +266,7 @@ def from_questdb( cache: Any = 'memory', insecure: bool = False, ca_bundle: Optional[str] = None, - open_browser: bool = False, + open_browser: bool = True, interactive: Optional[bool] = None, qr: bool = False, renderer: Optional[Renderer] = None, @@ -764,8 +764,9 @@ def _is_interactive(self) -> bool: return detect_interactive() def _maybe_open_browser(self, resp: Dict[str, Any]) -> None: - # Never auto-open on a (possibly remote) notebook kernel; only on an - # opted-in local terminal. + # Open on a local terminal by default; never on a (possibly remote) + # notebook kernel, where the prompt is already a clickable link and the + # kernel host isn't the user's machine. Suppress with open_browser=False. if not self.open_browser or in_ipython_kernel(): return # Only http(s) — never a javascript:/data: scheme from a malicious or diff --git a/test/test_auth.py b/test/test_auth.py index cc7e9183..95bd1c03 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -302,6 +302,12 @@ class AuthTestBase(unittest.TestCase): def setUp(self): _MEMORY_STORE.clear() _MEMORY_GENERATION.clear() + # open_browser defaults to True, so stub webbrowser.open: device-flow + # tests must never spawn a real browser. Tests asserting open/skip + # behaviour use self.mock_browser_open (or patch it themselves). + patcher = mock.patch('webbrowser.open') + self.mock_browser_open = patcher.start() + self.addCleanup(patcher.stop) self.server = _MockServer() self.state = self.server.state self.thread = threading.Thread( @@ -713,6 +719,42 @@ def test_open_browser_rejects_dangerous_scheme(self): {'verification_uri': 'https://idp.example.com/device'}) opener.assert_called_once_with('https://idp.example.com/device') + def test_open_browser_default_is_true(self): + # We try to open the browser by default ("always when possible"), via + # both the explicit constructor and discovery. + auth = OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint=self.base + '/device', + token_endpoint=self.base + '/token', + insecure=True, renderer=Renderer(), _clock=FakeClock()) + self.assertTrue(auth.open_browser) + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': self.base + '/token', + 'acl.oidc.device.authorization.endpoint': self.base + '/device'}} + disc = OidcDeviceAuth.from_questdb( + self.base, insecure=True, renderer=Renderer(), _clock=FakeClock()) + self.assertTrue(disc.open_browser) + + def test_signin_opens_browser_by_default(self): + # On a (non-kernel) terminal, signing in opens the verification URL with + # no opt-in — make_auth() leaves open_browser at its default. + auth = self.make_auth() + auth.token() + self.mock_browser_open.assert_called_once_with( + 'https://idp.example.com/device?user_code=WDJB-MJHT') + + def test_open_browser_suppressed_in_notebook_kernel(self): + # Never open on a (possibly remote) notebook kernel, even when enabled: + # the kernel host is not the user's machine. + auth = self.make_auth(open_browser=True) + with mock.patch('questdb.auth._device.in_ipython_kernel', + return_value=True): + auth._maybe_open_browser( + {'verification_uri': 'https://idp.example.com/device'}) + self.mock_browser_open.assert_not_called() + def test_memory_cache_returns_independent_copy(self): cache = MemoryCache() stored = TokenSet(access_token='a', refresh_token='r', expires_at=1.0) From f567e266b3171d3e0ba8c6c2513b559ae1f98b2f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 17:00:18 +0100 Subject: [PATCH 045/104] fix(auth): floor the device-flow poll interval at 5s (RFC 8628 default) The poll-interval clamp floored at 1s, which could poll the IdP faster than the RFC 8628 baseline and turned a malformed interval=0 into 1s of hammering. Raise the floor to the spec's default of 5s via a named _MIN_POLL_INTERVAL (mirroring _MAX_POLL_INTERVAL); the [5s, 60s] band keeps the tight upper cap that bounds the lock-holding sleep. Add a min-clamp test. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 8 +++++--- test/test_auth.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 0a6f68d0..ba7a2ac7 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -66,6 +66,7 @@ # (and lock) alive indefinitely. _DEFAULT_DEVICE_CODE_LIFETIME = 600 # expires_in fallback (absent/invalid/<=0) _MAX_DEVICE_CODE_LIFETIME = 1800 # cap on how long we keep polling +_MIN_POLL_INTERVAL = 5 # floor on the poll interval (RFC 8628 default) _MAX_POLL_INTERVAL = 60 # cap on the poll interval (incl. slow_down) @@ -652,9 +653,10 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: interval = int(resp.get('interval', self._default_interval)) except (TypeError, ValueError, OverflowError): interval = self._default_interval - # At least 1s (RFC 8628 floor), capped so a hostile value can't pin the - # polling thread (which holds the lock) in one enormous sleep. - interval = min(_MAX_POLL_INTERVAL, max(1, interval)) + # Floor at the RFC 8628 default (5s) so we never poll faster than the + # spec baseline; cap so a hostile value can't pin the polling thread + # (which holds the lock) in one enormous sleep. + interval = min(_MAX_POLL_INTERVAL, max(_MIN_POLL_INTERVAL, interval)) try: expires_in = int(resp.get('expires_in', _DEFAULT_DEVICE_CODE_LIFETIME)) except (TypeError, ValueError, OverflowError): diff --git a/test/test_auth.py b/test/test_auth.py index 95bd1c03..899c14c6 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -513,6 +513,20 @@ def test_oversized_interval_is_clamped(self): self.assertTrue(self._clock.sleeps) self.assertLessEqual(max(self._clock.sleeps), _MAX_POLL_INTERVAL) + def test_small_interval_clamped_to_min(self): + # A sub-5s advertised interval is raised to the RFC 8628 default (5s): + # we never poll faster than the spec baseline. + from questdb.auth._device import _MIN_POLL_INTERVAL + self.state.device_response = { + 'device_code': 'DEV-CODE', 'user_code': 'X', + 'verification_uri': 'https://idp/device', + 'expires_in': 600, 'interval': 1, + } + self.state.token_script = [(200, None)] + auth = self.make_auth() + auth.token() + self.assertEqual(self._clock.sleeps, [_MIN_POLL_INTERVAL]) + def test_oversized_expires_in_is_capped(self): # A hostile expires_in must not keep the poll loop (and the lock) alive # indefinitely; the lifetime is capped so a never-authorized flow still From 9f5035f4974300e16ed988c1da92a6972cb87c0b Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 17:31:34 +0100 Subject: [PATCH 046/104] fix(auth): bound the HTTP response read (size cap + wall-clock deadline) urllib's timeout is per-socket-read, so a hostile or stalled server could dribble the body (a byte just inside each timeout window) to keep a bare resp.read() running indefinitely, or send a huge body that buffers unbounded into memory. Read in chunks via _read_body, enforcing a 4 MiB cap and a whole-read deadline (= the request timeout), on both the success and HTTPError body reads; exceeding either raises OidcNetworkError. Reachable via a buggy/compromised IdP or a MITM'd plaintext /settings (insecure=True), both fetched through this path. Mirrors the Java client's bounded reads. Adds direct tests for the cap, the dribble deadline, and the normal path. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_http.py | 49 +++++++++++++++++++++++++++++++++++---- test/test_auth.py | 35 ++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 5 deletions(-) diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index a07444ed..0e6b6c6f 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -42,6 +42,7 @@ import json import os import ssl +import time import urllib.error import urllib.parse import urllib.request @@ -52,6 +53,15 @@ _DEFAULT_TIMEOUT = 30 _USER_AGENT = 'questdb-python-client (oidc-auth)' +# Bound a response body by total size and wall-clock time. urllib's timeout is +# per-socket-read, so a server dribbling the body (a byte just inside each +# timeout window) could keep a bare read() running indefinitely, and a huge +# body would buffer unbounded into memory. OIDC / JSON responses are KBs, so +# 4 MiB is ample headroom. +_MAX_RESPONSE_BYTES = 4 * 1024 * 1024 +_READ_CHUNK = 65536 +_monotonic = time.monotonic + def build_ssl_context(ca_bundle: Optional[str] = None) -> ssl.SSLContext: """ @@ -172,6 +182,32 @@ def _opener(ctx: Optional[ssl.SSLContext]) -> urllib.request.OpenerDirector: return urllib.request.build_opener(*handlers) +def _read_body(resp: Any, *, max_bytes: int, deadline: float) -> bytes: + """ + Read a response body bounded by a total byte cap and a wall-clock deadline. + + Reads in chunks so a hostile or stalled server can neither dribble the body + past the caller's timeout (urllib's timeout is per-socket-read, not a + whole-read bound) nor exhaust memory with an unbounded body. + """ + chunks = [] + total = 0 + while True: + if _monotonic() > deadline: + raise OidcNetworkError( + 'Timed out reading the response body; the server is too slow ' + 'or is dribbling data.') + chunk = resp.read(_READ_CHUNK) + if not chunk: + return b''.join(chunks) + total += len(chunk) + if total > max_bytes: + raise OidcNetworkError( + f'Response body exceeded the {max_bytes}-byte limit; refusing ' + 'to buffer an unbounded response.') + chunks.append(chunk) + + def request( method: str, url: str, @@ -207,14 +243,17 @@ def request( with _opener(ctx).open(req, timeout=timeout) as resp: return HttpResponse( getattr(resp, 'status', resp.getcode()), - resp.read(), + _read_body(resp, max_bytes=_MAX_RESPONSE_BYTES, + deadline=_monotonic() + timeout), resp.headers) except urllib.error.HTTPError as e: - # 4xx/5xx still carry a (possibly JSON) body to inspect. Map a mid-body - # read failure to a network error, and close the response so its socket - # isn't leaked (the poll loop drives many 400s during a long sign-in). + # 4xx/5xx still carry a (possibly JSON) body to inspect. Bound the read + # (same cap/deadline), map a mid-body read failure to a network error, + # and close the response so its socket isn't leaked (the poll loop drives + # many 400s during a long sign-in). try: - body = e.read() + body = _read_body(e, max_bytes=_MAX_RESPONSE_BYTES, + deadline=_monotonic() + timeout) except (TimeoutError, OSError) as read_err: raise OidcNetworkError( f'Failed to read response from {url}: {read_err}') from read_err diff --git a/test/test_auth.py b/test/test_auth.py index 899c14c6..b80cfbf2 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -90,6 +90,16 @@ def headers(self): return {'Authorization': f'Bearer {self._token}'} +class _ChunkStream: + """A response stub whose read(n) yields preset chunks, then b'' at EOF.""" + + def __init__(self, *chunks): + self._chunks = list(chunks) + + def read(self, n): + return self._chunks.pop(0) if self._chunks else b'' + + def _jwt(claims): """Build an unsigned JWT-shaped string with the given payload claims.""" def b64(obj): @@ -2085,6 +2095,31 @@ def test_require_secure_rejects_malformed_ipv6(self): # A well-formed IPv6 URL is still accepted (loopback http is allowed). _require_secure('http://[::1]:8080/x', insecure=False) + def test_read_body_accepts_normal_body(self): + from questdb.auth._http import _read_body + resp = _ChunkStream(b'hello ', b'world') + self.assertEqual( + _read_body(resp, max_bytes=1000, deadline=1e18), b'hello world') + + def test_read_body_rejects_oversized(self): + # A body over the cap raises instead of buffering unbounded into memory. + from questdb.auth._http import _read_body + resp = _ChunkStream(b'x' * 60, b'y' * 60) # 120 bytes > 100-byte cap + with self.assertRaises(OidcNetworkError): + _read_body(resp, max_bytes=100, deadline=1e18) + + def test_read_body_aborts_on_slow_dribble(self): + # A steady dribble that never trips the per-read socket timeout must + # still abort once the whole-read wall-clock deadline passes — the gap + # urllib's per-operation timeout leaves open. + from questdb.auth import _http + body = _ChunkStream(*([b'a'] * 10000)) # endless trickle + ticks = iter([0.0, 0.2, 0.4, 2.0]) # advance past deadline=1.0 + with mock.patch.object(_http, '_monotonic', + lambda: next(ticks, 100.0)): + with self.assertRaises(OidcNetworkError): + _http._read_body(body, max_bytes=10 ** 9, deadline=1.0) + def test_bad_ca_bundle_raises_config_error(self): # A missing or invalid CA bundle path (explicit or via env) must surface # as OidcConfigError, not a raw FileNotFoundError / ssl.SSLError. See M1. From 2d75e01b099905c58e9fd053d0443e1051afb14a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 17:58:12 +0100 Subject: [PATCH 047/104] fix(test): drop stale REST-adapter test imports from test.py Commit b23e5db deleted the REST /exec adapter and its TestRestAdapter / TestRestAdapterAuthErrors test classes (replaced by TestAdapters), but left the now-dangling import in test/test.py. The standalone test_auth.py run stayed green, masking it; the aggregate `python test/test.py -v` step ran in CI failed at import with "cannot import name 'TestRestAdapter'". Remove the two dead names. TestAdapters is already imported, so adapter coverage is unchanged; the auth suite runs 138 tests, OK. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/test.py b/test/test.py index 054efe5e..9d1eda6f 100755 --- a/test/test.py +++ b/test/test.py @@ -41,8 +41,6 @@ TestRefresh, TestDiscovery, TestInsecureSettingsGuard, - TestRestAdapter, - TestRestAdapterAuthErrors, TestAdapters, TestConcurrency, TestConfigHelpers, From b32207fa555c8fe6c5cf54e7f977cb9b8f1fda10 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 18:33:23 +0100 Subject: [PATCH 048/104] fix(auth): clamp token lifetime to match the Java client When the IdP omits (or zeroes) expires_in, fall back to 300s instead of 3600s, and cap a positive IdP-stated lifetime at 3600s. This mirrors the Java client (DEFAULT_TOKEN_TTL_SECONDS=300, MAX_EXPIRES_IN_SECONDS=3600) so a cached token is re-validated at least hourly rather than trusting a multi-hour or unbounded lifetime. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 12 ++++++++++-- test/test_auth.py | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index ba7a2ac7..f2f27a5b 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -57,8 +57,13 @@ DEVICE_CODE_GRANT = 'urn:ietf:params:oauth:grant-type:device_code' REFRESH_GRANT = 'refresh_token' -# A non-positive expires_in is non-conformant; treat it as "unknown". -_DEFAULT_EXPIRES_IN = 3600 +# Clamp the token lifetime (access/id-token TTL) the same way as the Java +# client. An absent or non-positive expires_in is non-conformant; fall back to a +# short, conservative lifetime so a token with no stated lifetime is refreshed +# promptly. A very long (or hostile) IdP-stated lifetime is capped so a cached +# token is re-validated at least hourly. +_DEFAULT_EXPIRES_IN = 300 # token TTL fallback (absent/invalid/<=0) +_MAX_EXPIRES_IN = 3600 # cap on the token TTL # Clamp the device-authorization timing fields (RFC 8628): a hostile/buggy # response must not time the flow out before its first poll, pin the polling @@ -516,6 +521,9 @@ def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: # A non-positive lifetime marks a just-issued token as expired, # causing refresh/re-prompt churn. Treat it as unknown. expires_in = _DEFAULT_EXPIRES_IN + # Cap a long (or hostile) IdP-stated lifetime so a cached token is + # re-validated at least hourly (matches the Java client). + expires_in = min(expires_in, _MAX_EXPIRES_IN) claims = (_decode_jwt_claims(body.get('id_token')) or _decode_jwt_claims(body.get('access_token'))) now = self._now() diff --git a/test/test_auth.py b/test/test_auth.py index b80cfbf2..84526015 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -667,6 +667,28 @@ def test_overflow_expires_in_treated_as_unknown(self): self.assertEqual(auth.token(), ID_TOKEN) self.assertTrue(auth._tokens.is_valid(self._clock.now())) + def test_missing_expires_in_defaults_to_short_ttl(self): + # When the IdP omits expires_in, fall back to a short, conservative TTL + # (300s) so the token is refreshed promptly, matching the Java client. + self.state.token_script = [(200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'token_type': 'Bearer'})] # no expires_in + auth = self.make_auth() + auth.token() + t = auth._tokens + self.assertEqual(round(t.expires_at - t.issued_at), 300) + + def test_oversized_token_expires_in_is_capped(self): + # A very long (or hostile) token lifetime is capped at 3600s so a cached + # token is re-validated at least hourly, matching the Java client. + self.state.token_script = [(200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'token_type': 'Bearer', 'expires_in': 10 ** 9})] + auth = self.make_auth() + auth.token() + t = auth._tokens + self.assertEqual(round(t.expires_at - t.issued_at), 3600) + def test_overflow_device_timing_fields_do_not_crash(self): # Non-finite interval / expires_in in the device-auth response (JSON # Infinity) must be treated as unknown, not raise OverflowError. See M1. From 48aae3b38c776257a4098b3a18cf7b50c4c6a9c8 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 18:49:28 +0100 Subject: [PATCH 049/104] refactor(auth): drop the pluggable cache; memory cache is always on Remove the 'cache' parameter from OidcDeviceAuth.__init__/from_questdb, the TokenCache interface, NullCache, and the make_cache factory. The in-process memory cache is the only sensible backend, so it is now unconditional (MemoryCache, kept internal). _store/_cache_generation call the cache directly instead of probing for optional methods. Drops the three cache classes from the public API and docs; TokenSet stays. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/api.rst | 15 ---------- docs/auth.rst | 16 ++++------- src/questdb/auth/__init__.py | 5 +--- src/questdb/auth/_cache.py | 56 ++++-------------------------------- src/questdb/auth/_device.py | 30 ++++++------------- test/test_auth.py | 23 ++++----------- 6 files changed, 26 insertions(+), 119 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 54d7a654..a9cc14aa 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -87,26 +87,11 @@ See the :ref:`oidc_auth` guide for an overview. :undoc-members: :show-inheritance: -.. autoclass:: questdb.auth.TokenCache - :members: - :undoc-members: - :show-inheritance: - .. autoclass:: questdb.auth.TokenSet :members: :undoc-members: :show-inheritance: -.. autoclass:: questdb.auth.MemoryCache - :members: - :undoc-members: - :show-inheritance: - -.. autoclass:: questdb.auth.NullCache - :members: - :undoc-members: - :show-inheritance: - .. autoexception:: questdb.auth.OidcError :show-inheritance: diff --git a/docs/auth.rst b/docs/auth.rst index 726cb177..8d1080d4 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -129,8 +129,7 @@ entirely: token_endpoint="https://idp/.../token", scope="openid groups", groups_in_token=True, # send id_token (True) vs access_token (False) - audience="questdb", # optional; some IdPs need it to set `aud` - cache="memory") + audience="questdb") # optional; some IdPs need it to set `aud` Which token is sent ------------------- @@ -160,15 +159,10 @@ margin). When it nears expiry the helper silently refreshes it using the is raised instead, so you can retry without being needlessly re-prompted. A lock serializes refresh so parallel cells/threads don't double-prompt. -Cache backends (``cache=`` argument): - -* ``"memory"`` *(default)* — process-global, nothing written to disk. - Re-running cells is silent; a kernel restart re-prompts once. -* ``None`` — never persist; prompt every time. - -Tokens are deliberately never written to disk: a kernel restart re-prompts -(an interactive sign-in is cheap relative to the risk of a refresh token -sitting in a plaintext file at rest). +The token is held in a process-global, in-memory cache, so re-running a cell +reuses it instead of re-prompting; a kernel restart re-prompts once. Tokens are +deliberately never written to disk: an interactive sign-in is cheap relative to +the risk of a refresh token sitting in a plaintext file at rest. Non-interactive contexts ------------------------- diff --git a/src/questdb/auth/__init__.py b/src/questdb/auth/__init__.py index 3623830f..1e8b190b 100644 --- a/src/questdb/auth/__init__.py +++ b/src/questdb/auth/__init__.py @@ -52,7 +52,7 @@ from ._device import OidcDeviceAuth from ._discovery import OidcConfig -from ._cache import TokenCache, TokenSet, MemoryCache, NullCache +from ._cache import TokenSet from ._errors import ( OidcError, OidcConfigError, @@ -64,8 +64,6 @@ from ._adapters import sqlalchemy_engine, psycopg_connect __all__ = [ - 'MemoryCache', - 'NullCache', 'OidcConfig', 'OidcConfigError', 'OidcDeviceAuth', @@ -74,7 +72,6 @@ 'OidcInteractionRequired', 'OidcNetworkError', 'OidcTimeoutError', - 'TokenCache', 'TokenSet', 'psycopg_connect', 'sqlalchemy_engine', diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index b8373fb6..b7c5ea61 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -22,15 +22,13 @@ ## ################################################################################ -"""Token state and cache backends for :mod:`questdb.auth`.""" +"""Token state and the in-memory token cache for :mod:`questdb.auth`.""" from __future__ import annotations import threading from dataclasses import dataclass, field, replace -from typing import Dict, Optional, Union - -from ._errors import OidcConfigError +from typing import Dict, Optional # Refresh a little before the real expiry to absorb clock skew / latency. DEFAULT_SKEW_SECONDS = 30 @@ -70,19 +68,6 @@ def is_valid(self, now: float, skew: float = DEFAULT_SKEW_SECONDS) -> bool: return now < (self.expires_at - skew) -class TokenCache: - """Interface for token caches.""" - - def load(self, key: str) -> Optional[TokenSet]: # pragma: no cover - raise NotImplementedError - - def store(self, key: str, tokens: TokenSet) -> None: # pragma: no cover - raise NotImplementedError - - def clear(self, key: str) -> None: # pragma: no cover - raise NotImplementedError - - # Module-global so a re-run notebook cell (fresh ``OidcDeviceAuth``) reuses the # acquired token instead of re-prompting. _MEMORY_STORE: Dict[str, TokenSet] = {} @@ -94,12 +79,12 @@ def clear(self, key: str) -> None: # pragma: no cover _MEMORY_LOCK = threading.Lock() -class MemoryCache(TokenCache): +class MemoryCache: """ - Process-global, in-memory cache (the default). + Process-global, in-memory token cache (always on). - Safest backend: nothing hits disk. Tokens live for the life of the process, - so re-running cells is silent; a kernel restart re-prompts once. + Nothing ever hits disk. Tokens live for the life of the process, so + re-running cells is silent; a kernel restart re-prompts once. """ def load(self, key: str) -> Optional[TokenSet]: @@ -142,32 +127,3 @@ def store_if_current( return False _MEMORY_STORE[key] = replace(tokens) return True - - -class NullCache(TokenCache): - """Never persists anything; prompts every time.""" - - def load(self, key: str) -> Optional[TokenSet]: - return None - - def store(self, key: str, tokens: TokenSet) -> None: - pass - - def clear(self, key: str) -> None: - pass - - -_CacheSpec = Union[str, None, TokenCache] - - -def make_cache(spec: _CacheSpec) -> TokenCache: - """Resolve a cache spec (``"memory"`` / ``None`` / a TokenCache instance).""" - if isinstance(spec, TokenCache): - return spec - if spec is None or spec == 'none': - return NullCache() - if spec == 'memory': - return MemoryCache() - raise OidcConfigError( - f'Unknown cache backend {spec!r}; ' - "expected 'memory', None, or a TokenCache instance.") diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index f2f27a5b..776b6067 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -35,7 +35,7 @@ from dataclasses import replace from typing import Any, Dict, Optional -from ._cache import TokenSet, make_cache +from ._cache import MemoryCache, TokenSet from ._discovery import OidcConfig, resolve_config, validate_endpoint_origins from ._errors import ( OidcConfigError, @@ -171,8 +171,7 @@ class OidcDeviceAuth: token_endpoint="https://idp/.../token", scope="openid groups", groups_in_token=True, - audience="questdb", - cache="memory") + audience="questdb") """ def __init__( @@ -185,7 +184,6 @@ def __init__( groups_in_token: bool = False, audience: Optional[str] = None, issuer: Optional[str] = None, - cache: Any = 'memory', insecure: bool = False, ca_bundle: Optional[str] = None, open_browser: bool = True, @@ -238,7 +236,7 @@ def __init__( # the IdP stalls; the total poll duration is separately capped by # _MAX_DEVICE_CODE_LIFETIME. self._timeout = timeout - self._cache = make_cache(cache) + self._cache = MemoryCache() self._ctx = build_ssl_context(ca_bundle) self._renderer = renderer if renderer is not None else make_renderer(qr=qr) # Serializes token *acquisition* (silent refresh or interactive sign-in) @@ -269,7 +267,6 @@ def from_questdb( discovery_url: Optional[str] = None, token_endpoint: Optional[str] = None, device_authorization_endpoint: Optional[str] = None, - cache: Any = 'memory', insecure: bool = False, ca_bundle: Optional[str] = None, open_browser: bool = True, @@ -309,7 +306,6 @@ def from_questdb( groups_in_token=cfg.groups_in_token, audience=cfg.audience, issuer=cfg.issuer, - cache=cache, insecure=insecure, ca_bundle=ca_bundle, open_browser=open_browser, @@ -492,23 +488,15 @@ def _acquire(self, generation: int) -> TokenSet: def _store(self, tokens: TokenSet, generation: int) -> None: # self._tokens is this instance's own view, so always set it (the caller # uses what it just acquired). The shared-cache write is conditional: a - # clear() (here or on another instance sharing the store) that bumped the - # generation drops the write, so clear() isn't silently undone. Backends - # without generation support (NullCache / custom TokenCache) store - # unconditionally. + # clear() (here or on another instance sharing the process-global store) + # that bumped the generation drops the write, so clear() isn't silently + # undone. self._tokens = tokens - store_if_current = getattr(self._cache, 'store_if_current', None) - if store_if_current is not None: - store_if_current(self.cache_key, tokens, generation) - else: - self._cache.store(self.cache_key, tokens) + self._cache.store_if_current(self.cache_key, tokens, generation) def _cache_generation(self) -> int: - # MemoryCache tracks a per-key clear()-generation for the cross-instance - # CAS in _store; other backends don't, so default to 0 (unconditional - # store). - generation = getattr(self._cache, 'generation', None) - return generation(self.cache_key) if generation is not None else 0 + # Per-key clear()-generation for the cross-instance CAS in _store. + return self._cache.generation(self.cache_key) def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: try: diff --git a/test/test_auth.py b/test/test_auth.py index 84526015..ddd6cbaf 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -331,7 +331,7 @@ def tearDown(self): self.server.server_close() self.thread.join(timeout=5) - def make_auth(self, *, clock=None, groups_in_token=True, cache='memory', + def make_auth(self, *, clock=None, groups_in_token=True, interactive=True, **kw): clock = clock or FakeClock() self._clock = clock @@ -341,7 +341,6 @@ def make_auth(self, *, clock=None, groups_in_token=True, cache='memory', token_endpoint=self.base + '/token', scope='openid groups', groups_in_token=groups_in_token, - cache=cache, insecure=True, interactive=interactive, renderer=Renderer(), @@ -631,7 +630,7 @@ def test_openid_scope_auto_added_for_groups_in_token(self): device_authorization_endpoint=self.base + '/device', token_endpoint=self.base + '/token', scope='groups', groups_in_token=True, # no 'openid' - cache='memory', insecure=True, renderer=Renderer()) + insecure=True, renderer=Renderer()) self.assertIn('openid', auth.config.scope.split()) def test_zero_expires_in_is_treated_as_unknown(self): @@ -942,7 +941,7 @@ def test_refresh_network_error_propagates_without_reprompt(self): client_id='questdb', device_authorization_endpoint='http://127.0.0.1:1/device', token_endpoint='http://127.0.0.1:1/token', # connection refused - scope='openid groups', groups_in_token=True, cache='memory', + scope='openid groups', groups_in_token=True, insecure=True, interactive=True, renderer=Renderer(), _clock=clock) expired = TokenSet( @@ -1849,18 +1848,6 @@ def test_settings_config_ignores_user_writable_preferences(self): self.assertEqual(settings_config({'acl.oidc.client.id': 'q'}), {'acl.oidc.client.id': 'q'}) - def test_make_cache_variants(self): - # The cache factory resolves the documented specs and rejects an - # unknown one with a typed error. See M4. - from questdb.auth._cache import make_cache, MemoryCache, NullCache - self.assertIsInstance(make_cache('memory'), MemoryCache) - self.assertIsInstance(make_cache(None), NullCache) - self.assertIsInstance(make_cache('none'), NullCache) - custom = MemoryCache() - self.assertIs(make_cache(custom), custom) # a TokenCache passes through - with self.assertRaises(OidcConfigError): - make_cache('disk') - class TestEndpointValidation(unittest.TestCase): def setUp(self): @@ -1953,7 +1940,7 @@ def _auth(self, **kw): client_id='questdb', device_authorization_endpoint='https://idp.example.com/device', token_endpoint='https://idp.example.com/token', - scope='openid groups', groups_in_token=True, cache='memory', + scope='openid groups', groups_in_token=True, renderer=Renderer()) opts.update(kw) return OidcDeviceAuth(**opts) @@ -2046,7 +2033,7 @@ def test_insecure_does_not_downgrade_idp(self): client_id='questdb', device_authorization_endpoint='http://idp.example.com/device', token_endpoint='http://idp.example.com/token', - scope='openid', groups_in_token=False, cache='memory', + scope='openid', groups_in_token=False, insecure=True, interactive=True, renderer=Renderer(), _clock=FakeClock()) with self.assertRaises(OidcConfigError): From 0a70fa55d0df31bc3a5b7157e156c3764d18cb94 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 19:29:25 +0100 Subject: [PATCH 050/104] fix(auth): close path-pin bypass via inner-segment ;param _endpoint_path_under_issuer only folded the FINAL path segment's ;params back before scanning for dot segments (urllib splits ;params off .path for the last segment only). An inner "..;" therefore stayed a literal segment that `'..' in ep_segs` never matched, so a tampered /settings could advertise https://idp/realms/prod/..;/realms/attacker/.../token under a pinned issuer https://idp/realms/prod and pass both the issuer path pin and the same-origin check. A proxy in the "..;/" traversal class (Tomcat/Undertow strip path parameters before normalizing) resolves it to a different realm, redirecting the device code and refresh token to an attacker. Strip the ;-matrix suffix and surrounding whitespace from EVERY decoded segment before the dot test, so "..;", "%2e%2e;" and "..%09" collapse to ".." and are rejected in any position, not just the last. Legitimate matrix params and percent-escaped segments are unaffected. Add regression assertions for the inner-segment vectors. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 43 +++++++++++++++++++++++++--------- test/test_auth.py | 9 +++++++ 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index c9324360..7a6454fe 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -182,6 +182,23 @@ def _decode_path_segments(path: str) -> list: return decoded.replace('\\', '/').split('/') +def _strip_matrix_params(segment: str) -> str: + """ + Reduce a decoded path segment to the form a server normalizes it to *before* + dot-segment removal: drop a ``;`` matrix-parameter suffix and trim + surrounding whitespace. + + So ``..;`` (and ``..\t`` from ``..%09``, or any ``;``-hidden / whitespace- + padded dot segment) reduces to ``..`` and is caught by the traversal check — + in *any* segment, not just the last. ``urllib`` only splits the **final** + segment's ``;params`` off ``.path``; an inner ``;`` stays inside the segment, + so this must run per-segment. Defeats the well-known ``..;/`` proxy-traversal + class (Tomcat/Undertow & co. strip path parameters before normalizing the + path), which an origin check and a last-segment-only fold can't catch. + """ + return segment.split(';', 1)[0].strip() + + def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: """ True if ``endpoint``'s path is the issuer's path or a sub-path of it. @@ -192,25 +209,29 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: on a path-based multi-tenant IdP (Keycloak issuers are ``https://host/realms/{realm}``), which an origin-only check can't catch. - Compared on fully *decoded* path segments, not the raw wire string. A ``.`` / - ``..`` segment is rejected outright: the server normalizes it, so - ``/realms/prod/../attacker/token`` passes a naive prefix test yet resolves to - a *different* realm. ``_decode_path_segments`` unmasks encoded dot segments, - and the last segment's ``;params`` (which urllib splits off ``.path``) is - folded back, so neither can hide a traversal. Legitimate paths have no dot - segments. + Compared on fully *decoded*, matrix-param-stripped path segments, not the raw + wire string. A ``.`` / ``..`` segment is rejected outright: the server + normalizes it, so ``/realms/prod/../attacker/token`` passes a naive prefix + test yet resolves to a *different* realm. ``_decode_path_segments`` unmasks + encoded dot segments (incl. an encoded slash / backslash), and + ``_strip_matrix_params`` reduces *every* segment the way a proxy does before + normalizing — dropping a ``;params`` suffix and surrounding whitespace — so a + ``..;`` / ``..%09`` hidden in any segment can't mask a traversal. (urllib + only splits the **final** segment's ``;params`` off ``.path``, hence both the + fold below and the per-segment pass.) Legitimate paths have no dot segments. """ base = (safe_urlparse(issuer)[0].path or '').rstrip('/') if not base: return True - base_segs = _decode_path_segments(base) + base_segs = [_strip_matrix_params(s) for s in _decode_path_segments(base)] eparts = safe_urlparse(endpoint)[0] - # Fold the last segment's ;params back into the path so a traversal hidden - # there (…/token;..%2f..%2fEVIL) can't slip past the scan. + # Fold the final segment's ;params back into the path so a traversal hidden + # there (…/token;..%2f..%2fEVIL) is decoded and scanned; an inner-segment + # ;params stays in .path and is handled per-segment by _strip_matrix_params. ep_path = eparts.path or '' if eparts.params: ep_path = f'{ep_path};{eparts.params}' - ep_segs = _decode_path_segments(ep_path) + ep_segs = [_strip_matrix_params(s) for s in _decode_path_segments(ep_path)] if '.' in ep_segs or '..' in ep_segs: return False return ep_segs[:len(base_segs)] == base_segs diff --git a/test/test_auth.py b/test/test_auth.py index ddd6cbaf..b0d7d817 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1928,6 +1928,15 @@ def test_endpoint_path_under_issuer(self): self.assertFalse(under(iss + '/%252e%252e/EVIL/token', iss)) # 2x-enc self.assertFalse(under(iss + '/..\\EVIL/token', iss)) # backslash self.assertFalse(under(iss + '/token;..%2f..%2fEVIL', iss)) # ;params + # A ;-matrix param or trailing whitespace in a NON-last segment: urllib + # only splits the FINAL segment's ;params off .path, so an inner '..;' / + # '..\t' stays a literal segment. A proxy in the '..;/' traversal class + # (Tomcat/Undertow) strips the param / trims the segment before + # normalizing and resolves these to a DIFFERENT realm, so they must be + # rejected too (regression test for the inner-segment path-pin bypass). + self.assertFalse(under(iss + '/..;/EVIL/protocol/token', iss)) # ..; + self.assertFalse(under(iss + '/%2e%2e;/EVIL/token', iss)) # enc ..; + self.assertFalse(under(iss + '/..%09/EVIL/token', iss)) # .. # A legitimate sub-path with a (non-traversal) percent-escape or matrix # param is still accepted — only dot traversal is rejected. self.assertTrue(under(iss + '/some%20path/token', iss)) From 2fed74aa6032f2797a020f6b299d78cbb6d70ef6 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 19:33:39 +0100 Subject: [PATCH 051/104] fix(auth): coerce non-string IdP token fields to None MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A non-string access_token / id_token in the IdP token response (a JSON number/bool/object from a buggy or hostile IdP) crashed token() with a raw AttributeError: _decode_jwt_claims evaluated token.count('.') before its try block, so the non-string escaped the except. This violated the module's contract that malformed server payloads map to an OidcError, and was reachable from both the poll-success and silent-refresh paths. Coerce access/id/refresh tokens to str-or-None in _tokenset_from_response so a non-string reads as absent — never stored, re-sent on a refresh, or emitted as "Bearer " — and guard _decode_jwt_claims with an isinstance check so the best-effort decode is total. A missing required kind then raises the clear terminal error instead of crashing, matching the existing expires_in coercion. Add regression tests for the poll path and the coercion helper. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 36 +++++++++++++++++++++++++++++------- test/test_auth.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 776b6067..5326f568 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -85,14 +85,28 @@ class _SystemClock: _SYSTEM_CLOCK = _SystemClock() +def _str_or_none(value: Any) -> Optional[str]: + """ + A credential/token field from an untrusted JSON response as a ``str``, else + ``None``. + + A non-string token (a JSON number/bool/object from a buggy or hostile IdP) + reads as absent so it is never stored, sent on a refresh, or emitted as + ``Bearer `` — and can't crash the best-effort JWT decode. A missing + required kind then raises the clear terminal error rather than caching an + unusable token. + """ + return value if isinstance(value, str) else None + + def _decode_jwt_claims(token: Optional[str]) -> Dict[str, Any]: """ Best-effort decode of a JWT payload **without signature verification**. Used only to show a friendly identity in the sign-in message; QuestDB does - the real validation. Returns ``{}`` for opaque/invalid tokens. + the real validation. Returns ``{}`` for opaque/invalid or non-string tokens. """ - if not token or token.count('.') < 2: + if not isinstance(token, str) or token.count('.') < 2: return {} try: payload = token.split('.')[1] @@ -512,13 +526,21 @@ def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: # Cap a long (or hostile) IdP-stated lifetime so a cached token is # re-validated at least hourly (matches the Java client). expires_in = min(expires_in, _MAX_EXPIRES_IN) - claims = (_decode_jwt_claims(body.get('id_token')) - or _decode_jwt_claims(body.get('access_token'))) + # Coerce the credential fields to str-or-None up front: a non-string + # token from a buggy/hostile IdP must read as absent rather than be + # stored, re-sent on a refresh, or emitted as ``Bearer `` — and + # the best-effort JWT decode below must not see a non-string. A missing + # required kind then raises the clear terminal error (see _select). + access_token = _str_or_none(body.get('access_token')) + id_token = _str_or_none(body.get('id_token')) + refresh_token = _str_or_none(body.get('refresh_token')) + claims = (_decode_jwt_claims(id_token) + or _decode_jwt_claims(access_token)) now = self._now() return TokenSet( - access_token=body.get('access_token'), - id_token=body.get('id_token'), - refresh_token=body.get('refresh_token'), + access_token=access_token, + id_token=id_token, + refresh_token=refresh_token, expires_at=now + expires_in, issued_at=now, token_type=body.get('token_type', 'Bearer'), diff --git a/test/test_auth.py b/test/test_auth.py index b0d7d817..8e95a416 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -713,6 +713,36 @@ def test_deeply_nested_jwt_payload_does_not_crash(self): auth = self.make_auth() self.assertEqual(auth.token(), nested) + def test_non_string_token_fields_do_not_crash(self): + # A buggy/hostile IdP returning a non-string access_token / id_token (a + # JSON number/bool/object) must not crash token() with a raw + # AttributeError from the best-effort JWT decode, nor be stored and + # emitted as ``Bearer ``. The non-string token reads as absent, + # so the grant fails with the clear terminal error and nothing is + # cached. See M2. + self.state.token_script = [(200, { + 'access_token': 12345, 'id_token': {'not': 'a-jwt'}, + 'token_type': 'Bearer', 'expires_in': 3600})] + auth = self.make_auth() # groups_in_token=False -> needs access_token + with self.assertRaises(OidcDeviceFlowError): + auth.token() + self.assertIsNone(auth._tokens) # nothing was cached + + def test_tokenset_from_response_coerces_non_string_credentials(self): + # _tokenset_from_response coerces every non-string credential field to + # None (treated as absent), and _decode_jwt_claims is total on any + # non-string input. See M2. + from questdb.auth._device import _decode_jwt_claims + auth = self.make_auth() + ts = auth._tokenset_from_response({ + 'access_token': 123, 'id_token': True, + 'refresh_token': ['x'], 'expires_in': 3600}) + self.assertIsNone(ts.access_token) + self.assertIsNone(ts.id_token) + self.assertIsNone(ts.refresh_token) + for bad in (123, 1.5, True, {'a': 1}, [1, 2], None, ''): + self.assertEqual(_decode_jwt_claims(bad), {}) + def test_idp_requests_use_configured_timeout(self): # The device-code / poll / refresh POSTs must use the configured # timeout, so a stalled IdP can't pin the acquisition lock for the From deb111d8f392f5b0eb494c1e00af2cd78bbeb4a7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 23 Jun 2026 23:58:02 +0100 Subject: [PATCH 052/104] fix(auth): map truncated error body to OidcNetworkError MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the 4xx/5xx branch of request(), the error-body read caught only (TimeoutError, OSError), but a server resetting the connection mid-chunked-body makes the read raise http.client.IncompleteRead — an HTTPException, not an OSError — so it escaped raw, breaking the typed OidcError contract on the path the device-flow poll loop drives hardest (many 4xx during a long sign-in). The success path already caught HTTPException; the error path's handler was simply narrower. Widen the except to (TimeoutError, OSError, http.client.HTTPException) to mirror the success path. _read_body's own OidcNetworkError for the size/deadline cap is not an HTTPException, so it still propagates cleanly, and the finally still closes the response. Add a regression test driving a truncated chunked 4xx body (a Content-Length truncation would not reproduce it: read(amt) returns the partial bytes rather than raising). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_http.py | 7 ++++++- test/test_auth.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 0e6b6c6f..dcddef2c 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -254,7 +254,12 @@ def request( try: body = _read_body(e, max_bytes=_MAX_RESPONSE_BYTES, deadline=_monotonic() + timeout) - except (TimeoutError, OSError) as read_err: + except (TimeoutError, OSError, http.client.HTTPException) as read_err: + # Mirror the success path's handler: a mid-body read failure such as + # http.client.IncompleteRead (an HTTPException, NOT an OSError) when + # the server resets the connection mid-error-body must map to a typed + # network error, not escape raw. (_read_body's own OidcNetworkError + # for the size/deadline cap is already typed and propagates here.) raise OidcNetworkError( f'Failed to read response from {url}: {read_err}') from read_err finally: diff --git a/test/test_auth.py b/test/test_auth.py index 8e95a416..15ee2d2a 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2065,6 +2065,42 @@ def test_post_form_attaches_status_to_non_json_error(self): post_form(raw + '/token', {'grant_type': 'x'}) self.assertEqual(cm.exception.status, 503) + def test_incomplete_error_body_maps_to_network_error(self): + # A 4xx/5xx with a truncated CHUNKED body (the server announces a chunk, + # sends fewer bytes, then closes) makes the error-body read raise + # http.client.IncompleteRead — an HTTPException, NOT an OSError. The poll + # loop drives many 4xx during sign-in, so this must map to a typed + # OidcNetworkError, not escape raw (mirrors the success path's handler). + # See M3. + import http.client + from questdb.auth import _http + self.assertFalse(issubclass(http.client.IncompleteRead, OSError)) + + class _ChunkedTrunc(http.server.BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' # chunked transfer needs HTTP/1.1 + + def log_message(self, *a): + pass + + def do_GET(self): + self.send_response(400) + self.send_header('Transfer-Encoding', 'chunked') + self.end_headers() + self.wfile.write(b'64\r\n') # announce a 0x64 = 100-byte chunk + self.wfile.write(b'short') # send only 5 of them, then close + self.wfile.flush() + self.close_connection = True + + srv = http.server.HTTPServer(('127.0.0.1', 0), _ChunkedTrunc) + threading.Thread(target=srv.serve_forever, daemon=True).start() + try: + with self.assertRaises(OidcNetworkError): + _http.request( + 'GET', f'http://127.0.0.1:{srv.server_port}/x', timeout=5) + finally: + srv.shutdown() + srv.server_close() + def test_insecure_does_not_downgrade_idp(self): # insecure=True must NOT permit plaintext to a non-loopback IdP: the # device code / refresh token must never traverse the network in clear. From 5adb3f49e9ba21c7a0eaec82ffcf2ee349c255fc Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 24 Jun 2026 00:02:57 +0100 Subject: [PATCH 053/104] test(auth): cover proactive refresh in the skew window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every refresh test seeded a fully-expired token (valid even at skew=0), so the proactive refresh-before-expiry that the whole cache+skew design exists for — refresh while still within the 30s clock-skew margin so a fresh connection never races a mid-flight 401 — was never exercised end-to-end. Seed a token with 15s left and a 65s lifetime (so the adaptive min(skew, lifetime/2) cap doesn't reduce the 30s skew), assert it is valid at skew=0 but invalid at the real skew, then assert token() silently refreshes once via the refresh_token with no device prompt and caches the result. Test-only; no behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/test_auth.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/test_auth.py b/test/test_auth.py index 15ee2d2a..85bacce4 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -890,6 +890,39 @@ def test_silent_refresh(self): self.assertEqual(self.state.refresh_requests, 1) self.assertEqual(self.state.device_requests, 0) # no re-prompt + def test_skew_window_triggers_proactive_refresh(self): + # The point of the cache+skew design: a token still within its lifetime + # but inside the 30s clock-skew margin is refreshed PROACTIVELY (silently + # via the refresh_token) so a fresh connection never races a mid-flight + # 401 — and with no device prompt. Every other refresh test seeds a + # fully-expired token (valid even at skew=0); this is the only end-to-end + # exercise of the skew window itself. See M4. + auth = self.make_auth() + now = self._clock.now() + seeded = TokenSet( + access_token='old-access', id_token='old-id', + refresh_token='REFRESH-1', + issued_at=now - 50, # 65s lifetime, so the adaptive cap + expires_at=now + 15) # (lifetime/2) doesn't bite -> full 30s skew + # Not actually expired (still valid at skew=0), but inside the real skew. + self.assertTrue(seeded.is_valid(now, skew=0)) + self.assertFalse(seeded.is_valid(now)) + self.assertLess(now, seeded.expires_at) + auth._cache.store(auth.cache_key, seeded) + + token = auth.token() + + self.assertEqual(token, ID_TOKEN) # groups mode -> id_token + self.assertEqual(self.state.refresh_requests, 1) # refreshed once... + self.assertEqual(self.state.device_requests, 0) # ...with NO prompt + self.assertEqual( + self.state.refresh_forms[0]['refresh_token'], 'REFRESH-1') + # The refreshed token is cached: a second call neither refreshes nor + # prompts (it is now far from expiry). + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual(self.state.device_requests, 0) + def test_refresh_failure_falls_back_to_device_flow(self): auth = self.make_auth() self._seed_expired(auth) From 0c0f499f1cb75c596ec232f6abe1a1c114d7bcab Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 24 Jun 2026 00:12:25 +0100 Subject: [PATCH 054/104] fix(auth): validate constructor arg types; redact sub from repr Two review-minor hardenings of the public surface: - OidcDeviceAuth.__init__ validated only truthiness, so a bad-typed arg surfaced later as a bare AttributeError/TypeError (scope=None reaching scope.split(); a non-string endpoint/issuer/audience reaching safe_urlparse() or the cache-key join), violating the typed- error contract. Type-check client_id/endpoints (non-empty str), scope (str), and audience/issuer (str-or-None) up front, raising OidcConfigError. from_questdb is unaffected (resolve_config already returns strings). - TokenSet repr already redacts the access/id/refresh tokens; also keep the JWT-derived subject (sub, PII) out of repr. scope stays visible as non-secret debugging metadata. Add a constructor bad-typed-args test and extend the repr-redaction test. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_cache.py | 3 ++- src/questdb/auth/_device.py | 27 +++++++++++++++++++++------ test/test_auth.py | 27 ++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index b7c5ea61..646a3f3f 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -52,7 +52,8 @@ class TokenSet: expires_at: float = 0.0 # epoch seconds; 0 == unknown token_type: str = 'Bearer' scope: Optional[str] = None - sub: Optional[str] = None + # subject id, derived from the (unverified) JWT — PII, so keep it out of repr + sub: Optional[str] = field(default=None, repr=False) issued_at: float = 0.0 # epoch seconds; 0 == unknown def is_valid(self, now: float, skew: float = DEFAULT_SKEW_SECONDS) -> bool: diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 5326f568..6cc3f4e3 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -207,12 +207,27 @@ def __init__( default_interval: int = 5, timeout: float = 30, _clock=None): # injectable time source for testing - if not client_id: - raise OidcConfigError('client_id is required') - if not device_authorization_endpoint: - raise OidcConfigError('device_authorization_endpoint is required') - if not token_endpoint: - raise OidcConfigError('token_endpoint is required') + # Validate types up front so a bad-typed arg raises the module's typed + # error, not a bare AttributeError/TypeError surfacing later from + # scope.split(), safe_urlparse(), or the cache-key join. + # from_questdb is unaffected: resolve_config already returns strings. + if not isinstance(client_id, str) or not client_id: + raise OidcConfigError( + 'client_id is required and must be a non-empty string') + if (not isinstance(device_authorization_endpoint, str) + or not device_authorization_endpoint): + raise OidcConfigError( + 'device_authorization_endpoint is required and must be a ' + 'non-empty string') + if not isinstance(token_endpoint, str) or not token_endpoint: + raise OidcConfigError( + 'token_endpoint is required and must be a non-empty string') + if not isinstance(scope, str): + raise OidcConfigError('scope must be a string') + if audience is not None and not isinstance(audience, str): + raise OidcConfigError('audience must be a string or None') + if issuer is not None and not isinstance(issuer, str): + raise OidcConfigError('issuer must be a string or None') # Sending the id_token requires the ``openid`` scope. if groups_in_token and 'openid' not in scope.split(): diff --git a/test/test_auth.py b/test/test_auth.py index 85bacce4..34cc5bf1 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -633,6 +633,28 @@ def test_openid_scope_auto_added_for_groups_in_token(self): insecure=True, renderer=Renderer()) self.assertIn('openid', auth.config.scope.split()) + def test_constructor_rejects_bad_typed_args(self): + # A bad-typed constructor arg must raise the typed OidcConfigError, not a + # bare AttributeError/TypeError surfacing later from scope.split(), + # safe_urlparse(), or the cache-key join. (from_questdb is + # unaffected — resolve_config guarantees strings.) See review Minors. + good = dict( + client_id='questdb', + device_authorization_endpoint='https://idp.example.com/device', + token_endpoint='https://idp.example.com/token', + renderer=Renderer()) + OidcDeviceAuth(**good) # sanity: the good kwargs construct fine + for bad in ( + {'client_id': None}, {'client_id': 123}, {'client_id': ''}, + {'device_authorization_endpoint': 123}, + {'device_authorization_endpoint': None}, + {'token_endpoint': 123}, {'token_endpoint': ''}, + {'scope': None}, {'scope': 123}, + {'scope': None, 'groups_in_token': True}, # the scope.split() case + {'audience': 123}, {'issuer': 123}): + with self.assertRaises(OidcConfigError): + OidcDeviceAuth(**{**good, **bad}) + def test_zero_expires_in_is_treated_as_unknown(self): # A non-positive expires_in must not mark the just-issued token expired. self.state.token_script = [(200, { @@ -858,11 +880,14 @@ def test_tokenset_is_frozen(self): def test_tokenset_repr_redacts_secrets(self): # The access/id/refresh tokens must never appear in repr() — a TokenSet # landing in a log line or traceback would otherwise leak credentials. + # The JWT subject (PII) is redacted too; non-secret metadata stays. r = repr(TokenSet(access_token='SECRET-A', id_token='SECRET-I', - refresh_token='SECRET-R', scope='openid')) + refresh_token='SECRET-R', sub='subject-PII-12345', + scope='openid')) self.assertNotIn('SECRET-A', r) self.assertNotIn('SECRET-I', r) self.assertNotIn('SECRET-R', r) + self.assertNotIn('subject-PII-12345', r) self.assertIn('openid', r) # non-secret metadata still shown From c7cd7f5226c44e38104e9de6636d951ac8a1d93e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 24 Jun 2026 01:13:45 +0100 Subject: [PATCH 055/104] fix(auth): validate default_interval and timeout args A non-numeric (or non-positive / NaN / bool) default_interval or timeout escaped the constructor's typed-error contract as a bare TypeError: when the IdP omits the optional RFC 8628 `interval`, the poll-interval clamp `max(_MIN_POLL_INTERVAL, default_interval)` raised TypeError, and a bad `timeout` surfaced the same way from the urllib socket call. Add a _validate_positive_number helper and apply it to both args in __init__ and at the top of from_questdb (which consumes `timeout` on its /settings and discovery requests before the constructor would validate it), so a bad value fails fast with OidcConfigError. Extend the bad-typed-arg test to cover both entry points. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 28 ++++++++++++++++++++++++++++ test/test_auth.py | 22 +++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 6cc3f4e3..89658cd3 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -146,6 +146,23 @@ def _http_status_is_transient(status: Optional[int]) -> bool: return status is not None and (status >= 500 or status == 429) +def _validate_positive_number(value: Any, name: str) -> None: + """ + Require a duration argument to be a positive, finite number of seconds. + + Mirrors the constructor's other up-front type checks: a non-numeric value + would otherwise surface later as a bare TypeError from the poll-interval + clamp (``max(_MIN_POLL_INTERVAL, default_interval)``) or a urllib socket + call (``timeout``), escaping the module's typed-error contract. ``bool`` is + an ``int`` subclass, so reject it explicitly; ``NaN`` fails ``> 0`` and is + rejected too (``<= 0`` would let it through). + """ + if (isinstance(value, bool) or not isinstance(value, (int, float)) + or not value > 0): + raise OidcConfigError( + f'{name} must be a positive number of seconds, got {value!r}') + + class OidcDeviceAuth: """ Acquire and refresh an OIDC token via the device authorization grant. @@ -228,6 +245,11 @@ def __init__( raise OidcConfigError('audience must be a string or None') if issuer is not None and not isinstance(issuer, str): raise OidcConfigError('issuer must be a string or None') + # default_interval feeds the poll-interval clamp and timeout every IdP + # socket call; a non-numeric value would otherwise escape as a bare + # TypeError rather than the typed error this block exists to raise. + _validate_positive_number(default_interval, 'default_interval') + _validate_positive_number(timeout, 'timeout') # Sending the id_token requires the ``openid`` scope. if groups_in_token and 'openid' not in scope.split(): @@ -313,6 +335,12 @@ def from_questdb( device-authorization endpoint when QuestDB doesn't advertise it. Any explicit keyword overrides discovery. """ + # Validate before resolve_config consumes `timeout` on its /settings and + # discovery HTTP calls (which run before cls() would validate it), so a + # bad timeout fails fast with the typed error rather than a bare + # TypeError from urllib. + _validate_positive_number(default_interval, 'default_interval') + _validate_positive_number(timeout, 'timeout') ctx = build_ssl_context(ca_bundle) cfg = resolve_config( questdb_url=url, diff --git a/test/test_auth.py b/test/test_auth.py index 34cc5bf1..b3f3d8e8 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -651,9 +651,29 @@ def test_constructor_rejects_bad_typed_args(self): {'token_endpoint': 123}, {'token_endpoint': ''}, {'scope': None}, {'scope': 123}, {'scope': None, 'groups_in_token': True}, # the scope.split() case - {'audience': 123}, {'issuer': 123}): + {'audience': 123}, {'issuer': 123}, + # default_interval / timeout feed the poll-interval clamp and + # urllib socket calls; a non-numeric/non-positive/NaN value must + # raise the typed error, not a bare TypeError later. bool is an + # int subclass, so it's rejected explicitly. + {'default_interval': 'soon'}, {'default_interval': 0}, + {'default_interval': -1}, {'default_interval': True}, + {'default_interval': float('nan')}, + {'timeout': 'slow'}, {'timeout': 0}, {'timeout': -5}, + {'timeout': True}, {'timeout': float('nan')}): with self.assertRaises(OidcConfigError): OidcDeviceAuth(**{**good, **bad}) + # A float interval/timeout is fine (clamped / passed to the socket). + OidcDeviceAuth(**{**good, 'default_interval': 7.5, 'timeout': 12.0}) + # from_questdb consumes `timeout` before the constructor runs, so it + # validates up front too — and before any network call, so a bad value + # fails fast without reaching the (unreachable) server. + with self.assertRaises(OidcConfigError): + OidcDeviceAuth.from_questdb( + 'https://db.example.com:9000', timeout='slow') + with self.assertRaises(OidcConfigError): + OidcDeviceAuth.from_questdb( + 'https://db.example.com:9000', default_interval=-1) def test_zero_expires_in_is_treated_as_unknown(self): # A non-positive expires_in must not mark the just-issued token expired. From 02a29f287d49f84fcaa6ed0dda718abb4a750ded Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 24 Jun 2026 11:17:13 +0100 Subject: [PATCH 056/104] fix(auth): propagate HTTP status on non-dict JSON body A token-endpoint response that is valid JSON but not an object (e.g. a JSON array from a non-conformant IdP) raised OidcError without a status, so the poll loop treated it as non-terminal and polled on to a misleading "code expired". Attach resp.status so a non-object 4xx fails fast. Also fold in the remaining review minors: - _cache: apply the lifetime/2 skew cap when issued_at is unknown (0), treating it as `now`, so a short-lived token that arrives without an issue time isn't reported expired the instant it is issued. - _device: coerce TokenSet.sub via _str_or_none like the other credential fields, so a non-string JWT sub reads as absent; drop the stale QuestDB.sender reference in the _ca_bundle comment. - docs/auth.rst: lengthen three section underlines that were one char short (Sphinx "title underline too short" warnings). - test_auth: cover NaN expires_in/interval (ValueError, vs inf's OverflowError), negative expires_in, a non-JSON 5xx/429 through the poll loop, the issued_at==0 skew cap, and the status now attached above. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/auth.rst | 6 +-- src/questdb/auth/_cache.py | 11 ++--- src/questdb/auth/_device.py | 9 ++-- src/questdb/auth/_http.py | 6 ++- test/test_auth.py | 89 +++++++++++++++++++++++++++++++++++-- 5 files changed, 106 insertions(+), 15 deletions(-) diff --git a/docs/auth.rst b/docs/auth.rst index 8d1080d4..bbd1f05c 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -199,7 +199,7 @@ For REST (``Authorization: Bearer``) and ingestion (the .. _oidc_idp_requirements: IdP requirements -=============== +================ The OIDC client referenced by ``acl.oidc.client.id`` must: @@ -212,7 +212,7 @@ The OIDC client referenced by ``acl.oidc.client.id`` must: or expose it via the **userinfo** endpoint (``false``), matching the server. Security notes -============= +============== * No IdP passwords are ever entered in the notebook; MFA/SSO happen at the IdP. * ``https`` is required. Plaintext ``http`` to a **loopback** address @@ -245,7 +245,7 @@ Security notes also pass ``ca_bundle=``. Dependencies -=========== +============ ``token()`` / ``headers()`` need nothing beyond the standard library. The following are imported lazily, only when used: diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index 646a3f3f..7b0479cb 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -61,11 +61,12 @@ def is_valid(self, now: float, skew: float = DEFAULT_SKEW_SECONDS) -> bool: if self.expires_at <= 0: return False # Cap skew at half the token lifetime, so a short-lived (< 2*skew) - # token isn't reported expired the instant it's issued. - if self.issued_at: - lifetime = self.expires_at - self.issued_at - if lifetime > 0: - skew = min(skew, lifetime / 2) + # token isn't reported expired the instant it's issued. issued_at == 0 + # means the issue time is unknown; treat it as `now` so the cap still + # applies to a short-lived token that arrives without one. + lifetime = self.expires_at - (self.issued_at or now) + if lifetime > 0: + skew = min(skew, lifetime / 2) return now < (self.expires_at - skew) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 89658cd3..930c8c7d 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -277,8 +277,9 @@ def __init__( # device code / refresh token are never sent in cleartext even when set. self.insecure = insecure self.open_browser = open_browser - # Kept so adapters with their own transport (QuestDB.sender's ILP Sender) - # can forward the same private CA as _ctx rather than the default roots. + # Kept so a caller wiring the token into their own transport (e.g. the + # ingestion Sender) can forward the same private CA as _ctx rather than + # the default roots. self._ca_bundle = ca_bundle self._interactive = interactive self._default_interval = default_interval @@ -588,7 +589,9 @@ def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: issued_at=now, token_type=body.get('token_type', 'Bearer'), scope=body.get('scope', self.config.scope), - sub=claims.get('sub')) + # Coerce like the credential fields: a non-string sub from a hostile + # JWT reads as absent rather than landing in an Optional[str] field. + sub=_str_or_none(claims.get('sub'))) def _idp_post(self, url: str, form: Dict[str, Any]): # IdP POSTs carry the device code / refresh token, so always https diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index dcddef2c..fe9c2094 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -329,5 +329,9 @@ def post_form( f'HTTP {resp.status} from {url}: {resp.text()[:200]}', status=resp.status) if not isinstance(parsed, dict): - raise OidcError(f'Unexpected JSON shape from {url}: {parsed!r}') + # Attach the status (mirroring the non-JSON branches) so a non-object + # body on a terminal 4xx — e.g. a JSON array from a non-conformant IdP + # — fails the poll loop fast instead of polling on to "code expired". + raise OidcError( + f'Unexpected JSON shape from {url}: {parsed!r}', status=resp.status) return resp.status, parsed diff --git a/test/test_auth.py b/test/test_auth.py index b3f3d8e8..71d9a54f 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -435,6 +435,34 @@ def test_transient_5xx_and_429_during_poll_keep_polling(self): # 503 polled at the base interval; 429 bumps the interval by 5. self.assertEqual(self._clock.sleeps, [5, 5, 10]) + def test_non_json_5xx_and_429_during_poll_keep_polling(self): + # A non-JSON 5xx/429 from a proxy in front of the token endpoint makes + # post_form RAISE a bare OidcError(status=...) — it can't return a JSON + # body like the case above. The poll loop must treat it as transient and + # keep polling, exercising the `except OidcError` transient arm and its + # 429 back-off (distinct from the status-based arm). Review m6. + self.state.token_script = [(200, None)] # success once actually polled + auth = self.make_auth() + real_idp_post = auth._idp_post + polls = {'n': 0} + + def flaky(url, form): + if url == auth.config.token_endpoint: + polls['n'] += 1 + if polls['n'] == 1: + raise OidcError('proxy 503', status=503) + if polls['n'] == 2: + raise OidcError('rate limited', status=429) + return real_idp_post(url, form) + + auth._idp_post = flaky + self.assertEqual(auth.token(), ID_TOKEN) + # 503 retried at the base interval; 429 retried with a +5 bump; the 3rd + # poll reached the IdP and succeeded. + self.assertEqual(polls['n'], 3) + self.assertEqual(len(self.state.token_requests), 1) + self.assertEqual(self._clock.sleeps, [5, 5, 10]) + def test_non_json_4xx_during_poll_is_terminal(self): # A non-JSON 4xx during polling (an HTML/plain error page from a WAF or # reverse proxy in front of the IdP, or a non-conformant IdP) is a @@ -684,6 +712,17 @@ def test_zero_expires_in_is_treated_as_unknown(self): auth.token() self.assertTrue(auth._tokens.is_valid(self._clock.now())) + def test_negative_expires_in_treated_as_unknown(self): + # A negative expires_in (like zero) must be treated as unknown, not mark + # the just-issued token expired — guards the `<= 0` check against an + # `== 0` regression. Review m6. + self.state.token_script = [(200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'token_type': 'Bearer', 'expires_in': -100})] + auth = self.make_auth() + auth.token() + self.assertTrue(auth._tokens.is_valid(self._clock.now())) + def test_short_lived_token_valid_at_issue(self): # A small positive expires_in (< 2*skew) must not read as expired the # instant it is issued (adaptive skew = min(skew, lifetime/2)). @@ -697,6 +736,19 @@ def test_short_lived_token_valid_at_issue(self): self.assertTrue(t.is_valid(t.issued_at)) # usable right after issue self.assertFalse(t.is_valid(t.expires_at)) # but still does expire + def test_is_valid_caps_skew_when_issued_at_unknown(self): + # issued_at == 0 means "unknown issue time": a short-lived token that + # arrives without one must still be usable at issue (skew capped to half + # the remaining lifetime), not read as expired immediately. (Before the + # cap applied for issued_at == 0, is_valid(now) returned False here.) Not + # reachable via _tokenset_from_response, which always sets issued_at — a + # guard for a future caller or a token restored without one. Review m3. + now = 1_000_000.0 + short = TokenSet(access_token='a', expires_at=now + 20, issued_at=0.0) + self.assertTrue(short.is_valid(now)) # usable right at issue + self.assertFalse(short.is_valid(now + 20)) # but still expires + self.assertFalse(short.is_valid(now + 100)) # and stays expired after + def test_overflow_expires_in_treated_as_unknown(self): # A non-finite expires_in (JSON Infinity, which json.loads accepts and # int(inf) turns into an OverflowError — not a ValueError) must not @@ -708,6 +760,18 @@ def test_overflow_expires_in_treated_as_unknown(self): self.assertEqual(auth.token(), ID_TOKEN) self.assertTrue(auth._tokens.is_valid(self._clock.now())) + def test_nan_expires_in_treated_as_unknown(self): + # A NaN token expires_in: int(nan) raises ValueError — a DIFFERENT + # exception type from the OverflowError that inf raises above — so it + # exercises a separate except arm. Must be treated as unknown so the + # token stays usable, not crash. Review m6. + self.state.token_script = [(200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'token_type': 'Bearer', 'expires_in': float('nan')})] + auth = self.make_auth() + self.assertEqual(auth.token(), ID_TOKEN) + self.assertTrue(auth._tokens.is_valid(self._clock.now())) + def test_missing_expires_in_defaults_to_short_ttl(self): # When the IdP omits expires_in, fall back to a short, conservative TTL # (300s) so the token is refreshed promptly, matching the Java client. @@ -741,6 +805,18 @@ def test_overflow_device_timing_fields_do_not_crash(self): auth = self.make_auth() self.assertEqual(auth.token(), ID_TOKEN) + def test_nan_device_timing_fields_do_not_crash(self): + # NaN interval / expires_in in the device-auth response: int(nan) raises + # ValueError, not the OverflowError that inf raises above — a different + # except arm. Must be treated as unknown, not crash. Review m6. + self.state.device_response = { + 'device_code': 'DEV-CODE', 'user_code': 'X', + 'verification_uri': 'https://idp/device', + 'expires_in': float('nan'), 'interval': float('nan')} + self.state.token_script = [(200, None)] # success on the first poll + auth = self.make_auth() + self.assertEqual(auth.token(), ID_TOKEN) + def test_deeply_nested_jwt_payload_does_not_crash(self): # A hostile/buggy IdP returning an id_token whose payload base64-decodes # to deeply-nested JSON must not crash token() with a raw RecursionError @@ -2327,12 +2403,19 @@ def test_post_form_non_json_2xx_raises_oidc_error(self): _http.post_form(b + '/token', {'a': 'b'}, timeout=5) def test_post_form_non_dict_json_raises_oidc_error(self): - # A 2xx JSON array (valid JSON but not an object) from the token - # endpoint must surface as OidcError. See M4. + # A JSON array (valid JSON but not an object) from the token endpoint + # must surface as OidcError WITH the HTTP status attached, so the poll + # loop can tell a terminal 4xx from a transient/2xx instead of polling on + # to a misleading "code expired". See M4 / review m1. from questdb.auth import _http with _raw_response_server(200, 'application/json', b'[1, 2, 3]') as b: - with self.assertRaises(OidcError): + with self.assertRaises(OidcError) as cm: + _http.post_form(b + '/token', {'a': 'b'}, timeout=5) + self.assertEqual(cm.exception.status, 200) + with _raw_response_server(400, 'application/json', b'["nope"]') as b: + with self.assertRaises(OidcError) as cm: _http.post_form(b + '/token', {'a': 'b'}, timeout=5) + self.assertEqual(cm.exception.status, 400) def test_get_json_non_2xx_raises_oidc_error(self): # A non-2xx /settings or discovery response must surface as OidcError. From 117f06a1985f6bf58448092ec82f75c14a63f758 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 24 Jun 2026 13:33:54 +0100 Subject: [PATCH 057/104] fix(auth): honor read deadline on slow-dribble responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _read_body checked its wall-clock deadline only between resp.read() calls, but http.client's read(n) blocks until it has buffered the full n bytes (or EOF). A server dribbling one byte per socket-timeout window kept a single read(_READ_CHUNK) blocked indefinitely: the per-socket timeout kept resetting and the deadline was never reached, defeating the timeout the acquisition lock relies on (a hung IdP/proxy leg wedges every thread sharing the auth object). Read via read1() instead, which returns after a single underlying socket read, so the deadline check runs between reads. Falls back to read() for streams without read1 (e.g. test stubs); real HTTPResponse/HTTPError both provide it. Add a regression test driving a real dribbling socket — the existing _ChunkStream mock can't catch this, since it defines read() itself. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_http.py | 11 +++++- test/test_auth.py | 71 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index fe9c2094..8d95dea7 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -190,6 +190,15 @@ def _read_body(resp: Any, *, max_bytes: int, deadline: float) -> bytes: past the caller's timeout (urllib's timeout is per-socket-read, not a whole-read bound) nor exhaust memory with an unbounded body. """ + # Read via read1(): it returns after a SINGLE underlying socket read, so the + # deadline check below actually runs between reads. resp.read(n) on an + # http.client response instead blocks until it has buffered the full n bytes + # (or hits EOF), so a server dribbling one byte per socket-timeout window + # would keep one read(_READ_CHUNK) blocked indefinitely — the per-socket + # timeout keeps resetting and the deadline is never reached. read1 is + # provided by http.client.HTTPResponse and (by delegation) urllib's + # HTTPError; fall back to read() for any stream that lacks it. + read = getattr(resp, 'read1', None) or resp.read chunks = [] total = 0 while True: @@ -197,7 +206,7 @@ def _read_body(resp: Any, *, max_bytes: int, deadline: float) -> bytes: raise OidcNetworkError( 'Timed out reading the response body; the server is too slow ' 'or is dribbling data.') - chunk = resp.read(_READ_CHUNK) + chunk = read(_READ_CHUNK) if not chunk: return b''.join(chunks) total += len(chunk) diff --git a/test/test_auth.py b/test/test_auth.py index 71d9a54f..f8043a25 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2358,6 +2358,77 @@ def test_read_body_aborts_on_slow_dribble(self): with self.assertRaises(OidcNetworkError): _http._read_body(body, max_bytes=10 ** 9, deadline=1.0) + def test_read_body_aborts_real_socket_dribble(self): + # Regression for M1, against the REAL socket stack (the _ChunkStream + # unit test above cannot catch this — the mock defines read() itself). + # A real http.client response's read(n) blocks until n bytes are + # buffered, so a server dribbling one byte per socket-timeout window + # would keep a single read(_READ_CHUNK) blocked forever and the + # wall-clock deadline (checked only between reads) would never fire. + # _read_body must read via read1() so each read returns after one + # socket read and the deadline is honored; this would hang the calling + # thread (which holds the acquisition lock) before the fix. + import socket + from questdb.auth import _http + + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(('127.0.0.1', 0)) + srv.listen(1) + port = srv.getsockname()[1] + stop = threading.Event() + + def serve(): + try: + conn, _ = srv.accept() + except OSError: + return + try: + conn.recv(65536) # consume the request line/headers + # Announce a large body, then dribble it one byte at a time, + # each well inside the per-socket timeout window so urllib's + # per-read timeout never fires. + conn.sendall( + b'HTTP/1.1 200 OK\r\n' + b'Content-Type: application/json\r\n' + b'Content-Length: 1000000\r\n\r\n') + while not stop.is_set(): + try: + conn.sendall(b'a') + except OSError: + break + stop.wait(0.1) + finally: + conn.close() + + server_thread = threading.Thread(target=serve, daemon=True) + server_thread.start() + + result = {} + + def call(): + try: + _http.request( + 'GET', f'http://127.0.0.1:{port}/x', timeout=1.0) + result['returned'] = True + except Exception as e: # noqa: BLE001 - record for the assert below + result['error'] = e + + t = threading.Thread(target=call, daemon=True) + t.start() + t.join(8.0) + hung = t.is_alive() # capture before cleanup unblocks it + stop.set() + srv.close() + server_thread.join(2.0) + + self.assertFalse( + hung, + 'request() hung on a dribbling server: the whole-read wall-clock ' + 'deadline never fired (M1 regression — _read_body must read via ' + 'read1()).') + self.assertIsInstance(result.get('error'), OidcNetworkError) + def test_bad_ca_bundle_raises_config_error(self): # A missing or invalid CA bundle path (explicit or via env) must surface # as OidcConfigError, not a raw FileNotFoundError / ssl.SSLError. See M1. From 9e622394ac46fea5c1edf8205948c2ae8393595f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 24 Jun 2026 13:48:00 +0100 Subject: [PATCH 058/104] fix(auth): strip terminal control chars by Unicode category _strip_control used a hand-enumerated regex of control/format/bidi/zero-width code points, which silently missed any not listed: the deprecated U+206x format chars, the Tags block (U+E0001/E0020-E007F), unassigned code points (U+2065) and Arabic format marks could still reach a TTY / notebook DOM in an untrusted device-response field. Strip by Unicode general category instead (Cc/Cf/Cn/Co/Cs/Zl/Zp) so a newly-assigned format codepoint is covered automatically. Keep an explicit set for the invisible Hangul fillers (U+115F/1160/3164/FFA0), which Unicode classifies as letters (Lo) and the category rule won't catch; spaces (Zs) and combining marks (Mn, e.g. accents) are deliberately kept so a legitimate URL/identity still renders. Drop the now-unused re import. Extend the strip test with the newly-covered code points. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_render.py | 29 ++++++++++++++++++++--------- test/test_auth.py | 7 ++++++- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 85dc29c0..145e5483 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -33,8 +33,8 @@ from __future__ import annotations import html -import re import sys +import unicodedata import urllib.parse from typing import Any, Dict, Optional, TextIO @@ -120,13 +120,21 @@ def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: f'rel="noopener noreferrer">{label}') -# Strips C0/C1/ESC, bidi overrides, zero-width and line/paragraph separators -# — all can spoof the prompt (e.g. U+202E reverses a URL's host). Applied to -# untrusted device-response fields on both paths; html.escape would not catch -# these. -_CONTROL_CHARS = re.compile( - r'[\x00-\x1f\x7f-\x9f\u00ad\u061c\u115f\u180e\u200b-\u200f' - r'\u2028-\u202e\u2060-\u2064\u2066-\u2069\ufeff\ufff9-\ufffb]') +# Untrusted device-response fields are echoed to a TTY / notebook DOM, where a +# control, bidi-override (e.g. U+202E reverses a URL's host) or zero-width char +# could spoof the prompt or hide the real sign-in URL (html.escape guards +# markup, not these). Strip by Unicode general category so a newly-assigned +# format codepoint is covered automatically, rather than an enumerated regex +# that silently misses additions: control (Cc), format (Cf: bidi / zero-width / +# soft hyphen / tag chars / the deprecated U+206x), unassigned (Cn), +# private-use (Co), surrogates (Cs) and line/paragraph separators (Zl/Zp). +# Spaces (Zs) and combining marks (Mn, e.g. accents) are kept so a legitimate +# URL/identity still renders. +_STRIP_CATEGORIES = frozenset({'Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Zl', 'Zp'}) +# Invisible characters Unicode classifies as letters (category Lo), so the rule +# above won't catch them, but they render as nothing and are used to hide/spoof +# text: the Hangul fillers. Stripped explicitly. +_STRIP_EXTRA = frozenset('\u115f\u1160\u3164\uffa0') def _strip_control(text: Optional[str]) -> str: @@ -140,7 +148,10 @@ def _strip_control(text: Optional[str]) -> str: """ if not text: return '' - return _CONTROL_CHARS.sub('', text) + return ''.join( + ch for ch in text + if ch not in _STRIP_EXTRA + and unicodedata.category(ch) not in _STRIP_CATEGORIES) def format_prompt(resp: Dict[str, Any]) -> str: diff --git a/test/test_auth.py b/test/test_auth.py index f8043a25..3b16008d 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2667,7 +2667,12 @@ def test_strip_control_removes_bidi_and_zero_width(self): for cp in (0x202e, 0x202d, 0x2066, 0x2069, 0x200b, 0x200f, 0x2028, 0x2029, 0xfeff, # also the format/bidi code points added for M3: - 0x00ad, 0x061c, 0x115f, 0x180e, 0x2060, 0x2064, 0xfff9): + 0x00ad, 0x061c, 0x115f, 0x180e, 0x2060, 0x2064, 0xfff9, + # the category-based strip also covers the deprecated U+206x + # format chars, the Tags block, unassigned code points, + # Arabic format marks and the other invisible Hangul fillers: + 0x206a, 0x206f, 0x2065, 0xe0001, 0xe007f, 0x0600, + 0x1160, 0x3164, 0xffa0): self.assertEqual(_strip_control('a' + chr(cp) + 'b'), 'ab', f'U+{cp:04X} not stripped') # Legitimate text (incl. accents / CJK / printable ASCII) is preserved. From bf7ea912a4806e4c150587d9389fea7eab93c525 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 24 Jun 2026 14:03:49 +0100 Subject: [PATCH 059/104] fix(auth): harden parsing of untrusted IdP response fields Four small robustness fixes for non-conformant or hostile IdP responses: - expires_in / interval: a JSON bool was read as int(True) == 1, minting a 1-second token (or device-code lifetime) that churns refreshes. Add an _int_or_default helper that maps bool to the default and keeps the existing None / non-numeric / NaN / Infinity fallbacks, applied at all three sites (token TTL and the poll-loop interval/expires_in). - token_type / scope: coerce like the other credential fields, so a non-string value falls back to 'Bearer' / the configured scope instead of landing raw in the frozen TokenSet. - device_code / user_code: the 200-response guard used bare truthiness, so a non-string code (JSON number/list) passed and was later stringified into the poll request. Coerce via _str_or_none so it reads as absent and raises the clear "missing device_code/user_code" error instead of polling a bogus code. - endpoint path-under-issuer: reject any endpoint segment that still contains '%' after the bounded percent-decode loop (fail closed), so a dot segment wrapped in more encoding layers than the loop peels can't slip the traversal check. Legitimate plain-ASCII endpoint paths are unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 54 ++++++++++++++++++++----------- src/questdb/auth/_discovery.py | 8 ++++- test/test_auth.py | 59 ++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 19 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 930c8c7d..990e55b1 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -99,6 +99,26 @@ def _str_or_none(value: Any) -> Optional[str]: return value if isinstance(value, str) else None +def _int_or_default(value: Any, default: int) -> int: + """ + ``int(value)`` for an untrusted numeric field (``expires_in`` / ``interval``), + else ``default``. + + ``bool`` is an ``int`` subclass, but a JSON ``true``/``false`` is never a + meaningful duration: ``int(True) == 1`` would mint a 1-second lifetime and + churn refreshes, so map a bool to the default. A missing key (``None``), a + non-numeric string, or a JSON ``Infinity``/``NaN`` (``json.loads`` accepts + both) raises ``TypeError``/``ValueError``/``OverflowError`` from ``int()`` + and falls back too, keeping the typed contract. + """ + if isinstance(value, bool): + return default + try: + return int(value) + except (TypeError, ValueError, OverflowError): + return default + + def _decode_jwt_claims(token: Optional[str]) -> Dict[str, Any]: """ Best-effort decode of a JWT payload **without signature verification**. @@ -557,12 +577,8 @@ def _cache_generation(self) -> int: return self._cache.generation(self.cache_key) def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: - try: - expires_in = int(body.get('expires_in', _DEFAULT_EXPIRES_IN)) - except (TypeError, ValueError, OverflowError): - # OverflowError: a JSON Infinity (json.loads accepts it) → int(inf) - # isn't a ValueError, so list it to keep the typed contract. - expires_in = _DEFAULT_EXPIRES_IN + expires_in = _int_or_default( + body.get('expires_in'), _DEFAULT_EXPIRES_IN) if expires_in <= 0: # A non-positive lifetime marks a just-issued token as expired, # causing refresh/re-prompt churn. Treat it as unknown. @@ -587,8 +603,11 @@ def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: refresh_token=refresh_token, expires_at=now + expires_in, issued_at=now, - token_type=body.get('token_type', 'Bearer'), - scope=body.get('scope', self.config.scope), + # Coerce like the credential fields: a non-string token_type/scope + # from a buggy/hostile IdP falls back to the default rather than + # landing in the dataclass as a raw object. + token_type=_str_or_none(body.get('token_type')) or 'Bearer', + scope=_str_or_none(body.get('scope')) or self.config.scope, # Coerce like the credential fields: a non-string sub from a hostile # JWT reads as absent rather than landing in an Optional[str] field. sub=_str_or_none(claims.get('sub'))) @@ -681,11 +700,14 @@ def _request_device_code(self) -> Dict[str, Any]: form['audience'] = self.config.audience status, body = self._idp_post( self.config.device_authorization_endpoint, form) - if status == 200 and body.get('device_code') and body.get('user_code'): + if (status == 200 and _str_or_none(body.get('device_code')) + and _str_or_none(body.get('user_code'))): return body error = body.get('error') if status == 200: - # 200 but the guard above failed: device_code/user_code missing. + # 200 but the guard above failed: device_code/user_code missing or + # non-string (coerced via _str_or_none, so a JSON number/list reads + # as absent instead of being stringified into the poll request). # A non-conformant body, not an HTTP failure — say so plainly rather # than a contradictory "failed (HTTP 200)". raise OidcDeviceFlowError( @@ -713,18 +735,14 @@ def _request_device_code(self) -> Dict[str, Any]: def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: device_code = resp['device_code'] - try: - interval = int(resp.get('interval', self._default_interval)) - except (TypeError, ValueError, OverflowError): - interval = self._default_interval + interval = _int_or_default( + resp.get('interval'), self._default_interval) # Floor at the RFC 8628 default (5s) so we never poll faster than the # spec baseline; cap so a hostile value can't pin the polling thread # (which holds the lock) in one enormous sleep. interval = min(_MAX_POLL_INTERVAL, max(_MIN_POLL_INTERVAL, interval)) - try: - expires_in = int(resp.get('expires_in', _DEFAULT_DEVICE_CODE_LIFETIME)) - except (TypeError, ValueError, OverflowError): - expires_in = _DEFAULT_DEVICE_CODE_LIFETIME + expires_in = _int_or_default( + resp.get('expires_in'), _DEFAULT_DEVICE_CODE_LIFETIME) # A non-positive lifetime would time out before the first poll (the code # is already shown); treat it as unknown. Cap the upper end so a hostile # value can't keep the loop — and the lock — alive indefinitely. diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 7a6454fe..6bd08ca5 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -232,7 +232,13 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: if eparts.params: ep_path = f'{ep_path};{eparts.params}' ep_segs = [_strip_matrix_params(s) for s in _decode_path_segments(ep_path)] - if '.' in ep_segs or '..' in ep_segs: + # A residual '%' means the path did not fully decode within the bounded loop + # (an over-deeply multiply-encoded escape) or is a malformed escape; a server + # may yet decode it further to a dot-segment, so fail closed rather than + # prefix-match a value that could still resolve to a different path. + # Legitimate credential-endpoint paths are plain ASCII with no encoding. + if ('.' in ep_segs or '..' in ep_segs + or any('%' in s for s in ep_segs)): return False return ep_segs[:len(base_segs)] == base_segs diff --git a/test/test_auth.py b/test/test_auth.py index 3b16008d..f199278f 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -499,6 +499,23 @@ def test_device_200_without_codes_is_rejected_clearly(self): self.assertIn('device_code', msg) self.assertEqual(self.state.token_requests, []) # never polled + def test_device_200_with_nonstring_codes_is_rejected_clearly(self): + # A 200 whose device_code/user_code is a non-string (a JSON number/list + # from a buggy/hostile IdP) must be treated as missing — coerced via + # _str_or_none so it can't be stringified into the poll request — and + # raise the same clear error rather than polling with a bogus code. + self.state.device_status = 200 + self.state.device_response = { + 'device_code': 12345, 'user_code': ['X'], + 'verification_uri': 'https://idp/device'} + auth = self.make_auth() + with self.assertRaises(OidcDeviceFlowError) as cm: + auth.token() + msg = str(cm.exception) + self.assertNotIn('HTTP 200', msg) + self.assertIn('device_code', msg) + self.assertEqual(self.state.token_requests, []) # never polled + def test_timeout_when_never_authorized(self): self.state.device_response = { 'device_code': 'DEV-CODE', 'user_code': 'X', @@ -723,6 +740,29 @@ def test_negative_expires_in_treated_as_unknown(self): auth.token() self.assertTrue(auth._tokens.is_valid(self._clock.now())) + def test_bool_expires_in_treated_as_unknown(self): + # A JSON bool expires_in must NOT be read as int(True) == 1 (a 1-second + # token that churns refreshes / re-prompts); treat it as unknown so the + # just-issued token is valid. + self.state.token_script = [(200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'token_type': 'Bearer', 'expires_in': True})] + auth = self.make_auth() + auth.token() + self.assertTrue(auth._tokens.is_valid(self._clock.now())) + + def test_int_or_default_rejects_bool_and_nonnumeric(self): + # _int_or_default underpins expires_in / interval parsing on both the + # token and device-authorization responses: a JSON bool maps to the + # default (not int(True) == 1), and non-numeric / NaN / Infinity / + # missing fall back too, while a real number (or numeric string) passes. + from questdb.auth._device import _int_or_default + for bad in (True, False, 'abc', None, float('nan'), float('inf'), [1]): + self.assertEqual(_int_or_default(bad, 300), 300, repr(bad)) + self.assertEqual(_int_or_default(600, 300), 600) + self.assertEqual(_int_or_default('600', 300), 600) + self.assertEqual(_int_or_default(1.9, 300), 1) # truncates, like int() + def test_short_lived_token_valid_at_issue(self): # A small positive expires_in (< 2*skew) must not read as expired the # instant it is issued (adaptive skew = min(skew, lifetime/2)). @@ -858,6 +898,19 @@ def test_tokenset_from_response_coerces_non_string_credentials(self): self.assertIsNone(ts.access_token) self.assertIsNone(ts.id_token) self.assertIsNone(ts.refresh_token) + # token_type / scope are coerced too: a non-string falls back to the + # default ('Bearer' / the configured scope) instead of landing raw in + # the frozen dataclass; a valid string passes through unchanged. + ts_bad = auth._tokenset_from_response({ + 'access_token': ACCESS_TOKEN, 'token_type': ['x'], + 'scope': 123, 'expires_in': 3600}) + self.assertEqual(ts_bad.token_type, 'Bearer') + self.assertEqual(ts_bad.scope, auth.config.scope) + ts_ok = auth._tokenset_from_response({ + 'access_token': ACCESS_TOKEN, 'token_type': 'DPoP', + 'scope': 'openid email', 'expires_in': 3600}) + self.assertEqual(ts_ok.token_type, 'DPoP') + self.assertEqual(ts_ok.scope, 'openid email') for bad in (123, 1.5, True, {'a': 1}, [1, 2], None, ''): self.assertEqual(_decode_jwt_claims(bad), {}) @@ -2121,6 +2174,12 @@ def test_endpoint_path_under_issuer(self): self.assertFalse(under(iss + '/..;/EVIL/protocol/token', iss)) # ..; self.assertFalse(under(iss + '/%2e%2e;/EVIL/token', iss)) # enc ..; self.assertFalse(under(iss + '/..%09/EVIL/token', iss)) # .. + # A dot segment wrapped in MORE encoding layers than the bounded decode + # loop peels leaves a residual '%'; a server that decodes it further + # would resolve to a DIFFERENT realm, so a segment that did not fully + # decode is rejected (fail closed). + enc_dot = '%' + '25' * 11 + '2e' # a single '.' wrapped in 12 layers + self.assertFalse(under(f'{iss}/{enc_dot}{enc_dot}/EVIL/token', iss)) # A legitimate sub-path with a (non-traversal) percent-escape or matrix # param is still accepted — only dot traversal is rejected. self.assertTrue(under(iss + '/some%20path/token', iss)) From 0b62d09b90f4b5b5d4c9e27fdf5432e8cd2430e7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 24 Jun 2026 14:16:49 +0100 Subject: [PATCH 060/104] fix(auth): normalize empty audience; drop dead field; review cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - audience: normalize an empty string to None in OidcDeviceAuth.__init__ so it is omitted consistently. It was previously sent as `audience=` on the refresh request but never on device-authorization (post_form drops only None). - OidcConfig.authorization_endpoint: remove the field, its constant, the resolve_config parameter, both resolution sites and the construction arg. It was resolved from /settings and discovery but never consumed (the device flow uses only the device and token endpoints). - CHANGELOG: reword the Python-floor note — pyproject already declared >=3.10 and 3.9 was dropped in 4.1.0; this only corrects the stale setup.py (3.8) and docs, so 3.8/3.9 installs are now correctly rejected. - tests: delete the dead /exec mock scaffolding (no REST adapter exists), and add coverage for the empty-audience case, _identity_from_claims precedence, webbrowser.open errors being swallowed, the psycopg2 drivername branch, Jupyter QR suppression for dangerous URLs, _fmt_mmss, interactive detection, and extra _as_bool / _resolve_endpoint branches. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.rst | 4 +- src/questdb/auth/_device.py | 6 ++ src/questdb/auth/_discovery.py | 12 +-- test/test_auth.py | 155 ++++++++++++++++++++++++++------- 4 files changed, 134 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d56e18b3..52e42f8a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -52,7 +52,9 @@ See the :ref:`OIDC authentication guide ` for details. Python Version Support ~~~~~~~~~~~~~~~~~~~~~~~~ -* Raised the minimum supported Python version to 3.10. +* Corrected the minimum supported Python declared in ``setup.py`` to 3.10, + matching the floor already adopted in 4.1.0 (which dropped Python 3.9). + Installs on Python 3.8 / 3.9 are now correctly rejected. 4.1.0 (2025-11-28) ------------------ diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 990e55b1..e85eb647 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -263,6 +263,12 @@ def __init__( raise OidcConfigError('scope must be a string') if audience is not None and not isinstance(audience, str): raise OidcConfigError('audience must be a string or None') + # Normalize an empty audience to None so it is omitted consistently: + # _request_device_code skips a falsy audience, but _refresh puts it in + # the form unconditionally and post_form drops only None — so an empty + # string would be sent as `audience=` on refresh yet not on device-auth. + if not audience: + audience = None if issuer is not None and not isinstance(issuer, str): raise OidcConfigError('issuer must be a string or None') # default_interval feeds the poll-interval clamp and timeout every IdP diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 6bd08ca5..1bdefcc0 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -48,7 +48,6 @@ _K_CLIENT_ID = 'acl.oidc.client.id' _K_SCOPE = 'acl.oidc.scope' _K_TOKEN_ENDPOINT = 'acl.oidc.token.endpoint' -_K_AUTHORIZATION_ENDPOINT = 'acl.oidc.authorization.endpoint' _K_DEVICE_ENDPOINT = 'acl.oidc.device.authorization.endpoint' # design §7 (new) _K_GROUPS_IN_TOKEN = 'acl.oidc.groups.encoded.in.token' _K_AUDIENCE = 'acl.oidc.audience' @@ -65,7 +64,6 @@ class OidcConfig: groups_in_token: bool = False audience: Optional[str] = None issuer: Optional[str] = None - authorization_endpoint: Optional[str] = None def _as_bool(value: Any, default: Optional[bool] = None) -> Optional[bool]: @@ -352,7 +350,6 @@ def resolve_config( groups_in_token: Optional[bool] = None, token_endpoint: Optional[str] = None, device_authorization_endpoint: Optional[str] = None, - authorization_endpoint: Optional[str] = None, issuer: Optional[str] = None, discovery_url: Optional[str] = None, ctx: Optional[ssl.SSLContext] = None, @@ -398,9 +395,6 @@ def resolve_config( token_endpoint = ( token_endpoint or _resolve_endpoint(cfg.get(_K_TOKEN_ENDPOINT))) - authorization_endpoint = ( - authorization_endpoint - or _resolve_endpoint(cfg.get(_K_AUTHORIZATION_ENDPOINT))) device_authorization_endpoint = ( device_authorization_endpoint or _resolve_endpoint(cfg.get(_K_DEVICE_ENDPOINT))) @@ -484,9 +478,6 @@ def resolve_config( or _str_setting(doc.get('device_authorization_endpoint'))) token_endpoint = ( token_endpoint or _str_setting(doc.get('token_endpoint'))) - authorization_endpoint = ( - authorization_endpoint - or _str_setting(doc.get('authorization_endpoint'))) # OIDC Discovery §4.3 / RFC 8414 §3: when pinned ONLY by discovery_url, # the document's self-declared issuer (the anchor # validate_endpoint_origins would use) comes from that same untrusted @@ -535,5 +526,4 @@ def resolve_config( scope=scope, groups_in_token=bool(groups_in_token), audience=audience, - issuer=issuer, - authorization_endpoint=authorization_endpoint) + issuer=issuer) diff --git a/test/test_auth.py b/test/test_auth.py index f199278f..23d5be2b 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -180,16 +180,11 @@ def __init__(self): self.refresh_response = None # (status, body) or None self.device_response = None # override device-auth response body self.device_status = 200 - self.expected_bearer = None # for /exec auth check - self.exec_response = None - self.exec_status = 200 - self.exec_raw = None # (status, content_type, bytes) override # Recording. self.device_requests = 0 self.token_requests = [] self.refresh_requests = 0 self.refresh_forms = [] - self.exec_requests = [] class _Handler(http.server.BaseHTTPRequestHandler): @@ -222,32 +217,6 @@ def do_GET(self): self._send_json(404, {'error': 'not found'}) else: self._send_json(200, self.state.well_known) - elif path == '/exec': - auth = self.headers.get('Authorization') - if self.state.expected_bearer and auth != ( - 'Bearer ' + self.state.expected_bearer): - self._send_json(401, {'error': 'unauthorized'}) - return - self.state.exec_requests.append(self.path) - if self.state.exec_raw is not None: - status, ctype, raw = self.state.exec_raw - self.send_response(status) - self.send_header('Content-Type', ctype) - self.send_header('Content-Length', str(len(raw))) - self.end_headers() - self.wfile.write(raw) - return - self._send_json(self.state.exec_status, self.state.exec_response or { - 'columns': [ - {'name': 'ts', 'type': 'TIMESTAMP'}, - {'name': 'price', 'type': 'DOUBLE'}, - ], - 'dataset': [ - ['2021-01-01T00:00:00.000000Z', 1.5], - ['2021-01-02T00:00:00.000000Z', 2.5], - ], - 'count': 2, - }) else: self._send_json(404, {'error': 'not found'}) @@ -1001,6 +970,31 @@ def test_open_browser_suppressed_in_notebook_kernel(self): {'verification_uri': 'https://idp.example.com/device'}) self.mock_browser_open.assert_not_called() + def test_maybe_open_browser_swallows_open_error(self): + # webbrowser.open raising (no browser / a bad $BROWSER) must not break + # sign-in: opening is best-effort, the prompt is already shown. + auth = self.make_auth(open_browser=True) + with mock.patch('webbrowser.open', side_effect=RuntimeError('boom')): + auth._maybe_open_browser( # must not raise + {'verification_uri': 'https://idp.example.com/device'}) + + def test_identity_from_claims_precedence(self): + # The sign-in success message picks an identity in a fixed precedence: + # email > preferred_username > upn > name > sub. + from questdb.auth._device import _identity_from_claims + self.assertEqual(_identity_from_claims({ + 'email': 'a@x', 'preferred_username': 'pu', 'upn': 'u', + 'name': 'N', 'sub': 's'}), 'a@x') + self.assertEqual(_identity_from_claims({ + 'preferred_username': 'pu', 'upn': 'u', 'name': 'N', + 'sub': 's'}), 'pu') + self.assertEqual(_identity_from_claims({'upn': 'u', 'sub': 's'}), 'u') + self.assertEqual(_identity_from_claims({'name': 'N', 'sub': 's'}), 'N') + self.assertEqual(_identity_from_claims({'sub': 's'}), 's') + self.assertEqual(_identity_from_claims({'sub': 123}), '123') # stringified + self.assertIsNone(_identity_from_claims({})) + self.assertIsNone(_identity_from_claims({'email': ''})) # empty skipped + def test_memory_cache_returns_independent_copy(self): cache = MemoryCache() stored = TokenSet(access_token='a', refresh_token='r', expires_at=1.0) @@ -1239,6 +1233,19 @@ def test_refresh_includes_audience_when_configured(self): auth2.token() self.assertNotIn('audience', self.state.refresh_forms[-1]) + def test_empty_audience_normalized_and_not_sent_on_refresh(self): + # An empty-string audience is normalized to None in __init__, so it is + # omitted on refresh too (it was previously sent as `audience=` on + # refresh only, never on device-auth). + _MEMORY_STORE.clear() + _MEMORY_GENERATION.clear() + auth = self.make_auth(audience='') + self.assertIsNone(auth.config.audience) + self._seed_expired(auth) + auth.token() + self.assertEqual(self.state.refresh_requests, 1) + self.assertNotIn('audience', self.state.refresh_forms[-1]) + def test_refresh_transient_5xx_non_interactive_does_not_hard_fail(self): # The worst case: in a non-interactive context (papermill / cron / CI) a # transient refresh error must surface as a retryable OidcNetworkError, @@ -1883,6 +1890,39 @@ def create(**kw): self.assertEqual(cparams['password'], 'TKN') self.assertEqual(auth.calls - before, 2) + def test_sqlalchemy_engine_uses_psycopg2_drivername(self): + # When only psycopg2 (v2) is importable, the SQLAlchemy driver name is + # postgresql+psycopg2, not +psycopg (the v3 branch). + created = {} + fake_sa = types.ModuleType('sqlalchemy') + fake_sa.__path__ = [] + fake_sa.create_engine = lambda url, **kw: object() + + class _Event: + @staticmethod + def listens_for(target, name): + return lambda fn: fn + + fake_sa.event = _Event + fake_eng = types.ModuleType('sqlalchemy.engine') + + class _URL: + @staticmethod + def create(**kw): + created.update(kw) + return 'URL' + + fake_eng.URL = _URL + fake_pg2 = types.ModuleType('psycopg2') + + with mock.patch.dict(sys.modules, { + 'sqlalchemy': fake_sa, + 'sqlalchemy.engine': fake_eng, + 'psycopg': None, # force the psycopg2 fallback in _pg_module + 'psycopg2': fake_pg2}): + sqlalchemy_engine(_FakeAuth(), 'http://db.example.com:9000') + self.assertEqual(created['drivername'], 'postgresql+psycopg2') + def test_require_host_rejects_hostless_url(self): # A URL with no extractable host must raise, not pass None to a driver; # an explicit host= override still resolves. @@ -1951,6 +1991,12 @@ def test_as_bool_variants(self): self.assertIs(_as_bool(v), False) self.assertIsNone(_as_bool(None)) self.assertIs(_as_bool(None, default=True), True) + # A non-0/1 number coerces via bool(); an unrecognized string / type + # falls back to the default rather than guessing True/False. + self.assertIs(_as_bool(2), True) + self.assertIs(_as_bool(0.0), False) + self.assertIsNone(_as_bool('maybe')) + self.assertIs(_as_bool('maybe', default=False), False) def test_resolve_endpoint_accepts_only_absolute_url(self): # Matching the Java client, a /settings endpoint is trusted only as a @@ -1963,6 +2009,8 @@ def test_resolve_endpoint_accepts_only_absolute_url(self): 'http://idp:9000/x') self.assertIsNone(_resolve_endpoint('/as/token.oauth2')) self.assertIsNone(_resolve_endpoint('')) + self.assertIsNone(_resolve_endpoint('//idp/x')) # scheme-relative + self.assertIsNone(_resolve_endpoint('ftp://idp/x')) # non-http scheme def test_resolve_endpoint_ignores_non_string(self): # A non-string endpoint from /settings (e.g. a JSON number) must be @@ -2617,6 +2665,51 @@ def _display(self, html_str): # avoid importing IPython self.assertIn(': the data-URI is gated on _safe_link_url before + # qrcode is ever called. Holds whether or not the optional qrcode dep is + # installed (a dangerous URL never reaches the encoder). + from questdb.auth._render import JupyterRenderer + captured = {} + + class _Capturing(JupyterRenderer): + def _display(self, html_str): + captured['html'] = html_str + + _Capturing(qr=True).on_prompt({ + 'user_code': 'X', 'verification_uri': 'javascript:alert(1)', + 'expires_in': 600, 'interval': 5}) + self.assertNotIn(' Date: Wed, 24 Jun 2026 18:05:58 +0100 Subject: [PATCH 061/104] fix(auth): evict unusable token; sanitize error output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes from the PR review, both hardening the new questdb.auth module. M1 — re-refresh loop: when a silent refresh succeeded but lacked the required token kind (groups mode + an IdP that doesn't re-issue the id_token), or the refresh token was rejected, _acquire fell through to the device flow without dropping the stale token. If that flow then failed (non-interactive, user cancels, IdP rejects), the doomed refresh_token stayed cached and was re-tried on every later token() call. Evict it (instance + shared cache) before the flow, via a new generation-preserving MemoryCache.evict() so the acquisition's own store_if_current still lands the fresh token. The existing regression test was blind (one token() call, no refresh-count assertion); it now calls token() repeatedly and asserts the refresh count stays at 1. M2 — terminal/HTML injection via uncaught traceback: error messages interpolate untrusted IdP fields (error_description, response bodies), and an uncaught exception's traceback is a display sink the renderer's _strip_control never sees, so a hostile/MITM'd IdP could inject ANSI escapes or a bidi override to spoof the prompt. Strip control chars centrally in OidcError.__init__ (all message args) and OidcDeviceFlowError (error/error_description attrs) so no raise site can forget. All 163 auth tests pass; each fix was confirmed to fail-closed by temporarily reverting it. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_cache.py | 15 +++++++++ src/questdb/auth/_device.py | 11 +++++++ src/questdb/auth/_errors.py | 25 +++++++++++++-- test/test_auth.py | 61 +++++++++++++++++++++++++++++++++++-- 4 files changed, 107 insertions(+), 5 deletions(-) diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index 7b0479cb..c2c0e281 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -104,6 +104,21 @@ def clear(self, key: str) -> None: _MEMORY_STORE.pop(key, None) _MEMORY_GENERATION[key] = _MEMORY_GENERATION.get(key, 0) + 1 + def evict(self, key: str) -> None: + """ + Drop ``key`` WITHOUT bumping the clear()-generation. + + For an in-flight acquisition that found the cached token unusable (its + refresh_token can't yield the required kind) and is about to replace it. + Unlike :meth:`clear` (a user-facing "forget this"), this must NOT bump + the generation: the same acquisition's :meth:`store_if_current` would + otherwise mistake its own eviction for a concurrent ``clear()`` and drop + the fresh token it is about to store. A genuine concurrent ``clear()`` + still bumps the generation and is still honored. + """ + with _MEMORY_LOCK: + _MEMORY_STORE.pop(key, None) + def generation(self, key: str) -> int: """ Current clear()-generation for ``key``. diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index e85eb647..95720c22 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -564,6 +564,17 @@ def _acquire(self, generation: int) -> TokenSet: if self._has_required_token(refreshed): self._store(refreshed, generation) return refreshed + # The refresh path is exhausted: the refresh_token is proven useless + # (rejected, or the IdP won't re-issue the required kind), so the + # device flow below is the only way forward. Drop the stale token — + # from this instance AND the shared cache — before the flow, so that + # if it then FAILS (non-interactive, user cancels, IdP rejects the + # device request) the doomed token isn't left cached to be reloaded + # and re-refreshed fruitlessly on every later token() call. evict() + # doesn't bump the clear()-generation, so the _store below still + # lands `fresh` (unless a genuine concurrent clear() intervenes). + self._tokens = None + self._cache.evict(self.cache_key) fresh = self._run_device_flow() self._store(fresh, generation) diff --git a/src/questdb/auth/_errors.py b/src/questdb/auth/_errors.py index c4741cf4..841e40e7 100644 --- a/src/questdb/auth/_errors.py +++ b/src/questdb/auth/_errors.py @@ -28,11 +28,26 @@ from typing import Optional +# _render is stdlib-only (no internal imports), so this introduces no cycle. +from ._render import _strip_control + class OidcError(Exception): """Base class for every error raised by :mod:`questdb.auth`.""" def __init__(self, *args, status: Optional[int] = None): + # Strip terminal/bidi/zero-width control characters from every string + # message argument before it can reach a display sink. Error messages + # routinely interpolate untrusted IdP fields (error_description, response + # bodies, verification URIs), and an uncaught exception's traceback — + # printed to a terminal or rendered by Jupyter, both of which interpret + # ANSI — is a sink the renderer's own sanitization never sees. Without + # this, a hostile or MITM'd IdP could inject ANSI escapes or a bidi + # override into that traceback to spoof the prompt. Doing it here (not at + # each raise site) means no raise site can forget; non-string args (rare) + # pass through unchanged. + args = tuple( + _strip_control(a) if isinstance(a, str) else a for a in args) super().__init__(*args) # HTTP status behind a non-JSON HTTP response (else None), so the poll # loop and silent refresh can tell a terminal 4xx (e.g. a WAF error @@ -73,8 +88,14 @@ def __init__( error: Optional[str] = None, error_description: Optional[str] = None): super().__init__(message) - self.error = error - self.error_description = error_description + # error / error_description come straight from the untrusted IdP + # response and are exposed as attributes (a caller may re-display them), + # so strip them too — same rationale as the message in OidcError. None is + # kept as None (not coerced to '') so "absent" stays distinguishable. + self.error = _strip_control(error) if error is not None else None + self.error_description = ( + _strip_control(error_description) + if error_description is not None else None) class OidcTimeoutError(OidcDeviceFlowError): diff --git a/test/test_auth.py b/test/test_auth.py index 23d5be2b..150daee6 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -579,6 +579,30 @@ def test_access_denied_is_surfaced(self): self.assertEqual(cm.exception.error, 'access_denied') self.assertIn('user said no', str(cm.exception)) + def test_hostile_error_fields_sanitized_in_exception(self): + # A hostile/MITM'd IdP error/error_description must not smuggle terminal + # escapes or a bidi override into the raised exception: an uncaught + # traceback is a display sink (a terminal and Jupyter both interpret + # ANSI) that the renderer's own sanitization never sees. OidcError + # strips them centrally, so the message and the exposed attributes are + # clean while the human-readable text still survives. (M2) + self.state.token_script = [ + (400, {'error': 'access_denied\x1b[31m', + 'error_description': + 'denied \x1b[2J\x1b[1;1H\u202eevil.example\x07'}), + ] + auth = self.make_auth() + with self.assertRaises(OidcDeviceFlowError) as cm: + auth.token() + for s in (str(cm.exception), + cm.exception.error or '', + cm.exception.error_description or ''): + self.assertNotIn('\x1b', s) # ESC stripped + self.assertNotIn('\x07', s) # BEL stripped + self.assertNotIn('\u202e', s) # bidi override stripped + self.assertIn('denied', str(cm.exception)) + self.assertIn('evil.example', cm.exception.error_description) + def test_device_endpoint_rejects_grant(self): self.state.device_status = 400 self.state.device_response = {'error': 'invalid_client'} @@ -1139,14 +1163,22 @@ def test_refresh_without_id_token_falls_back_to_device_flow(self): def test_refresh_without_id_token_non_interactive_does_not_loop(self): # Same situation but non-interactive: surface a clear error rather than - # repeatedly re-running a refresh that can never satisfy _select. + # repeatedly re-running a refresh that can never satisfy _select. The + # doomed refresh_token must be evicted on the first failure so a later + # call goes straight to the (failing) device flow instead of re-issuing + # the same fruitless refresh on every token() call. Calling token() + # several times must therefore not climb the refresh count. auth = self.make_auth(groups_in_token=True, interactive=False) self._seed_expired(auth) self.state.refresh_response = (200, { 'access_token': ACCESS_TOKEN, 'token_type': 'Bearer', 'expires_in': 3600}) # no id_token - with self.assertRaises(OidcInteractionRequired): - auth.token() + for _ in range(3): + with self.assertRaises(OidcInteractionRequired): + auth.token() + # Exactly one refresh across all three calls (without the eviction the + # stale token would be reloaded and re-refreshed every call). + self.assertEqual(self.state.refresh_requests, 1) self.assertEqual(self.state.device_requests, 0) def test_cached_token_missing_required_kind_is_refreshed(self): @@ -2835,6 +2867,29 @@ def test_strip_control_removes_bidi_and_zero_width(self): self.assertNotIn(chr(0x202e), text) self.assertIn('idp.example.com', text) + def test_oidc_error_sanitizes_message_and_fields(self): + # OidcError strips control/bidi chars from its message centrally, so no + # raise site can leak an ANSI/bidi sequence into an uncaught traceback (a + # display sink the renderer never sees). The device-flow subclass strips + # its untrusted error / error_description attributes too. See M2. + e = OidcError('boom ' + chr(0x1b) + '[31m' + chr(0x202e) + + 'hidden' + chr(0x07) + ' done') + self.assertEqual(str(e), 'boom [31mhidden done') + self.assertEqual(OidcError('x', status=503).status, 503) # status kept + d = OidcDeviceFlowError( + 'failed ' + chr(0x1b) + '[2Jx', + error='bad' + chr(0x1b) + ']0;t' + chr(0x07), + error_description='why ' + chr(0x202e) + 'flip' + chr(0x1b) + '[0m') + for s in (str(d), d.error, d.error_description): + self.assertNotIn(chr(0x1b), s) + self.assertNotIn(chr(0x07), s) + self.assertNotIn(chr(0x202e), s) + self.assertIn('flip', d.error_description) # readable text survives + # Absent error / error_description stay None (not coerced to ''). + d2 = OidcDeviceFlowError('x') + self.assertIsNone(d2.error) + self.assertIsNone(d2.error_description) + def test_qr_helpers_degrade_without_qrcode(self): # The QR helpers must degrade gracefully (return None), never raise, # when `qrcode` is absent or the data is empty. See M4. From b5690140a9cf3df426015fe5d5d7f3fa83a7f98f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 24 Jun 2026 18:14:23 +0100 Subject: [PATCH 062/104] fix(auth): enforce the discovery_url pin on /settings endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C1 (Critical) from the PR review: the plaintext-/settings guard tells the user that pinning with issuer= OR discovery_url= makes a tampered /settings safe, but discovery_url was never actually enforced when /settings advertised BOTH credential endpoints. In that case the plaintext guard was disabled (a pin was present), the issuer path-check was skipped (no issuer), and the IdP-discovery block — the only place discovery_url was checked — was skipped (no endpoint missing). issuer stayed None, so the co-location check in __init__ passed trivially and the device code and refresh token were POSTed to the attacker origin the pin was meant to forbid. A user who followed the documented mitigation (discovery_url=) was left unprotected. Add a discovery_url-origin check for /settings-supplied credential endpoints that runs even when discovery is skipped, mirroring the issuer path-check directly above it: each settings-advertised endpoint must share the pinned discovery_url origin. Caller-explicit endpoints stay authoritative; when discovery does run, the existing block still pins the discovered endpoints. Reproduced end-to-end via resolve_config; the new reject test fails (no error raised) without this change. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 28 ++++++++++++++++++++++++++++ test/test_auth.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 1bdefcc0..543e53dd 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -446,6 +446,34 @@ def resolve_config( 'them explicitly (token_endpoint=..., ' 'device_authorization_endpoint=...).') + # Same scoping for a discovery_url-only pin (no issuer): require each + # /settings credential endpoint on the pinned discovery origin. The plaintext + # guard above accepts discovery_url= as a sufficient pin, but the only other + # discovery_url enforcement lives in the IdP-discovery block below, which is + # SKIPPED when /settings already advertises both endpoints — so without this + # a tampered plaintext /settings could route the device code and refresh + # token to an attacker origin the pin was meant to forbid. When discovery + # DOES run, that block additionally pins the discovered endpoint(s) to this + # origin. Caller-explicit endpoints are authoritative and skip this. + if discovery_url and not issuer: + discovery_origin = _normalized_origin(discovery_url) + for label, url, from_settings in ( + ('token endpoint', token_endpoint, + not explicit_token_endpoint), + ('device-authorization endpoint', + device_authorization_endpoint, not explicit_device_endpoint)): + if (url and from_settings + and _normalized_origin(url) != discovery_origin): + raise OidcConfigError( + f'The OIDC {label} advertised by QuestDB /settings ' + f'({url!r}) is not on the pinned discovery_url origin ' + f'({_origin_str(discovery_url)}); refusing to send ' + 'credentials to an endpoint off the pinned IdP origin. If ' + 'your IdP serves discovery and tokens from different ' + 'origins, pin with issuer=... or pass the endpoints ' + 'explicitly (token_endpoint=..., ' + 'device_authorization_endpoint=...).') + # Fall back to IdP discovery when QuestDB doesn't advertise the device # (and/or token) endpoint. This contacts the IdP, so it is held to # https/loopback (insecure=False) regardless of the QuestDB flag. diff --git a/test/test_auth.py b/test/test_auth.py index 150daee6..0cea2f94 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1646,6 +1646,37 @@ def test_pin_satisfies_guard_over_plaintext(self): insecure=True, issuer='https://evil.example.com') self.assertEqual(cfg.token_endpoint, 'https://evil.example.com/token') + def test_discovery_url_pin_scopes_settings_endpoints(self): + # C1: the plaintext guard accepts discovery_url= as a sufficient pin, but + # when /settings advertises BOTH credential endpoints the IdP-discovery + # block (the only other discovery_url enforcement) is skipped. A + # discovery_url pin must still constrain settings-supplied endpoints to + # its origin, else a tampered plaintext /settings routes credentials to + # an attacker the pin was meant to forbid. IdP discovery must NOT run. + with self.assertRaises(OidcConfigError) as cm: + self._resolve( + self._TAMPERED, # both endpoints on evil.example.com + questdb_url='http://qdb.internal.example:9000', insecure=True, + discovery_url='https://idp.example.com/.well-known/' + 'openid-configuration') + self.assertIn('discovery_url', str(cm.exception)) + + def test_discovery_url_pin_accepts_on_origin_settings_endpoints(self): + # The legitimate case: /settings endpoints on the pinned discovery_url + # origin are accepted (origin-level, so a different path is fine), with + # no IdP round-trip since both endpoints are present. + good = { + 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': 'https://idp.example.com/oauth/token', + 'acl.oidc.device.authorization.endpoint': + 'https://idp.example.com/oauth/device'} + cfg = self._resolve( + good, questdb_url='http://qdb.internal.example:9000', insecure=True, + discovery_url='https://idp.example.com/.well-known/' + 'openid-configuration') + self.assertEqual(cfg.token_endpoint, + 'https://idp.example.com/oauth/token') + def test_issuer_path_scopes_settings_endpoints(self): # M1: a tampered /settings advertising a DIFFERENT realm's endpoints on # the SAME host (Keycloak path-based multi-tenancy) is rejected when the From baa4c4d6efed6f2112078365ccbab307b1384716 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 24 Jun 2026 18:25:09 +0100 Subject: [PATCH 063/104] fix(auth): fail fast on a 3xx poll; normalize issuer in cache key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two minor robustness fixes from the PR review. Poll classifier: a non-JSON 3xx from the token endpoint (e.g. an HTML redirect from a reverse proxy; _NoRedirect refuses to follow it, so post_form raises OidcError(status=3xx)) was neither terminal-4xx nor 5xx/429, so the loop kept polling to the ~10-min device-code deadline and raised a misleading OidcTimeoutError("code expired") — while a JSON-bodied 3xx already failed fast. Rename _http_status_is_terminal_4xx -> _http_status_is_terminal and classify any definitive non-transient status (3xx, a non-conformant 2xx, and 4xx; 5xx/429 stay transient) as terminal, so the poll fails fast with a clear error. cache_key: the issuer was interleaved raw while the token endpoint was normalized, so two functionally identical auth objects whose issuer differed only by a trailing slash, case, or an explicit :443 got distinct keys and re-prompted/refreshed unnecessarily (common: discovery returns "https://idp/" vs an explicit "https://idp"). Normalize the issuer the same way (lower-case scheme/host, drop a default port) and strip a trailing slash; the realm path is kept so multi-tenant issuers stay distinct. All 169 auth tests pass; each fix was confirmed to fail-closed by temporarily reverting it. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 44 +++++++++++++++++++---------- test/test_auth.py | 55 +++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 14 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 95720c22..0ae6a529 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -149,16 +149,23 @@ def _identity_from_claims(claims: Dict[str, Any]) -> Optional[str]: return None -def _http_status_is_terminal_4xx(status: Optional[int]) -> bool: +def _http_status_is_terminal(status: Optional[int]) -> bool: """ - True for a 4xx that is a definitive rejection. - - A non-JSON body with such a status (e.g. an HTML ``403`` from a WAF/proxy or - non-conformant IdP) is never an ``authorization_pending`` / ``slow_down`` - (those are always JSON), so the poll must fail fast rather than retry to a - misleading "code expired". ``429`` is excluded — it's a transient rate-limit. + True for an HTTP status that is a definitive rejection, not a transient poll + state worth retrying. + + A conformant token-endpoint poll reply is JSON (a 200 success body, or a 4xx + whose JSON body carries ``authorization_pending`` / ``slow_down``). A NON-JSON + body — which makes ``post_form`` raise with the status attached — therefore + means a proxy / WAF / non-conformant IdP, never a poll state. Any such status + that is not a transient 5xx/429 (and not a bare network error, which carries + no status) is terminal, so the poll fails fast instead of retrying to a + misleading "code expired". This covers a non-JSON ``3xx`` redirect (these + endpoints never legitimately redirect, and ``_NoRedirect`` refuses to follow + one) and a non-conformant non-JSON ``2xx``, as well as a non-JSON ``4xx``; + ``429`` stays transient. """ - return status is not None and 400 <= status < 500 and status != 429 + return status is not None and status < 500 and status != 429 def _http_status_is_transient(status: Optional[int]) -> bool: @@ -435,8 +442,14 @@ def cache_key(self) -> str: """ c = self.config scope = ' '.join(sorted(c.scope.split())) if c.scope else '' + # Normalize the issuer like the token endpoint (lower-case scheme/host, + # drop a default port) and strip a trailing slash, so a discovered + # "https://idp/" and an explicit "https://idp" — or a stray :443 / case + # difference — don't yield different keys and force an avoidable + # re-prompt. The realm path is kept (multi-tenant issuers differ by it). + issuer = _normalize_url(c.issuer).rstrip('/') if c.issuer else '' return '\x1f'.join([ - c.issuer or '', + issuer, _normalize_url(c.token_endpoint), c.client_id, scope, @@ -790,11 +803,14 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: 'client_id': self.config.client_id, }) except OidcError as e: - # A non-JSON 4xx is a terminal rejection (e.g. an HTML error page - # from a WAF/proxy, or a non-conformant IdP): a conformant OAuth - # error is JSON, so it can never be authorization_pending / - # slow_down. Fail fast instead of polling on to "code expired". - if _http_status_is_terminal_4xx(getattr(e, 'status', None)): + # A non-JSON, non-transient status is a terminal rejection (an + # HTML error page or a redirect from a WAF/proxy, or a non- + # conformant IdP): a conformant OAuth poll reply is JSON, so it + # can never be authorization_pending / slow_down. Fail fast — + # including on a 3xx, which these endpoints never legitimately + # return and _NoRedirect won't follow — instead of polling on to + # a misleading "code expired". + if _http_status_is_terminal(getattr(e, 'status', None)): self._renderer.on_failure( 'Sign-in failed: the identity provider rejected the ' 'request.') diff --git a/test/test_auth.py b/test/test_auth.py index 0cea2f94..79f1b7db 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -453,6 +453,38 @@ def test_non_json_4xx_during_poll_is_terminal(self): self.assertNotIsInstance(cm.exception, OidcTimeoutError) self.assertLessEqual(len(self._clock.sleeps), 1) + def test_non_json_3xx_during_poll_is_terminal(self): + # A non-JSON 3xx (an HTML redirect from a reverse proxy in front of the + # token endpoint) must fail fast too: these endpoints never legitimately + # redirect, _NoRedirect refuses to follow one, and post_form surfaces it + # as OidcError(status=3xx). Without classifying 3xx as terminal the loop + # would poll on to a misleading "code expired". + auth = self.make_auth() + with _raw_response_server( + 302, 'text/html', b'see /login') as raw: + auth.config.token_endpoint = raw + '/token' # post-construction + with self.assertRaises(OidcDeviceFlowError) as cm: + auth.token() + self.assertNotIsInstance(cm.exception, OidcTimeoutError) + self.assertLessEqual(len(self._clock.sleeps), 1) + + def test_http_status_terminal_vs_transient_classifier(self): + # The poll classifier: 3xx (a redirect these endpoints never return) and + # a non-conformant 2xx are terminal alongside 4xx, so a non-JSON such + # response fails fast; only 5xx/429 (and a status-less network error) are + # transient and retried to the deadline. The two are mutually exclusive + # over any real status. + from questdb.auth._device import ( + _http_status_is_terminal, _http_status_is_transient) + for s in (200, 204, 301, 302, 307, 308, 400, 403, 404): + self.assertTrue(_http_status_is_terminal(s), s) + self.assertFalse(_http_status_is_transient(s), s) + for s in (500, 502, 503, 429): + self.assertFalse(_http_status_is_terminal(s), s) + self.assertTrue(_http_status_is_transient(s), s) + self.assertFalse(_http_status_is_terminal(None)) # network error + self.assertFalse(_http_status_is_transient(None)) + def test_device_200_without_codes_is_rejected_clearly(self): # A 200 device-authorization response missing device_code/user_code is # a non-conformant body, not an HTTP failure: the error must say so @@ -2350,6 +2382,29 @@ def test_default_port_normalized(self): self._auth( token_endpoint='https://idp.example.com:443/token').cache_key) + def test_issuer_trailing_slash_and_case_do_not_change_key(self): + # A discovered issuer often carries a trailing slash ("https://idp/") + # while an explicit one does not ("https://idp"); case and a default :443 + # likewise vary. None of these change the security context, so they must + # not split the cache key and force an avoidable re-prompt. + base = self._auth(issuer='https://idp.example.com') + for variant in ('https://idp.example.com/', + 'https://IDP.example.com', + 'https://idp.example.com:443', + 'https://idp.example.com:443/'): + self.assertEqual(base.cache_key, + self._auth(issuer=variant).cache_key, variant) + + def test_issuer_realm_path_distinguishes_key(self): + # A different realm PATH on the same host is a different issuer and must + # stay a distinct key — origin-only normalization would wrongly collide + # them. + self.assertNotEqual( + self._auth( + issuer='https://idp.example.com/realms/prod').cache_key, + self._auth( + issuer='https://idp.example.com/realms/staging').cache_key) + def test_groups_in_token_distinguishes_key(self): # groups_in_token selects which token kind _select returns, so two # sessions differing ONLY in that mode must not collide on one cache From 075a01c235193880ced351d00ebc5c822f16060f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 02:38:44 +0100 Subject: [PATCH 064/104] fix(auth): scope the issuer-origin pin to /settings; drop discovery_url Pin the issuer ORIGIN only for credential endpoints that came from the untrusted QuestDB /settings response, not for caller-explicit or IdP-discovered endpoints. The OIDC issuer is an identifier, not necessarily the endpoints' host, so the old blanket pin rejected legitimate cross-origin providers (e.g. Google issues from accounts.google.com but serves tokens from oauth2.googleapis.com). validate_endpoint_origins is now co-location-only; the issuer-origin pin moved into resolve_config, where each endpoint's provenance is known. A /settings endpoint off the issuer origin is accepted only when the IdP's own TLS-fetched .well-known confirms the same URL, so a tampered /settings still cannot redirect the device code / refresh token. Also drop the discovery_url argument: issuer= is now the single out-of-band IdP pin (discovery is always {issuer}/.well-known/...). It was a weaker, origin-only anchor that could not handle cross-origin IdPs, and explicit token_endpoint=/device_authorization_endpoint= already cover non-standard discovery URLs. Removes the two discovery_url-only origin checks and the dead doc-issuer adoption from resolve_config. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.rst | 6 +- docs/auth.rst | 32 ++--- src/questdb/auth/_device.py | 15 +-- src/questdb/auth/_discovery.py | 220 +++++++++++++++------------------ test/test_auth.py | 181 ++++++++++++++------------- 5 files changed, 224 insertions(+), 230 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 52e42f8a..08205e6f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -37,7 +37,11 @@ QuestDB over the auth paths it already supports (HTTP ``Bearer`` / PG-wire Highlights: * Auto-discovery of OIDC config from the QuestDB ``/settings`` endpoint, with a - fallback to the IdP ``.well-known`` document. + fallback to the IdP ``.well-known`` document. Works with IdPs whose issuer is + on a different origin than its token / device-authorization endpoints (e.g. + Google: ``accounts.google.com`` issuer, ``oauth2.googleapis.com`` endpoints), + while still pinning endpoints advertised over an untrusted ``/settings`` + channel to the issuer. * In-process token cache with silent refresh (tokens are never written to disk). * Convenience adapters (:func:`~questdb.auth.sqlalchemy_engine`, diff --git a/docs/auth.rst b/docs/auth.rst index bbd1f05c..0c1dc923 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -116,7 +116,7 @@ resolves the OIDC configuration in this order: 2. If the device-authorization endpoint is not advertised, the helper falls back to the IdP discovery document (``{issuer}/.well-known/openid-configuration``). This path **requires** an - explicit ``issuer=`` (or ``discovery_url=``) argument. + explicit ``issuer=`` argument. Anything you pass explicitly overrides discovery. You can also skip discovery entirely: @@ -226,19 +226,23 @@ Security notes QuestDB ``/settings``. The helper requires both endpoints to share a single origin and rejects the configuration otherwise. Because ``/settings`` is authoritative-by-QuestDB, a compromised server could in principle point them - elsewhere; pass ``issuer=`` to **pin** the IdP so the endpoints are verified - to belong to it and credentials can't be redirected to another host. The pin - checks both the **origin** and, for endpoints advertised by ``/settings``, the - issuer **path** — so on a path-based multi-tenant IdP (e.g. Keycloak issuers - ``https://host/realms/{realm}``) a tampered ``/settings`` cannot redirect the - device code / refresh token to a *different realm on the same host*. (Caller- - supplied endpoints and endpoints from the IdP's own discovery document are - trusted as-is and not path-restricted, since some IdPs — e.g. Azure AD — place - their endpoints outside the issuer path; pass such endpoints explicitly or let - discovery resolve them.) When the server does not advertise the device- - authorization endpoint (so it must be discovered from the IdP), ``issuer=`` - (or ``discovery_url=``) is **required** for exactly this reason — the helper - refuses to guess the discovery origin from the server-supplied token endpoint. + elsewhere; pass ``issuer=`` to **pin** the IdP so endpoints advertised over + ``/settings`` are verified to belong to it and credentials can't be redirected + to another host. For a ``/settings`` endpoint the pin checks the issuer + **origin** and **path** — so on a path-based multi-tenant IdP (e.g. Keycloak + issuers ``https://host/realms/{realm}``) a tampered ``/settings`` cannot + redirect the device code / refresh token to a *different realm on the same + host*. (Caller-supplied endpoints and endpoints from the IdP's own + ``.well-known`` document are authoritative and are **not** pinned to the issuer + origin/path: the issuer is an OIDC *identifier*, not necessarily the + endpoints' host — e.g. Google issues from ``accounts.google.com`` but serves + tokens from ``oauth2.googleapis.com``, and some IdPs such as Azure AD place + endpoints outside the issuer path. A ``/settings`` endpoint that sits off the + issuer origin is still accepted when the IdP's own discovery document confirms + the same URL.) When the server does not advertise the device- + authorization endpoint (so it must be discovered from the IdP), ``issuer=`` is + **required** for exactly this reason — the helper refuses to guess the + discovery origin from the server-supplied token endpoint. * Adapters avoid logging the token / PG DSN. Avoid logging them yourself. * Standard proxy / CA settings (``HTTPS_PROXY``, ``REQUESTS_CA_BUNDLE``, ``SSL_CERT_FILE``) are honoured for the IdP / discovery transport; you can diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 0ae6a529..916672fd 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -297,13 +297,16 @@ def __init__( audience=audience, issuer=issuer) - # Enforce the credential-endpoint co-location / issuer pin here too (not - # just on the discovery path), so the guarantee holds for this - # constructor as well. + # Enforce credential-endpoint co-location here too (not just on the + # discovery path), so the guarantee holds for this constructor as well. + # The issuer-ORIGIN pin is provenance-aware and lives in resolve_config + # (it applies only to endpoints from the untrusted /settings); endpoints + # reaching this constructor are caller-explicit (authoritative), so a + # cross-origin issuer — e.g. Google's accounts.google.com issuer with + # oauth2.googleapis.com endpoints — is intentionally accepted here. validate_endpoint_origins( self.config.token_endpoint, - self.config.device_authorization_endpoint, - self.config.issuer) + self.config.device_authorization_endpoint) # `insecure` permits plaintext http only to QuestDB (e.g. local dev). # _idp_post always holds the IdP to https (or loopback http), so the @@ -349,7 +352,6 @@ def from_questdb( audience: Optional[str] = None, groups_in_token: Optional[bool] = None, issuer: Optional[str] = None, - discovery_url: Optional[str] = None, token_endpoint: Optional[str] = None, device_authorization_endpoint: Optional[str] = None, insecure: bool = False, @@ -385,7 +387,6 @@ def from_questdb( token_endpoint=token_endpoint, device_authorization_endpoint=device_authorization_endpoint, issuer=issuer, - discovery_url=discovery_url, ctx=ctx, insecure=insecure, timeout=timeout) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 543e53dd..0f5df6ab 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -243,29 +243,25 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: def validate_endpoint_origins( token_endpoint: str, - device_authorization_endpoint: str, - issuer: Optional[str] = None) -> None: + device_authorization_endpoint: str) -> None: """ - Reject an OIDC configuration that would send credentials off-origin. + Require the two credential endpoints to share a single origin. The device code and long-lived refresh token are POSTed to the device- - authorization and token endpoints. This limits a tampered or MITM'd config - from steering those credentials to an attacker host: - - * the two credential endpoints must share a single origin (always co-located - on the authorization server per RFC 8628); and - * when ``issuer`` is known independently (explicit or from the IdP - ``.well-known``), both endpoints must share its **origin**. - - Origin-level only: it does **not** isolate path-based multi-tenant realms - (e.g. Keycloak ``https://host/realms/{realm}``, one origin per realm). That - path-scoping lives in :func:`resolve_config`, and only for endpoints from the - untrusted QuestDB ``/settings``; endpoints from IdP discovery or the caller - are authoritative and not path-restricted (some IdPs, e.g. Azure AD, - legitimately place endpoints outside the issuer path). - - Pass ``issuer=`` to pin the IdP when QuestDB advertises the endpoints - directly, so a compromised server cannot redirect the token POST. + authorization and token endpoints, which RFC 8628 always co-locates on one + authorization server. A configuration that splits them across origins is + therefore malformed or tampered; refuse it rather than POST the credentials + to two different hosts. + + The issuer-**origin** pin for endpoints sourced from the untrusted QuestDB + ``/settings`` response is enforced separately, in :func:`resolve_config`, + where each endpoint's provenance is known. Endpoints passed explicitly by the + caller, or discovered from the IdP's own (authoritative, TLS-fetched) + ``.well-known`` document, are NOT pinned to the issuer origin: the OIDC + issuer is an *identifier*, not necessarily the endpoints' host (e.g. Google + issues from ``accounts.google.com`` but serves tokens from + ``oauth2.googleapis.com``), so requiring issuer-origin equality there would + reject a legitimate cross-origin IdP. """ if _normalized_origin(token_endpoint) != _normalized_origin( device_authorization_endpoint): @@ -275,18 +271,6 @@ def validate_endpoint_origins( f'{_origin_str(device_authorization_endpoint)}); refusing to send ' 'credentials. This indicates a misconfigured or tampered OIDC ' 'configuration.') - if issuer: - issuer_origin = _normalized_origin(issuer) - for label, url in ( - ('token endpoint', token_endpoint), - ('device-authorization endpoint', - device_authorization_endpoint)): - if _normalized_origin(url) != issuer_origin: - raise OidcConfigError( - f'OIDC {label} origin ({_origin_str(url)}) does not match ' - f'the issuer origin ({_origin_str(issuer)}); refusing to ' - 'send credentials to an endpoint outside the trusted ' - 'issuer.') def _resolve_endpoint(value: Any) -> Optional[str]: @@ -296,7 +280,7 @@ def _resolve_endpoint(value: Any) -> Optional[str]: Mirroring the Java client, a QuestDB-advertised endpoint is taken verbatim and only as an absolute URL. A path-only (or otherwise non-absolute, or non-string) value is treated as absent, so resolution falls back to the IdP - ``.well-known`` document — which requires an issuer / ``discovery_url`` pin. + ``.well-known`` document — which requires an ``issuer`` pin. We deliberately do **not** assemble a URL from ``acl.oidc.host`` / ``acl.oidc.port`` / ``acl.oidc.tls.enabled``: those are server building blocks, not a credential-routing source the client should trust. @@ -314,26 +298,26 @@ def well_known_url(issuer: str) -> str: def discover_device_endpoint_from_idp( *, issuer: Optional[str], - discovery_url: Optional[str], ctx: Optional[ssl.SSLContext] = None, insecure: bool = False, timeout: float = 30) -> Dict[str, Any]: """ Fetch the IdP ``.well-known/openid-configuration`` and return it. - The discovery URL comes from ``discovery_url``, else built from ``issuer``; - one is required. The discovery origin is **never** derived from a - QuestDB-advertised endpoint — that would let a tampered ``/settings`` choose - where credentials are sent, with the co-location / issuer-pin checks passing - trivially because every value would share the attacker's origin. + The discovery URL is built from the pinned ``issuer`` + (``{issuer}/.well-known/openid-configuration``). The discovery origin is + **never** derived from a QuestDB-advertised endpoint — that would let a + tampered ``/settings`` choose where credentials are sent, with the + co-location / issuer-pin checks passing trivially because every value would + share the attacker's origin. """ - url = discovery_url or (well_known_url(issuer) if issuer else None) - if not url: + if not issuer: raise OidcConfigError( 'Cannot discover the IdP device-authorization endpoint: no issuer ' - 'or discovery_url was given. Pass issuer=... (or ' - 'device_authorization_endpoint=... to skip discovery).') - doc = get_json(url, ctx=ctx, insecure=insecure, timeout=timeout) + 'was given. Pass issuer=... (or device_authorization_endpoint=... ' + 'to skip discovery).') + doc = get_json( + well_known_url(issuer), ctx=ctx, insecure=insecure, timeout=timeout) # get_json guarantees valid JSON, not a JSON *object*. Coerce a non-dict # document (from a captive portal, bad proxy, or hostile IdP) to empty so # resolve_config's doc.get(...) yields a clear "could not resolve" error @@ -351,7 +335,6 @@ def resolve_config( token_endpoint: Optional[str] = None, device_authorization_endpoint: Optional[str] = None, issuer: Optional[str] = None, - discovery_url: Optional[str] = None, ctx: Optional[ssl.SSLContext] = None, insecure: bool = False, timeout: float = 30) -> OidcConfig: @@ -399,28 +382,40 @@ def resolve_config( device_authorization_endpoint or _resolve_endpoint(cfg.get(_K_DEVICE_ENDPOINT))) + # Freeze each endpoint's provenance BEFORE discovery may fill a missing one. + # Only endpoints that came from the untrusted /settings get pinned to the + # issuer origin below; a caller-explicit or IdP-discovered endpoint is + # authoritative. `doc_*_endpoint` record what IdP discovery advertised, so a + # /settings endpoint the IdP's own document confirms can be trusted even when + # it sits off the issuer origin (e.g. Google: accounts.google.com issuer vs + # oauth2.googleapis.com endpoints). + token_from_settings = bool(token_endpoint) and not explicit_token_endpoint + device_from_settings = ( + bool(device_authorization_endpoint) and not explicit_device_endpoint) + doc_token_endpoint: Optional[str] = None + doc_device_endpoint: Optional[str] = None + # Over a plaintext-http /settings channel (insecure=True, non-loopback), a # tampered response can advertise BOTH credential endpoints at one attacker # origin: the discovery path below is skipped, co-location passes trivially # (shared origin) and the issuer-pin check is vacuous (no issuer), so nothing - # else catches it. Demand the same out-of-band pin (issuer= / discovery_url=) - # before trusting /settings endpoints here. Caller-explicit endpoints and - # those from an authenticated (https / loopback) /settings are unaffected. + # else catches it. Demand an out-of-band issuer pin before trusting /settings + # endpoints here. Caller-explicit endpoints and those from an authenticated + # (https / loopback) /settings are unaffected. settings_supplied_credentials = ( (token_endpoint and not explicit_token_endpoint) or (device_authorization_endpoint and not explicit_device_endpoint)) if (questdb_url and settings_supplied_credentials - and not issuer and not discovery_url + and not issuer and _settings_channel_is_plaintext(questdb_url)): raise OidcConfigError( 'QuestDB was reached over plaintext http (insecure=True), so its ' '/settings response — and the OIDC endpoints it advertises — can be ' 'tampered in transit and used to redirect the device-code and ' 'refresh-token requests to an attacker. Pin the identity provider ' - 'out-of-band with issuer="https://your-idp" (or discovery_url=...), ' - 'pass the endpoints explicitly (token_endpoint=..., ' - 'device_authorization_endpoint=...), or connect to QuestDB over ' - 'https so /settings is authenticated.') + 'out-of-band with issuer="https://your-idp", pass the endpoints ' + 'explicitly (token_endpoint=..., device_authorization_endpoint=...), ' + 'or connect to QuestDB over https so /settings is authenticated.') # For /settings endpoints with an out-of-band issuer, require each under the # issuer's PATH, not just its origin: path-based IdPs share one origin per @@ -446,34 +441,6 @@ def resolve_config( 'them explicitly (token_endpoint=..., ' 'device_authorization_endpoint=...).') - # Same scoping for a discovery_url-only pin (no issuer): require each - # /settings credential endpoint on the pinned discovery origin. The plaintext - # guard above accepts discovery_url= as a sufficient pin, but the only other - # discovery_url enforcement lives in the IdP-discovery block below, which is - # SKIPPED when /settings already advertises both endpoints — so without this - # a tampered plaintext /settings could route the device code and refresh - # token to an attacker origin the pin was meant to forbid. When discovery - # DOES run, that block additionally pins the discovered endpoint(s) to this - # origin. Caller-explicit endpoints are authoritative and skip this. - if discovery_url and not issuer: - discovery_origin = _normalized_origin(discovery_url) - for label, url, from_settings in ( - ('token endpoint', token_endpoint, - not explicit_token_endpoint), - ('device-authorization endpoint', - device_authorization_endpoint, not explicit_device_endpoint)): - if (url and from_settings - and _normalized_origin(url) != discovery_origin): - raise OidcConfigError( - f'The OIDC {label} advertised by QuestDB /settings ' - f'({url!r}) is not on the pinned discovery_url origin ' - f'({_origin_str(discovery_url)}); refusing to send ' - 'credentials to an endpoint off the pinned IdP origin. If ' - 'your IdP serves discovery and tokens from different ' - 'origins, pin with issuer=... or pass the endpoints ' - 'explicitly (token_endpoint=..., ' - 'device_authorization_endpoint=...).') - # Fall back to IdP discovery when QuestDB doesn't advertise the device # (and/or token) endpoint. This contacts the IdP, so it is held to # https/loopback (insecure=False) regardless of the QuestDB flag. @@ -482,7 +449,7 @@ def resolve_config( # target would be guessed from the /settings token endpoint, so a # tampered /settings could steer discovery (and the credential POSTs) to # an attacker origin with co-location / issuer-pin passing trivially. - if not issuer and not discovery_url: + if not issuer: raise OidcConfigError( 'QuestDB did not advertise the OIDC device-authorization ' 'endpoint (and/or the token endpoint), so it must be ' @@ -492,44 +459,22 @@ def resolve_config( 'the device-code and refresh-token requests to an attacker. ' 'Alternatively pass the endpoint(s) explicitly ' '(device_authorization_endpoint=..., token_endpoint=...) to ' - 'skip discovery, or discovery_url=... to pin the discovery ' - 'document.') + 'skip discovery.') doc = discover_device_endpoint_from_idp( - issuer=issuer, discovery_url=discovery_url, - ctx=ctx, insecure=False, timeout=timeout) - # The discovery document is untrusted too: coerce its values like - # /settings. A non-string endpoint / issuer reads as absent (clear - # "could not resolve" below, or no issuer pin) instead of reaching - # safe_urlparse / the cache-key join as a raw object. + issuer=issuer, ctx=ctx, insecure=False, timeout=timeout) + # The discovery document is authoritative — fetched over TLS from the + # pinned issuer's own origin — but still coerce its values: a non-string + # endpoint reads as absent (clear "could not resolve" below) instead of + # reaching safe_urlparse / the cache-key join as a raw object. These + # discovered URLs are trusted as-is (no issuer-origin pin), so a + # cross-origin IdP — e.g. Google, which issues from accounts.google.com + # but serves tokens from oauth2.googleapis.com — resolves correctly. + doc_token_endpoint = _str_setting(doc.get('token_endpoint')) + doc_device_endpoint = _str_setting( + doc.get('device_authorization_endpoint')) device_authorization_endpoint = ( - device_authorization_endpoint - or _str_setting(doc.get('device_authorization_endpoint'))) - token_endpoint = ( - token_endpoint or _str_setting(doc.get('token_endpoint'))) - # OIDC Discovery §4.3 / RFC 8414 §3: when pinned ONLY by discovery_url, - # the document's self-declared issuer (the anchor - # validate_endpoint_origins would use) comes from that same untrusted - # document, so it's a vacuous check. Anchor to the caller-pinned - # discovery_url instead: require the credential endpoints on its origin - # so the document can't redirect the POSTs off it. Origin-level; pass - # issuer= and explicit endpoints if discovery and tokens differ in origin. - if discovery_url and not issuer: - discovery_origin = _normalized_origin(discovery_url) - for label, url in ( - ('token endpoint', token_endpoint), - ('device-authorization endpoint', - device_authorization_endpoint)): - if url and _normalized_origin(url) != discovery_origin: - raise OidcConfigError( - f'The OIDC {label} ({url!r}) discovered via the pinned ' - f'discovery_url ({discovery_url!r}) is on a different ' - 'origin; refusing to let a discovery document redirect ' - 'credentials off the pinned IdP origin (OIDC Discovery ' - '§4.3). Pin the IdP with issuer="https://your-idp" and ' - 'pass token_endpoint=/device_authorization_endpoint= ' - 'explicitly if it serves discovery and tokens from ' - 'different origins.') - issuer = issuer or _str_setting(doc.get('issuer')) + device_authorization_endpoint or doc_device_endpoint) + token_endpoint = token_endpoint or doc_token_endpoint if not token_endpoint: raise OidcConfigError( @@ -543,9 +488,44 @@ def resolve_config( 'device grant, or pass device_authorization_endpoint=... ' 'explicitly.') - # The credential-endpoint origin check (validate_endpoint_origins) is + # Pin /settings-sourced credential endpoints to the out-of-band issuer's + # ORIGIN. /settings is untrusted (a tampered or MITM'd response can advertise + # an attacker endpoint), so an endpoint it supplies must resolve to the + # pinned issuer's origin — UNLESS the IdP's own (authoritative, TLS-fetched) + # discovery document advertised that very URL, which independently confirms + # it. Caller-explicit and IdP-discovered endpoints are authoritative and skip + # this: the issuer is an OIDC *identifier*, not necessarily the endpoints' + # host (Google issues from accounts.google.com but serves tokens from + # oauth2.googleapis.com), so pinning them to the issuer origin would reject a + # legitimate cross-origin IdP. The issuer-PATH check above and the + # co-location check in OidcDeviceAuth.__init__ still apply. + if issuer: + issuer_origin = _normalized_origin(issuer) + for label, url, from_settings, confirmed_by_idp in ( + ('token endpoint', token_endpoint, token_from_settings, + doc_token_endpoint), + ('device-authorization endpoint', + device_authorization_endpoint, device_from_settings, + doc_device_endpoint)): + if (from_settings + and _normalized_origin(url) != issuer_origin + and url != confirmed_by_idp): + raise OidcConfigError( + f'The OIDC {label} advertised by QuestDB /settings ' + f'({url!r}) is not on the pinned issuer origin ' + f'({_origin_str(issuer)}) and was not confirmed by the IdP ' + 'discovery document; refusing to send credentials to an ' + 'endpoint outside the trusted issuer. If your IdP serves ' + 'tokens from a different origin than its issuer, pass the ' + 'endpoint(s) explicitly (token_endpoint=..., ' + 'device_authorization_endpoint=...), or omit them from ' + '/settings so they are taken from authoritative IdP ' + 'discovery.') + + # The credential-endpoint CO-LOCATION check (validate_endpoint_origins) is # enforced centrally in OidcDeviceAuth.__init__, which every path goes - # through. + # through; the issuer-ORIGIN pin for /settings-sourced endpoints is enforced + # just above, where each endpoint's provenance is known. return OidcConfig( client_id=client_id, diff --git a/test/test_auth.py b/test/test_auth.py index 79f1b7db..7319bff4 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1447,49 +1447,6 @@ def test_device_fallback_without_issuer_is_rejected(self): OidcDeviceAuth.from_questdb(self.base, insecure=True) self.assertIn('issuer', str(cm.exception)) - def test_device_fallback_with_discovery_url_is_accepted(self): - # discovery_url= is an out-of-band pin too, accepted in lieu of issuer=. - self.state.settings = {'config': { - 'acl.oidc.enabled': True, - 'acl.oidc.client.id': 'questdb', - 'acl.oidc.token.endpoint': self.base + '/token', - }} - self.state.well_known = { - 'issuer': self.base, - 'token_endpoint': self.base + '/token', - 'device_authorization_endpoint': self.base + '/device', - } - auth = OidcDeviceAuth.from_questdb( - self.base, - discovery_url=self.base + '/.well-known/openid-configuration', - insecure=True, renderer=Renderer()) - self.assertEqual(auth.config.device_authorization_endpoint, - self.base + '/device') - - def test_discovery_url_rejects_off_origin_issuer_in_doc(self): - # M4: discovery_url= is advertised as an out-of-band pin, but the doc it - # points to could declare an attacker issuer AND endpoints all on one - # (attacker) origin — which passes co-location / issuer-origin vacuously. - # The discovered issuer must share the pinned discovery_url origin (OIDC - # Discovery §4.3), else refuse. /settings advertises NO endpoints, so - # both come from the (hostile) doc — the exact gap the fix closes. - self.state.settings = {'config': { - 'acl.oidc.enabled': True, - 'acl.oidc.client.id': 'questdb', - }} - self.state.well_known = { - 'issuer': 'https://attacker.example.net', - 'token_endpoint': 'https://attacker.example.net/token', - 'device_authorization_endpoint': - 'https://attacker.example.net/device', - } - with self.assertRaises(OidcConfigError) as cm: - OidcDeviceAuth.from_questdb( - self.base, - discovery_url=self.base + '/.well-known/openid-configuration', - insecure=True) - self.assertIn('origin', str(cm.exception).lower()) - def test_oidc_disabled_raises(self): self.state.settings = {'config': {'acl.oidc.enabled': False}} with self.assertRaises(OidcConfigError): @@ -1580,6 +1537,67 @@ def test_issuer_pin_accepts_matching_origin(self): self.assertEqual(auth.config.device_authorization_endpoint, self.base + '/device') + def test_settings_endpoint_off_issuer_origin_confirmed_by_discovery(self): + # Google-style IdP: /settings advertises the token endpoint on a + # DIFFERENT origin than the pinned issuer, and the device endpoint is + # discovered. The IdP's own .well-known (fetched from the pinned issuer) + # advertises the SAME off-origin token endpoint, which authoritatively + # confirms it — so it is accepted despite not being on the issuer origin. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': 'https://oauth2.idp.example/token', + }} + self.state.well_known = { + 'issuer': self.base, + 'token_endpoint': 'https://oauth2.idp.example/token', + 'device_authorization_endpoint': + 'https://oauth2.idp.example/device', + } + auth = OidcDeviceAuth.from_questdb( + self.base, issuer=self.base, insecure=True, renderer=Renderer()) + self.assertEqual(auth.config.token_endpoint, + 'https://oauth2.idp.example/token') + self.assertEqual(auth.config.device_authorization_endpoint, + 'https://oauth2.idp.example/device') + + def test_settings_off_origin_token_not_confirmed_by_discovery_rejected(self): + # The flip side of the confirmed case: /settings advertises an + # off-issuer-origin token endpoint that the IdP discovery document does + # NOT match (a tampered redirect). It is rejected — the device endpoint + # is discovered on the SAME (attacker) origin only so co-location passes + # and the issuer-origin pin is what does the rejecting. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': 'https://attacker.example/token', + }} + self.state.well_known = { + 'issuer': self.base, + 'token_endpoint': 'https://oauth2.idp.example/token', + 'device_authorization_endpoint': 'https://attacker.example/device', + } + with self.assertRaises(OidcConfigError) as cm: + OidcDeviceAuth.from_questdb(self.base, issuer=self.base, + insecure=True) + self.assertIn('issuer', str(cm.exception).lower()) + + def test_settings_both_endpoints_off_issuer_origin_rejected(self): + # When /settings advertises BOTH credential endpoints off the issuer + # origin, there is no IdP discovery round-trip to confirm them (both are + # present), so they cannot be trusted on the untrusted /settings channel + # -> reject. Pass them explicitly, or omit one so discovery confirms it. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': 'https://oauth2.idp.example/token', + 'acl.oidc.device.authorization.endpoint': + 'https://oauth2.idp.example/device', + }} + with self.assertRaises(OidcConfigError): + OidcDeviceAuth.from_questdb(self.base, issuer=self.base, + insecure=True) + def test_well_known_404_raises_oidc_error(self): # issuer pinned (so the IdP fallback is allowed), but the .well-known # document 404s: get_json maps the non-2xx to OidcError rather than a @@ -1613,8 +1631,8 @@ class TestInsecureSettingsGuard(unittest.TestCase): M1: a /settings response fetched over plaintext http to a non-loopback host (only reachable with insecure=True) is MITM-able, so IdP endpoints it advertises must not be trusted to route the device code / refresh token - without an out-of-band issuer/discovery_url pin — even when BOTH endpoints - are present (so the co-location check would otherwise pass trivially). + without an out-of-band issuer pin — even when BOTH endpoints are present (so + the co-location check would otherwise pass trivially). """ _TAMPERED = { @@ -1678,37 +1696,6 @@ def test_pin_satisfies_guard_over_plaintext(self): insecure=True, issuer='https://evil.example.com') self.assertEqual(cfg.token_endpoint, 'https://evil.example.com/token') - def test_discovery_url_pin_scopes_settings_endpoints(self): - # C1: the plaintext guard accepts discovery_url= as a sufficient pin, but - # when /settings advertises BOTH credential endpoints the IdP-discovery - # block (the only other discovery_url enforcement) is skipped. A - # discovery_url pin must still constrain settings-supplied endpoints to - # its origin, else a tampered plaintext /settings routes credentials to - # an attacker the pin was meant to forbid. IdP discovery must NOT run. - with self.assertRaises(OidcConfigError) as cm: - self._resolve( - self._TAMPERED, # both endpoints on evil.example.com - questdb_url='http://qdb.internal.example:9000', insecure=True, - discovery_url='https://idp.example.com/.well-known/' - 'openid-configuration') - self.assertIn('discovery_url', str(cm.exception)) - - def test_discovery_url_pin_accepts_on_origin_settings_endpoints(self): - # The legitimate case: /settings endpoints on the pinned discovery_url - # origin are accepted (origin-level, so a different path is fine), with - # no IdP round-trip since both endpoints are present. - good = { - 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', - 'acl.oidc.token.endpoint': 'https://idp.example.com/oauth/token', - 'acl.oidc.device.authorization.endpoint': - 'https://idp.example.com/oauth/device'} - cfg = self._resolve( - good, questdb_url='http://qdb.internal.example:9000', insecure=True, - discovery_url='https://idp.example.com/.well-known/' - 'openid-configuration') - self.assertEqual(cfg.token_endpoint, - 'https://idp.example.com/oauth/token') - def test_issuer_path_scopes_settings_endpoints(self): # M1: a tampered /settings advertising a DIFFERENT realm's endpoints on # the SAME host (Keycloak path-based multi-tenancy) is rejected when the @@ -2179,14 +2166,13 @@ def from_discovery(well_known, **kw): {'device_authorization_endpoint': ['nope'], 'token_endpoint': 'https://idp.example.com/token'}, issuer='https://idp.example.com') - # A non-string discovered issuer is dropped (no pin); valid endpoints - # still resolve and the cache key builds (the former crash site). + # Valid discovered endpoints still resolve and the cache key builds + # (the former non-string-field crash site). auth = from_discovery( {'device_authorization_endpoint': 'https://idp.example.com/device', - 'token_endpoint': 'https://idp.example.com/token', - 'issuer': ['not', 'a', 'string']}, - discovery_url='https://idp.example.com/.well-known/openid-configuration') - self.assertIsNone(auth.config.issuer) + 'token_endpoint': 'https://idp.example.com/token'}, + issuer='https://idp.example.com') + self.assertEqual(auth.config.issuer, 'https://idp.example.com') self.assertTrue(auth.cache_key) def test_settings_config_nesting(self): @@ -2245,12 +2231,31 @@ def test_off_origin_device_rejected(self): with self.assertRaises(OidcConfigError): self._validate('https://idp/token', 'https://evil.example/device') - def test_both_endpoints_off_issuer_rejected(self): - # Endpoints agree with each other but not with the pinned issuer: - # the issuer-pin loop must check both, not just their consistency. - with self.assertRaises(OidcConfigError): - self._validate('https://idp/token', 'https://idp/device', - issuer='https://other-issuer.example') + def test_co_located_endpoints_accepted(self): + # validate_endpoint_origins now enforces ONLY co-location: the two + # credential endpoints must share an origin. The issuer-ORIGIN pin for + # /settings-sourced endpoints moved to resolve_config (where each + # endpoint's provenance is known), so a caller/discovery endpoint set + # whose origin differs from the issuer is no longer rejected here — see + # test_issuer_pin_rejects_off_origin_endpoints (/settings rejection) and + # test_explicit_cross_origin_issuer_accepted (Google-style acceptance). + self._validate('https://idp/token', 'https://idp/device') + + def test_explicit_cross_origin_issuer_accepted(self): + # Google-style IdP: the issuer host differs from the endpoint host + # (issues from accounts.google.com, serves tokens from + # oauth2.googleapis.com). Endpoints passed explicitly are authoritative, + # so a cross-origin issuer must NOT be rejected — the issuer is an OIDC + # identifier, not necessarily the endpoints' host. + auth = OidcDeviceAuth( + client_id='questdb', + token_endpoint='https://oauth2.googleapis.com/token', + device_authorization_endpoint= + 'https://oauth2.googleapis.com/device/code', + issuer='https://accounts.google.com', + renderer=Renderer()) + self.assertEqual(auth.config.token_endpoint, + 'https://oauth2.googleapis.com/token') def test_malformed_port_raises_config_error(self): # A non-integer port must surface as OidcConfigError, not urllib's bare From 5f92dcb502ee3caefe64d3a07eba8ba8a7435a19 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 02:55:13 +0100 Subject: [PATCH 065/104] fix(auth): keep the Jupyter QR code visible across re-renders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JupyterRenderer.on_prompt added the QR , but every later re-render (on_waiting/on_success/on_failure) rebuilds the panel via _prompt_head, which didn't include it. on_waiting fires on the first poll tick, so the QR was wiped almost immediately — qr=True was effectively dead in Jupyter. Build the QR inside the shared _prompt_head so it appears on every render, cache the (the PNG is generated once, not per countdown tick), and reset the cache in on_prompt so a re-sign-in rebuilds it for the new code. A dangerous (javascript:/data:) verification URL still produces no QR (the target stays _safe_link_url-gated). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_render.py | 54 ++++++++++++++++++++++++++----------- test/test_auth.py | 42 +++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 16 deletions(-) diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 145e5483..7bea6465 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -247,6 +247,11 @@ def __init__(self, qr: bool = False): self._qr = qr self._handle = None self._resp: Dict[str, Any] = {} + # Cached QR tag. None = not built yet; '' = built but unavailable + # (no scheme-valid target / qrcode not installed). Built once per prompt + # so every re-render (countdown ticks, success/failure) keeps the QR + # instead of dropping it, and the PNG isn't regenerated each tick. + self._qr_html: Optional[str] = None def _display(self, html_str: str): from IPython.display import HTML, display # type: ignore @@ -262,14 +267,15 @@ def _panel(self, body: str) -> str: + body + '') def _prompt_head(self): - """Header + sanitized verification link and user code. - - Shared by :meth:`on_prompt` and :meth:`_render_with_status` so - sanitization is applied on both paths, never forgotten on one. The - untrusted device-response fields are stripped of control/bidi/zero-width - chars (which ``html.escape`` does NOT remove) before rendering; - ``_render_link`` also html-escapes and scheme-vets the URL. Returns - ``(body, uri, complete)`` so the QR target isn't re-derived. + """Header + sanitized verification link, user code, and QR (if enabled). + + Shared by :meth:`on_prompt` and :meth:`_render_with_status` so the QR and + the sanitized fields appear on EVERY render. The countdown re-renders go + through here too, so building the QR only in ``on_prompt`` would drop it + on the first tick. The untrusted device-response fields are stripped of + control/bidi/zero-width chars (which ``html.escape`` does NOT remove) + before rendering; ``_render_link`` also html-escapes and scheme-vets the + URL. Returns ``(body, uri, complete)``. """ resp = self._resp uri = _strip_control(_verification_uri(resp)) @@ -288,18 +294,34 @@ def _prompt_head(self): '
' + _render_link( complete, text='Click here to authorize directly →') + '
') + if self._qr: + qr_html = self._qr_img(complete, uri) + if qr_html: + body.append(qr_html) return body, uri, complete + def _qr_img(self, complete: Optional[str], uri: str) -> str: + """The QR ```` for the verification URL, built once and cached. + + Returns ``''`` when there is no scheme-valid target or ``qrcode`` is not + installed. Generated lazily on the first render and reused thereafter, so + the countdown re-renders neither drop the QR nor regenerate the PNG. + """ + if self._qr_html is None: + target = _safe_link_url(complete) or _safe_link_url(uri) + data_uri = _qr_data_uri(target) if target else None + self._qr_html = ( + f'QR code' + ) if data_uri else '' + return self._qr_html + def on_prompt(self, resp: Dict[str, Any]) -> None: self._resp = resp - body, uri, complete = self._prompt_head() - if self._qr: - qr_target = _safe_link_url(complete) or _safe_link_url(uri) - data_uri = _qr_data_uri(qr_target) if qr_target else None - if data_uri: - body.append( - f'QR code') + # Rebuild the QR for this response (a re-sign-in has a fresh user_code, + # so the cached image from a previous prompt would be stale/wrong). + self._qr_html = None + body, _uri, _complete = self._prompt_head() body.append( '
' '⏳ waiting for authorization…
') diff --git a/test/test_auth.py b/test/test_auth.py index 7319bff4..a6359bd0 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2805,6 +2805,48 @@ def _display(self, html_str): 'expires_in': 600, 'interval': 5}) self.assertNotIn(' is built in the shared _prompt_head, so it survives + # EVERY re-render. on_waiting fires on the first poll tick and used to + # wipe it (the countdown re-render dropped the QR), leaving qr=True + # effectively dead in Jupyter. Stub qrcode so this is deterministic + # whether or not the optional dep is installed. + import re + from questdb.auth._render import JupyterRenderer + renders = [] + + class _Capturing(JupyterRenderer): + def _display(self, html_str): + renders.append(html_str) + + fake_qrcode = types.ModuleType('qrcode') + + def _make(data): + class _Img: + def save(self, buf, format=None): + buf.write(b'\x89PNG' + data.encode()) + return _Img() + fake_qrcode.make = _make + + with mock.patch.dict(sys.modules, {'qrcode': fake_qrcode}): + r = _Capturing(qr=True) + r.on_prompt({ + 'user_code': 'WDJB-MJHT', + 'verification_uri': 'https://idp.example.com/device', + 'verification_uri_complete': + 'https://idp.example.com/device?user_code=WDJB-MJHT', + 'expires_in': 600, 'interval': 5}) + r.on_waiting(120.0) # the first countdown tick — used to drop the QR + r.on_success('alice@example.com', 3600) + + self.assertEqual(len(renders), 3) + self.assertTrue( + all('QR code must persist across on_prompt / on_waiting / on_success') + # The PNG is generated once and reused (same data-URI on every render). + uris = [re.search(r'src="(data:[^"]+)"', h).group(1) for h in renders] + self.assertEqual(len(set(uris)), 1) + def test_fmt_mmss(self): from questdb.auth._render import _fmt_mmss self.assertEqual(_fmt_mmss(0), '0:00') From af721f69685b4958c02a37a10863de4549d28a2d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 09:20:56 +0100 Subject: [PATCH 066/104] fix(auth): apply the issuer-path pin after discovery The issuer-PATH check ran before IdP discovery and lacked the `url == confirmed_by_idp` exemption the issuer-ORIGIN check has, so a /settings endpoint sitting off a path-bearing issuer's path but confirmed verbatim by the IdP's own TLS-fetched .well-known document was wrongly rejected (e.g. Azure AD: issuer .../{tenant}/v2.0, token endpoint .../{tenant}/oauth2/v2.0/token). Fold the path check into the post-discovery origin loop so both pins run after discovery and share one discovery-confirmation exemption. A cross-realm endpoint the IdP document does not confirm is still rejected, so the path pin's multi-tenant protection is preserved. Also add tests for _refresh's non-JSON error arm (a non-JSON 5xx/4xx from a proxy/WAF during a silent refresh), which no test exercised: 5xx -> retryable OidcNetworkError with the refresh token kept; 4xx -> fall back to the device flow. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/auth.rst | 4 +- src/questdb/auth/_discovery.py | 74 +++++++++++------------ test/test_auth.py | 105 +++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 40 deletions(-) diff --git a/docs/auth.rst b/docs/auth.rst index 0c1dc923..5b452789 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -238,8 +238,8 @@ Security notes endpoints' host — e.g. Google issues from ``accounts.google.com`` but serves tokens from ``oauth2.googleapis.com``, and some IdPs such as Azure AD place endpoints outside the issuer path. A ``/settings`` endpoint that sits off the - issuer origin is still accepted when the IdP's own discovery document confirms - the same URL.) When the server does not advertise the device- + issuer origin **or path** is still accepted when the IdP's own discovery + document confirms the same URL.) When the server does not advertise the device- authorization endpoint (so it must be discovered from the IdP), ``issuer=`` is **required** for exactly this reason — the helper refuses to guess the discovery origin from the server-supplied token endpoint. diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 0f5df6ab..0a4721df 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -417,30 +417,6 @@ def resolve_config( 'explicitly (token_endpoint=..., device_authorization_endpoint=...), ' 'or connect to QuestDB over https so /settings is authenticated.') - # For /settings endpoints with an out-of-band issuer, require each under the - # issuer's PATH, not just its origin: path-based IdPs share one origin per - # tenant (Keycloak https://host/realms/{realm}), so the origin check alone - # can't stop a tampered /settings steering credentials to a different realm. - # The out-of-band issuer can't be forged. Caller-explicit endpoints and those - # from IdP discovery are authoritative and skip this — some IdPs (e.g. Azure - # AD) legitimately place endpoints outside the issuer path. - if issuer: - for label, url, from_settings in ( - ('token endpoint', token_endpoint, - not explicit_token_endpoint), - ('device-authorization endpoint', - device_authorization_endpoint, not explicit_device_endpoint)): - if url and from_settings and not _endpoint_path_under_issuer( - url, issuer): - raise OidcConfigError( - f'The OIDC {label} advertised by QuestDB /settings ' - f'({url!r}) is not under the pinned issuer ({issuer!r}); ' - 'refusing to send credentials to an endpoint outside the ' - 'trusted issuer (e.g. a different realm on the same host). ' - 'If your IdP places endpoints outside the issuer path, pass ' - 'them explicitly (token_endpoint=..., ' - 'device_authorization_endpoint=...).') - # Fall back to IdP discovery when QuestDB doesn't advertise the device # (and/or token) endpoint. This contacts the IdP, so it is held to # https/loopback (insecure=False) regardless of the QuestDB flag. @@ -488,17 +464,28 @@ def resolve_config( 'device grant, or pass device_authorization_endpoint=... ' 'explicitly.') - # Pin /settings-sourced credential endpoints to the out-of-band issuer's - # ORIGIN. /settings is untrusted (a tampered or MITM'd response can advertise - # an attacker endpoint), so an endpoint it supplies must resolve to the - # pinned issuer's origin — UNLESS the IdP's own (authoritative, TLS-fetched) - # discovery document advertised that very URL, which independently confirms - # it. Caller-explicit and IdP-discovered endpoints are authoritative and skip - # this: the issuer is an OIDC *identifier*, not necessarily the endpoints' - # host (Google issues from accounts.google.com but serves tokens from - # oauth2.googleapis.com), so pinning them to the issuer origin would reject a - # legitimate cross-origin IdP. The issuer-PATH check above and the - # co-location check in OidcDeviceAuth.__init__ still apply. + # Pin /settings-sourced credential endpoints to the out-of-band issuer. + # /settings is untrusted (a tampered or MITM'd response can advertise an + # attacker endpoint), so an endpoint it supplies must sit BOTH on the pinned + # issuer's ORIGIN and — for a path-based multi-tenant IdP (Keycloak + # https://host/realms/{realm}, where every tenant shares one origin) — under + # the issuer's PATH; the origin check alone can't stop a tampered /settings + # steering credentials to a different realm on the same host. + # + # Both checks are waived for an endpoint the IdP's OWN (authoritative, + # TLS-fetched) discovery document advertised verbatim (url == confirmed_by_idp): + # that confirms it independently of /settings, exactly as trustworthy as the + # pinned IdP. So this runs AFTER discovery — a /settings endpoint the IdP + # confirms is accepted consistently under BOTH pins (the issuer-PATH check + # used to run before discovery and lacked this exemption, so it wrongly + # rejected a discovery-confirmed endpoint sitting off the path of an issuer + # that carries one, e.g. Azure AD's `.../{tenant}/v2.0` issuer). + # Caller-explicit and IdP-discovered endpoints are authoritative and skip + # this entirely (from_settings is False): the issuer is an OIDC *identifier*, + # not necessarily the endpoints' host (Google issues from accounts.google.com + # but serves tokens from oauth2.googleapis.com; Azure AD places endpoints + # outside the issuer path), so pinning them would reject a legitimate IdP. + # The co-location check in OidcDeviceAuth.__init__ still applies on top. if issuer: issuer_origin = _normalized_origin(issuer) for label, url, from_settings, confirmed_by_idp in ( @@ -507,9 +494,10 @@ def resolve_config( ('device-authorization endpoint', device_authorization_endpoint, device_from_settings, doc_device_endpoint)): - if (from_settings - and _normalized_origin(url) != issuer_origin - and url != confirmed_by_idp): + if not from_settings or url == confirmed_by_idp: + # Caller-explicit / IdP-discovered / discovery-confirmed: trusted. + continue + if _normalized_origin(url) != issuer_origin: raise OidcConfigError( f'The OIDC {label} advertised by QuestDB /settings ' f'({url!r}) is not on the pinned issuer origin ' @@ -521,6 +509,16 @@ def resolve_config( 'device_authorization_endpoint=...), or omit them from ' '/settings so they are taken from authoritative IdP ' 'discovery.') + if not _endpoint_path_under_issuer(url, issuer): + raise OidcConfigError( + f'The OIDC {label} advertised by QuestDB /settings ' + f'({url!r}) is not under the pinned issuer ({issuer!r}) and ' + 'was not confirmed by the IdP discovery document; refusing ' + 'to send credentials to an endpoint outside the trusted ' + 'issuer (e.g. a different realm on the same host). If your ' + 'IdP places endpoints outside the issuer path, pass them ' + 'explicitly (token_endpoint=..., ' + 'device_authorization_endpoint=...).') # The credential-endpoint CO-LOCATION check (validate_endpoint_origins) is # enforced centrally in OidcDeviceAuth.__init__, which every path goes diff --git a/test/test_auth.py b/test/test_auth.py index a6359bd0..f0697743 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1323,6 +1323,52 @@ def test_refresh_transient_5xx_non_interactive_does_not_hard_fail(self): auth.token() self.assertEqual(self.state.device_requests, 0) + def test_refresh_non_json_5xx_kept_for_retry(self): + # A NON-JSON 5xx during a silent refresh (an HTML error page from a proxy + # in front of the token endpoint) makes post_form RAISE OidcError(status=) + # rather than return a JSON body — exercising _refresh's `except OidcError` + # transient arm, distinct from the status-based arm a JSON 5xx hits (the + # poll loop has the analogous non-JSON coverage; the refresh path did not). + # It must surface as a retryable OidcNetworkError with the refresh token + # kept, NOT re-prompt. M2. + from questdb.auth._device import REFRESH_GRANT + auth = self.make_auth() + self._seed_expired(auth) + real_idp_post = auth._idp_post + + def flaky(url, form): + if form.get('grant_type') == REFRESH_GRANT: + raise OidcError('proxy 503', status=503) + return real_idp_post(url, form) + + auth._idp_post = flaky + with self.assertRaises(OidcNetworkError): + auth.token() + self.assertEqual(self.state.device_requests, 0) # NOT re-prompted + self.assertEqual(auth._tokens.refresh_token, 'REFRESH-1') # kept + + def test_refresh_non_json_4xx_falls_back_to_device_flow(self): + # A NON-JSON 4xx during a silent refresh (an HTML/plain rejection from a + # WAF/proxy) is terminal, not transient: post_form RAISES OidcError(status=) + # and _refresh's `except OidcError` non-transient arm re-raises, so + # _acquire falls through to a fresh interactive sign-in rather than keeping + # the rejected refresh token. The device-code POST and the subsequent poll + # go through the real (JSON) mock IdP, so the flow completes. M2. + from questdb.auth._device import REFRESH_GRANT + auth = self.make_auth() + self._seed_expired(auth) + real_idp_post = auth._idp_post + + def flaky(url, form): + if form.get('grant_type') == REFRESH_GRANT: + raise OidcError('proxy forbidden', status=403) + return real_idp_post(url, form) + + auth._idp_post = flaky + token = auth.token() + self.assertEqual(token, ID_TOKEN) # from the device flow + self.assertEqual(self.state.device_requests, 1) # fell back / re-prompted + class TestDiscovery(AuthTestBase): def test_from_questdb_reads_settings(self): @@ -1598,6 +1644,65 @@ def test_settings_both_endpoints_off_issuer_origin_rejected(self): OidcDeviceAuth.from_questdb(self.base, issuer=self.base, insecure=True) + def test_settings_endpoint_off_issuer_path_confirmed_by_discovery(self): + # Regression: a path-bearing issuer (Azure-AD-style `.../{tenant}/v2.0`) + # whose token endpoint sits OFF the issuer PATH but on its origin, + # advertised by /settings and CONFIRMED verbatim by the IdP's own + # (TLS-fetched) discovery document, must be ACCEPTED — the issuer-PATH + # pin shares the same discovery-confirmation exemption as the issuer- + # ORIGIN pin, and both now run AFTER discovery. The device endpoint is + # absent from /settings, so it is discovered. Before the fix the PATH + # check ran before discovery with no exemption and wrongly rejected this. + from questdb.auth import _discovery + issuer = 'https://idp.example.com/tenant/v2.0' + settings = { + 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', + # off the issuer PATH (/tenant/v2.0), but on the issuer ORIGIN: + 'acl.oidc.token.endpoint': + 'https://idp.example.com/tenant/oauth2/v2.0/token'} + well_known = { + 'issuer': issuer, + 'token_endpoint': + 'https://idp.example.com/tenant/oauth2/v2.0/token', + 'device_authorization_endpoint': + 'https://idp.example.com/tenant/oauth2/v2.0/devicecode'} + with mock.patch.object(_discovery, 'fetch_settings', + return_value=settings), \ + mock.patch.object(_discovery, 'discover_device_endpoint_from_idp', + return_value=well_known): + cfg = _discovery.resolve_config( + questdb_url='https://qdb.example.com:9000', issuer=issuer) + self.assertEqual(cfg.token_endpoint, + 'https://idp.example.com/tenant/oauth2/v2.0/token') + self.assertEqual(cfg.device_authorization_endpoint, + 'https://idp.example.com/tenant/oauth2/v2.0/devicecode') + + def test_settings_endpoint_off_issuer_path_not_confirmed_rejected(self): + # The flip side: an off-issuer-PATH /settings token endpoint (a tampered + # /settings steering credentials to a different realm on the same host) + # that the IdP discovery document does NOT confirm stays REJECTED — the + # exemption only lifts the pin for the exact URL the IdP itself advertised. + from questdb.auth import _discovery + issuer = 'https://idp.example.com/realms/prod' + settings = { + 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': + 'https://idp.example.com/realms/EVIL/token'} + well_known = { + 'issuer': issuer, + 'token_endpoint': # the IdP advertises a DIFFERENT token endpoint + 'https://idp.example.com/realms/prod/token', + 'device_authorization_endpoint': + 'https://idp.example.com/realms/prod/device'} + with mock.patch.object(_discovery, 'fetch_settings', + return_value=settings), \ + mock.patch.object(_discovery, 'discover_device_endpoint_from_idp', + return_value=well_known): + with self.assertRaises(OidcConfigError) as cm: + _discovery.resolve_config( + questdb_url='https://qdb.example.com:9000', issuer=issuer) + self.assertIn('issuer', str(cm.exception).lower()) + def test_well_known_404_raises_oidc_error(self): # issuer pinned (so the IdP fallback is allowed), but the .well-known # document 404s: get_json maps the non-2xx to OidcError rather than a From 899304d50fc30b8f4022203a0bdeb0256b6bfc93 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 13:27:17 +0100 Subject: [PATCH 067/104] fix(auth): harden the prompt renderer and error sanitizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Jupyter: reset the display handle in on_prompt so a second sign-in on the same renderer (after clear() then token()) opens a fresh display in the current cell instead of .update()-ing the previous one. - Terminal: scheme-vet the QR target via _safe_link_url before encoding, mirroring the Jupyter QR, so a javascript:/data: verification_uri from a hostile device response is never turned into a scannable QR. - _fmt_mmss: guard a non-finite countdown (inf/nan) that would make int() raise; degrade to 0:00 (defense-in-depth — callers pass finite values). - OidcError: coerce a non-string positional arg through str() before stripping, so its text representation is sanitized too (no raise site passes one today — defense-in-depth). - Correct the lock-free fast-path comment: the self._tokens read is race-tolerant, not race-free, on a free-threaded build. Adds renderer/error tests for each. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 9 ++- src/questdb/auth/_errors.py | 7 ++- src/questdb/auth/_render.py | 22 ++++++-- test/test_auth.py | 109 ++++++++++++++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 9 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 916672fd..e23028e2 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -515,8 +515,13 @@ def _obtain_tokens(self) -> TokenSet: # Fast path: return a valid token without the lock, so a caller with a # usable token never blocks behind another thread's refresh/sign-in. # READ-ONLY — never writes self._tokens; every write to that field is - # under the lock (the promotion below, _store, clear), so this lock-free - # reader can't race a write or resurrect a just-cleared token. + # under the lock (the promotion below, _store, clear). On the GIL build + # the single-reference read is atomic and ordered, so it can't see a torn + # value or race a write. On a free-threaded build the read is + # intentionally race-TOLERANT, not race-free: a stale None falls through + # to the locked slow path, and a stale-but-valid (frozen) TokenSet is + # returned once — never a torn or wrong-context one — and the read writes + # nothing, so it can't resurrect a cleared entry in the shared cache. tokens = self._valid_cached() if tokens is not None: return tokens diff --git a/src/questdb/auth/_errors.py b/src/questdb/auth/_errors.py index 841e40e7..70d0c5b1 100644 --- a/src/questdb/auth/_errors.py +++ b/src/questdb/auth/_errors.py @@ -44,10 +44,11 @@ def __init__(self, *args, status: Optional[int] = None): # ANSI — is a sink the renderer's own sanitization never sees. Without # this, a hostile or MITM'd IdP could inject ANSI escapes or a bidi # override into that traceback to spoof the prompt. Doing it here (not at - # each raise site) means no raise site can forget; non-string args (rare) - # pass through unchanged. + # each raise site) means no raise site can forget. A non-string arg is + # coerced through str() so its text representation is sanitized too (no + # raise site passes one today — this is defense-in-depth). args = tuple( - _strip_control(a) if isinstance(a, str) else a for a in args) + _strip_control(a if isinstance(a, str) else str(a)) for a in args) super().__init__(*args) # HTTP status behind a non-JSON HTTP response (else None), so the poll # loop and silent refresh can tell a terminal 4xx (e.g. a WAF error diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 7bea6465..508350d6 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -33,6 +33,7 @@ from __future__ import annotations import html +import math import sys import unicodedata import urllib.parse @@ -169,6 +170,11 @@ def format_prompt(resp: Dict[str, Any]) -> str: def _fmt_mmss(seconds: float) -> str: + # A non-finite input (inf/nan) would make int() raise (OverflowError / + # ValueError); treat it as 0 — it can't be a real countdown. Callers pass a + # clamped, finite remaining time today, so this is defense-in-depth. + if not math.isfinite(seconds): + seconds = 0 seconds = max(0, int(seconds)) return f'{seconds // 60}:{seconds % 60:02d}' @@ -216,8 +222,12 @@ def _write(self, text: str) -> None: def on_prompt(self, resp: Dict[str, Any]) -> None: self._write(format_prompt(resp) + '\n') if self._qr: - target = _verification_uri_complete(resp) or _verification_uri(resp) - art = _qr_ascii(target) + # Scheme-vet the target before encoding it, mirroring the Jupyter QR + # (_qr_img): never turn a javascript:/data: verification_uri from a + # hostile device response into a scannable QR. + target = (_safe_link_url(_verification_uri_complete(resp)) + or _safe_link_url(_verification_uri(resp))) + art = _qr_ascii(target) if target else None if art: self._write(art + '\n') @@ -318,8 +328,12 @@ def _qr_img(self, complete: Optional[str], uri: str) -> str: def on_prompt(self, resp: Dict[str, Any]) -> None: self._resp = resp - # Rebuild the QR for this response (a re-sign-in has a fresh user_code, - # so the cached image from a previous prompt would be stale/wrong). + # Start a FRESH display for this sign-in. Without resetting the handle, a + # second sign-in on the same renderer (e.g. after clear() then token()) + # would .update() the previous sign-in's output area instead of the cell + # the user just ran. Rebuild the QR too (a re-sign-in has a fresh + # user_code, so the cached image from a previous prompt would be stale). + self._handle = None self._qr_html = None body, _uri, _complete = self._prompt_head() body.append( diff --git a/test/test_auth.py b/test/test_auth.py index f0697743..dabd544b 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2961,6 +2961,100 @@ def test_fmt_mmss(self): self.assertEqual(_fmt_mmss(-5), '0:00') # clamped, never negative self.assertEqual(_fmt_mmss(125.9), '2:05') # truncates seconds + def test_fmt_mmss_handles_non_finite(self): + # A non-finite remaining time (inf/nan) must not crash _fmt_mmss with an + # OverflowError/ValueError from int(); it degrades to 0:00. Unreachable in + # practice (callers clamp to a finite value) — defense-in-depth. M5. + from questdb.auth._render import _fmt_mmss + self.assertEqual(_fmt_mmss(float('inf')), '0:00') + self.assertEqual(_fmt_mmss(float('nan')), '0:00') + self.assertEqual(_fmt_mmss(float('-inf')), '0:00') + + def test_jupyter_second_signin_creates_new_display(self): + # A second sign-in on the SAME renderer (e.g. after clear() then token()) + # must create a FRESH display in the current cell, not .update() the + # previous sign-in's output area: on_prompt resets the display handle. M3. + from questdb.auth._render import JupyterRenderer + events = [] + + class _Cap(JupyterRenderer): + def _display(self, html_str): + # Mimic IPython: create when handle is None, else update in place. + if self._handle is None: + self._handle = object() + events.append('create') + else: + events.append('update') + + r = _Cap() + resp = {'user_code': 'A', 'verification_uri': 'https://idp/d', + 'expires_in': 600, 'interval': 5} + r.on_prompt(resp) # first sign-in -> create + r.on_waiting(120.0) # same sign-in -> update in place + r.on_success('alice', 3600) # same sign-in -> update in place + r.on_prompt(resp) # SECOND sign-in -> must create afresh + self.assertEqual(events, ['create', 'update', 'update', 'create']) + + def test_terminal_qr_suppressed_for_dangerous_url(self): + # With qr=True, a dangerous (javascript:/data:) verification URL must NOT + # be encoded into a terminal QR: the target is scheme-vetted via + # _safe_link_url before qrcode is ever called, mirroring the Jupyter QR. + # Holds whether or not the optional qrcode dep is installed. M4. + import io + from questdb.auth._render import TerminalRenderer + invoked = {'n': 0} + fake_qrcode = types.ModuleType('qrcode') + + class _QR: + def __init__(self, *a, **k): + invoked['n'] += 1 + + def add_data(self, *a): + pass + + def make(self, *a, **k): + pass + + def print_ascii(self, *a, **k): + pass + + fake_qrcode.QRCode = _QR + with mock.patch.dict(sys.modules, {'qrcode': fake_qrcode}): + TerminalRenderer(stream=io.StringIO(), qr=True).on_prompt({ + 'user_code': 'X', 'verification_uri': 'javascript:alert(1)', + 'expires_in': 600, 'interval': 5}) + self.assertEqual(invoked['n'], 0) # qrcode never reached for a bad URL + + def test_terminal_qr_rendered_for_safe_url(self): + # The flip side: a legitimate https verification URL with qr=True IS + # encoded and written to the terminal — the scheme gate must not + # over-block the happy path. Stub qrcode so this is deterministic. M4. + import io + from questdb.auth._render import TerminalRenderer + fake_qrcode = types.ModuleType('qrcode') + + class _QR: + def __init__(self, *a, **k): + pass + + def add_data(self, *a): + pass + + def make(self, *a, **k): + pass + + def print_ascii(self, *a, out=None, **k): + out.write('QR-ART') + + fake_qrcode.QRCode = _QR + buf = io.StringIO() + with mock.patch.dict(sys.modules, {'qrcode': fake_qrcode}): + TerminalRenderer(stream=buf, qr=True).on_prompt({ + 'user_code': 'X', + 'verification_uri': 'https://idp.example.com/device', + 'expires_in': 600, 'interval': 5}) + self.assertIn('QR-ART', buf.getvalue()) + def test_detect_interactive_requires_tty(self): # Outside a notebook kernel, interactivity requires both stdin AND stdout # to be a TTY (guards against hanging in papermill / cron / CI). @@ -3128,6 +3222,21 @@ def test_oidc_error_sanitizes_message_and_fields(self): self.assertIsNone(d2.error) self.assertIsNone(d2.error_description) + def test_oidc_error_sanitizes_non_string_arg(self): + # A non-string positional arg is coerced through str() and sanitized too, + # so an object whose text representation embeds ANSI/bidi can't leak it + # into a traceback. No raise site passes one today — defense-in-depth. M6. + class _Evil: + def __str__(self): + return 'boom \x1b[31m' + chr(0x202e) + 'spoof\x07' + + e = OidcError(_Evil()) + self.assertNotIn('\x1b', str(e)) + self.assertNotIn('\x07', str(e)) + self.assertNotIn(chr(0x202e), str(e)) + self.assertIn('boom', str(e)) + self.assertIn('spoof', str(e)) + def test_qr_helpers_degrade_without_qrcode(self): # The QR helpers must degrade gracefully (return None), never raise, # when `qrcode` is absent or the data is empty. See M4. From a91c1f299171e667f1bd25b5cab77ec7d54367d5 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 13:45:17 +0100 Subject: [PATCH 068/104] fix(auth): bound the token-cache maps; fix the /settings URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M8 — the process-global _MEMORY_GENERATION map retained one entry per distinct cache key forever (clear() created an entry that was never reclaimed), a slow leak for a process cycling through many IdP configs. The counter's monotonicity is load-bearing for the store_if_current CAS, so entries can't simply be deleted (that risks ABA resurrection). Instead track in-flight acquisitions per key (generation() now increments a count, paired with a new release() in _obtain_tokens' finally) and reclaim a key's generation only when none are in flight; clear() likewise retains the bumped generation only while an acquisition is in flight, dropping it otherwise. A concurrent clear() during an in-flight sign-in is still honored (its store_if_current is dropped) — verified by test. The maps now stay empty across many distinct-key acquire/clear cycles. M9 — fetch_settings appended "/settings" by string concatenation, so a base URL carrying a query/fragment produced a malformed ".../?x=1/settings". Build it on the URL path via urlunparse instead, dropping query/fragment. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_cache.py | 43 +++++++++++++++++++++++-- src/questdb/auth/_device.py | 29 ++++++++++------- src/questdb/auth/_discovery.py | 14 ++++++-- test/test_auth.py | 59 +++++++++++++++++++++++++++++++++- 4 files changed, 128 insertions(+), 17 deletions(-) diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index c2c0e281..04cf9a61 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -78,6 +78,11 @@ def is_valid(self, now: float, skew: float = DEFAULT_SKEW_SECONDS) -> bool: # clear() on a different OidcDeviceAuth sharing this store, whose per-instance # lock doesn't serialize against this one — so clear() can't be silently undone. _MEMORY_GENERATION: Dict[str, int] = {} +# Count of in-flight acquisitions per key (a generation() capture not yet +# released). While > 0, a concurrent clear() retains the bumped generation so +# the in-flight store_if_current is still dropped; once it falls back to 0 the +# generation entry is reclaimed, bounding the maps' growth (see release()). +_MEMORY_INFLIGHT: Dict[str, int] = {} _MEMORY_LOCK = threading.Lock() @@ -102,7 +107,17 @@ def store(self, key: str, tokens: TokenSet) -> None: def clear(self, key: str) -> None: with _MEMORY_LOCK: _MEMORY_STORE.pop(key, None) - _MEMORY_GENERATION[key] = _MEMORY_GENERATION.get(key, 0) + 1 + # The clear()-generation only needs to outlive an IN-FLIGHT + # acquisition, so that acquisition's store_if_current (which captured + # the pre-clear value) is dropped. With none in flight there is no + # stale capturer to defend against, so drop the entry rather than + # retain one per cleared key forever: the slow path is the only + # writer and always captures a fresh generation, so a cleared token + # can't be silently resurrected. + if _MEMORY_INFLIGHT.get(key, 0) > 0: + _MEMORY_GENERATION[key] = _MEMORY_GENERATION.get(key, 0) + 1 + else: + _MEMORY_GENERATION.pop(key, None) def evict(self, key: str) -> None: """ @@ -121,14 +136,38 @@ def evict(self, key: str) -> None: def generation(self, key: str) -> int: """ - Current clear()-generation for ``key``. + Current clear()-generation for ``key``; marks an acquisition in flight. Capture before an IdP round-trip and pass to :meth:`store_if_current`, which drops the write if a ``clear()`` bumped the counter meanwhile. + Every call MUST be paired with a :meth:`release` (the caller does so in + a ``finally``) so the per-key generation can be reclaimed once no + acquisition is in flight for the key. """ with _MEMORY_LOCK: + _MEMORY_INFLIGHT[key] = _MEMORY_INFLIGHT.get(key, 0) + 1 return _MEMORY_GENERATION.get(key, 0) + def release(self, key: str) -> None: + """ + End the acquisition a :meth:`generation` capture began. + + When the last in-flight acquisition for ``key`` finishes, the per-key + clear()-generation is reclaimed: with no acquisition holding a captured + (possibly stale) value there is nothing left to compare against, so + retaining it would only grow the process-global maps by one entry per + distinct cache key. A later acquisition captures a fresh 0 and a later + ``clear()`` re-establishes a monotonic sequence with no stale capturer + to race, so reclaiming it can't resurrect a write a ``clear()`` dropped. + """ + with _MEMORY_LOCK: + remaining = _MEMORY_INFLIGHT.get(key, 0) - 1 + if remaining > 0: + _MEMORY_INFLIGHT[key] = remaining + else: + _MEMORY_INFLIGHT.pop(key, None) + _MEMORY_GENERATION.pop(key, None) + def store_if_current( self, key: str, tokens: TokenSet, generation: int) -> bool: """ diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index e23028e2..3aedf1a9 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -533,19 +533,24 @@ def _obtain_tokens(self) -> TokenSet: # clear() — including on another instance sharing the process-global # MemoryCache (whose per-instance lock doesn't serialize against # ours) — invalidates the store below instead of resurrecting the - # cleared entry. + # cleared entry. Paired with release() in the finally so the cache + # reclaims the per-key generation once no acquisition is in flight + # for it (bounds the process-global maps; see MemoryCache.release). generation = self._cache_generation() - # Promote a cached token under the lock (even expired, so _acquire - # can reuse its refresh_token). Here, not on the fast path, so every - # write to self._tokens stays serialized. - if self._tokens is None: - cached = self._cache.load(self.cache_key) - if cached is not None: - self._tokens = cached - tokens = self._valid_cached() - if tokens is not None: - return tokens - return self._acquire(generation) + try: + # Promote a cached token under the lock (even expired, so + # _acquire can reuse its refresh_token). Here, not on the fast + # path, so every write to self._tokens stays serialized. + if self._tokens is None: + cached = self._cache.load(self.cache_key) + if cached is not None: + self._tokens = cached + tokens = self._valid_cached() + if tokens is not None: + return tokens + return self._acquire(generation) + finally: + self._cache.release(self.cache_key) def _valid_cached(self) -> Optional[TokenSet]: # Read-only: reads the published field, falling back to the shared cache diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 0a4721df..649f46c4 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -118,6 +118,17 @@ def settings_config(settings: Any) -> Dict[str, Any]: return settings +def _settings_url(questdb_url: str) -> str: + # Build the /settings endpoint on the base URL's PATH, dropping any query or + # fragment, so a base like "https://host:9000/?x=1" can't yield a malformed + # ".../?x=1/settings". The path is rstrip('/')-ed to avoid a double slash. + # safe_urlparse maps a malformed URL to OidcConfigError, not a bare ValueError. + parts, _ = safe_urlparse(questdb_url) + path = (parts.path or '').rstrip('/') + '/settings' + return urllib.parse.urlunparse( + (parts.scheme, parts.netloc, path, '', '', '')) + + def fetch_settings( questdb_url: str, *, @@ -125,8 +136,7 @@ def fetch_settings( insecure: bool = False, timeout: float = 30) -> Dict[str, Any]: """Fetch and return the QuestDB ``/settings`` config map.""" - base = questdb_url.rstrip('/') - data = get_json(base + '/settings', ctx=ctx, insecure=insecure, + data = get_json(_settings_url(questdb_url), ctx=ctx, insecure=insecure, timeout=timeout) return settings_config(data) diff --git a/test/test_auth.py b/test/test_auth.py index dabd544b..cecc4097 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -64,7 +64,7 @@ psycopg_connect, ) from questdb.auth._cache import ( # noqa: E402 - MemoryCache, _MEMORY_GENERATION, _MEMORY_STORE) + MemoryCache, _MEMORY_GENERATION, _MEMORY_INFLIGHT, _MEMORY_STORE) from questdb.auth._render import Renderer # noqa: E402 from questdb.auth._adapters import _require_host # noqa: E402 @@ -281,6 +281,7 @@ class AuthTestBase(unittest.TestCase): def setUp(self): _MEMORY_STORE.clear() _MEMORY_GENERATION.clear() + _MEMORY_INFLIGHT.clear() # open_browser defaults to True, so stub webbrowser.open: device-flow # tests must never spawn a real browser. Tests asserting open/skip # behaviour use self.mock_browser_open (or patch it themselves). @@ -1975,6 +1976,46 @@ def test_store_if_current_drops_write_after_concurrent_clear(self): cache.store_if_current(key, TokenSet(access_token='T2'), gen2)) self.assertIsNotNone(cache.load(key)) + def test_generation_pruned_when_no_acquisition_in_flight(self): + # M8: the per-key clear()-generation must not accumulate forever. After a + # clear() and a completed re-acquisition (no acquisition left in flight), + # neither the generation nor the in-flight bookkeeping is retained for + # the key, so the process-global maps stay bounded. + auth = self.make_auth() + auth.token() # sign in (slow path: capture + release) + self.assertNotIn(auth.cache_key, _MEMORY_INFLIGHT) # released + auth.clear() # no acquisition in flight -> drop, no bump + self.assertNotIn(auth.cache_key, _MEMORY_GENERATION) + auth.token() # re-acquire; release() prunes on completion + self.assertNotIn(auth.cache_key, _MEMORY_GENERATION) + self.assertNotIn(auth.cache_key, _MEMORY_INFLIGHT) + + def test_concurrent_clear_retains_generation_until_acquire_done(self): + # The flip side of pruning: while an acquisition IS in flight, a + # concurrent clear() must RETAIN the bumped generation so the in-flight + # store_if_current is still dropped (clear() honored). The entry is + # reclaimed only once that acquisition releases — pruning must not weaken + # this in-flight defense. Strengthens the cross-instance clear() test. + seen = {} + a = self.make_auth() + b = self.make_auth() + self.assertEqual(a.cache_key, b.cache_key) + + class _ClearMidFlow(Renderer): + def on_prompt(self, resp): + b.clear() # concurrent clear during A's in-flight sign-in + # A is mid-acquisition, so the generation is retained here. + seen['gen_present'] = a.cache_key in _MEMORY_GENERATION + seen['inflight'] = _MEMORY_INFLIGHT.get(a.cache_key, 0) + + a._renderer = _ClearMidFlow() + self.assertEqual(a.token(), ID_TOKEN) + self.assertTrue(seen['gen_present']) # retained during the flow + self.assertGreaterEqual(seen['inflight'], 1) + self.assertNotIn(a.cache_key, _MEMORY_STORE) # A's store dropped + self.assertNotIn(a.cache_key, _MEMORY_GENERATION) # reclaimed on release + self.assertNotIn(a.cache_key, _MEMORY_INFLIGHT) + class TestAdapters(unittest.TestCase): """PG-wire connection adapters: tested via injected fake modules (the real @@ -2285,6 +2326,22 @@ def test_settings_config_nesting(self): self.assertEqual(settings_config({'config': {'a': 1}}), {'a': 1}) self.assertEqual(settings_config({'a': 1}), {'a': 1}) # flat fallback + def test_settings_url_drops_query_and_fragment(self): + # M9: the /settings endpoint is built on the QuestDB base URL's PATH, + # dropping any query/fragment, so a base carrying one can't yield a + # malformed ".../?x=1/settings". A trailing slash doesn't double up. + from questdb.auth._discovery import _settings_url + self.assertEqual(_settings_url('https://h:9000'), + 'https://h:9000/settings') + self.assertEqual(_settings_url('https://h:9000/'), + 'https://h:9000/settings') + self.assertEqual(_settings_url('https://h:9000/qdb'), + 'https://h:9000/qdb/settings') + self.assertEqual(_settings_url('https://h:9000/?x=1'), + 'https://h:9000/settings') + self.assertEqual(_settings_url('https://h:9000/base/#frag'), + 'https://h:9000/base/settings') + def test_settings_config_ignores_user_writable_preferences(self): # QuestDB /settings nests server-authoritative values under "config" # alongside a user-writable "preferences" sibling (the web console From 6c41d2c31b27f66806afd2c782e1ed89712b44a6 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 14:23:11 +0100 Subject: [PATCH 069/104] fix(auth): tighten origin/path normalization edge cases - _normalized_origin: `explicit_port or default` collapsed an explicit :0 (falsy) to the default port, aliasing two distinct origins. Compare against None so :0 stays distinct. Not exploitable (:0 isn't connectable); a normalization tidy. - _endpoint_path_under_issuer: a C0 control char (notably NUL from "..%00") survives _strip_matrix_params, so "..\x00" is not literally ".." and slipped the dot-segment traversal check. A NUL-truncating or control-stripping proxy/server could resolve it back to ".." and reach a different realm on the (already origin-pinned) host. Reject any segment carrying a C0 control or DEL (fail closed); printable-ASCII paths with %20 etc. still pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 24 ++++++++++++++++++++++-- test/test_auth.py | 21 +++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 649f46c4..e3c93c65 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -149,7 +149,11 @@ def _normalized_origin(url: str) -> tuple: parts, explicit_port = safe_urlparse(url) scheme = (parts.scheme or '').lower() host = (parts.hostname or '').lower() - port = explicit_port or _DEFAULT_PORTS.get(scheme) + # `explicit_port or default` would collapse an explicit :0 (falsy) to the + # default port; compare against None so :0 stays a distinct (if + # unconnectable) origin rather than aliasing the default. + port = (explicit_port if explicit_port is not None + else _DEFAULT_PORTS.get(scheme)) return (scheme, host, port) @@ -207,6 +211,21 @@ class (Tomcat/Undertow & co. strip path parameters before normalizing the return segment.split(';', 1)[0].strip() +def _has_control_char(segment: str) -> bool: + """ + True if ``segment`` carries a C0 control character (including NUL) or DEL. + + Such a char survives :func:`_strip_matrix_params` (``str.strip`` trims only + the whitespace controls, and only at the ends), so a percent-encoded + ``..%00`` decodes to a segment that is not literally ``..`` and would slip + the dot-segment check below — yet a NUL-truncating or control-stripping + proxy/server can resolve it back to ``..`` and reach a different path. + Legitimate credential-endpoint segments are plain printable ASCII, so any + control char is rejected (fail closed). + """ + return any(ord(ch) < 0x20 or ord(ch) == 0x7f for ch in segment) + + def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: """ True if ``endpoint``'s path is the issuer's path or a sub-path of it. @@ -246,7 +265,8 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: # prefix-match a value that could still resolve to a different path. # Legitimate credential-endpoint paths are plain ASCII with no encoding. if ('.' in ep_segs or '..' in ep_segs - or any('%' in s for s in ep_segs)): + or any('%' in s for s in ep_segs) + or any(_has_control_char(s) for s in ep_segs)): return False return ep_segs[:len(base_segs)] == base_segs diff --git a/test/test_auth.py b/test/test_auth.py index cecc4097..3202303a 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2386,6 +2386,16 @@ def test_default_port_equivalence_accepted(self): # https default (443) vs explicit :443 normalize to the same origin. self._validate('https://idp/token', 'https://idp:443/device') + def test_normalized_origin_keeps_explicit_zero_port(self): + # m6: an explicit :0 must not collapse to the default port (0 is falsy + # but a real, distinct port value), so it stays a distinct origin rather + # than aliasing the default. Not exploitable (:0 isn't connectable) — a + # normalization tidy. + from questdb.auth._discovery import _normalized_origin + self.assertEqual(_normalized_origin('https://h:0/x'), ('https', 'h', 0)) + self.assertNotEqual(_normalized_origin('https://h:0/x'), + _normalized_origin('https://h/x')) + def test_ipv6_same_origin_accepted(self): self._validate('https://[::1]/token', 'https://[::1]/device') @@ -2494,6 +2504,17 @@ def test_endpoint_path_under_issuer(self): # param is still accepted — only dot traversal is rejected. self.assertTrue(under(iss + '/some%20path/token', iss)) self.assertTrue(under(iss + '/token;jsessionid=abc', iss)) + # A NUL (or any other C0 control / DEL) in a segment is rejected (m7): + # it survives _strip_matrix_params (str.strip trims only whitespace + # controls), so "..%00" decodes to '..\x00' (not literally '..') and + # would slip the dot-check — but a NUL-truncating or control-stripping + # proxy/server resolves it back to '..' and reaches a different realm. + self.assertFalse(under(iss + '/..%00/EVIL/token', iss)) # ..NUL + self.assertFalse(under(iss + '/%2e%2e%00/EVIL/token', iss)) # enc ..NUL + self.assertFalse(under(iss + '/..%01/EVIL/token', iss)) # ..C0 + self.assertFalse(under(iss + '/..%7f/EVIL/token', iss)) # ..DEL + # A printable-ASCII segment with an internal space (%20) is still fine. + self.assertTrue(under(iss + '/ok%20name/token', iss)) class TestCacheKey(unittest.TestCase): From 4482b94101f5d207a2979bd7a07e11ed75a86798 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 15:30:36 +0100 Subject: [PATCH 070/104] fix(auth): harden error fields and verification-link safety Two correctness/security fixes from review, plus concurrency coverage. - OidcDeviceFlowError crashed with a raw TypeError when an IdP returned a non-string error/error_description (a JSON object/number/array): _strip_control iterated the value and called unicodedata.category() on it. This escaped the typed-error contract and, on the refresh path, slipped past _acquire's `except OidcError` (the TypeError was raised during exception construction), aborting token() instead of falling back to a fresh sign-in. Coerce the fields through str() like OidcError already does, and make _strip_control total so it never raises on untrusted input. - _safe_link_url only vetted the scheme, so a tampered/MITM'd verification URL with embedded userinfo (https://trusted@evil/) -- or a confusable/control-char host -- became a clickable Jupyter link and was auto-opened via webbrowser.open, navigating to the attacker host while reading as the trusted one. Reject userinfo and non-LDH hosts so such URLs render as inert text only. - Add a real multi-thread stress test hammering token()/clear() under contention (thread-safe clock, free-threaded-ready) and a precondition note that the lock-free fast path relies on TokenSet/cache_key immutability. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 9 ++ src/questdb/auth/_errors.py | 16 ++- src/questdb/auth/_render.py | 54 ++++++++-- test/test_auth.py | 202 ++++++++++++++++++++++++++++++++++++ 4 files changed, 271 insertions(+), 10 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 3aedf1a9..10a63169 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -556,6 +556,15 @@ def _valid_cached(self) -> Optional[TokenSet]: # Read-only: reads the published field, falling back to the shared cache # backend. Never writes self._tokens (that's lock-only), so it's safe on # the lock-free fast path. + # + # PRECONDITION for that lock-free safety (preserve both if refactoring): + # (a) TokenSet is frozen, so a concurrently-published reference is never + # mutated under the reader (no torn read); and + # (b) self.config / self.cache_key are set once in __init__ and never + # reassigned, so a token published by another thread is always for + # THIS instance's security context (never a wrong-context token). + # If either is broken, this read must move under self._lock. Exercised + # under real contention by TestConcurrency.test_token_clear_stress. tokens = self._tokens if tokens is None: tokens = self._cache.load(self.cache_key) diff --git a/src/questdb/auth/_errors.py b/src/questdb/auth/_errors.py index 70d0c5b1..dcb67689 100644 --- a/src/questdb/auth/_errors.py +++ b/src/questdb/auth/_errors.py @@ -91,11 +91,19 @@ def __init__( super().__init__(message) # error / error_description come straight from the untrusted IdP # response and are exposed as attributes (a caller may re-display them), - # so strip them too — same rationale as the message in OidcError. None is - # kept as None (not coerced to '') so "absent" stays distinguishable. - self.error = _strip_control(error) if error is not None else None + # so strip them too — same rationale as the message in OidcError. Coerce + # a non-string (a JSON object/number/array from a buggy or hostile IdP) + # through str() first, exactly as OidcError does for its message args, so + # a non-string field can't crash the strip with a TypeError and escape + # the typed-error contract. None is kept as None (not coerced to '') so + # "absent" stays distinguishable. + self.error = ( + _strip_control(error if isinstance(error, str) else str(error)) + if error is not None else None) self.error_description = ( - _strip_control(error_description) + _strip_control( + error_description if isinstance(error_description, str) + else str(error_description)) if error_description is not None else None) diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 508350d6..a901199a 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -34,6 +34,7 @@ import html import math +import re import sys import unicodedata import urllib.parse @@ -86,23 +87,56 @@ def _verification_uri_complete(resp: Dict[str, Any]) -> Optional[str]: return uri if isinstance(uri, str) else None +# A host safe to make clickable / auto-open: plain ASCII letters-digits-hyphen +# (a DNS name or punycode ``xn--`` label), dots, and the ``:``/``%`` an IPv6 +# literal carries once urlparse has stripped its brackets. Anything else — a +# non-ASCII confusable (e.g. a Cyrillic look-alike, or the fullwidth solidus +# ``U+FF0F``) or a stray control char in the authority — can misrepresent the +# real destination host, so such a URL is never made clickable/auto-opened. +_SAFE_HOST_RE = re.compile(r'\A[a-z0-9._%:-]+\Z') + + def _safe_link_url(url: Optional[str]) -> Optional[str]: """ - Return ``url`` only if it uses an ``http(s)`` scheme, else ``None``. + Return ``url`` only if it is safe to make clickable / auto-open, else + ``None``. The verification URL is untrusted (from the IdP's device-authorization - response); the scheme allowlist blocks a ``javascript:`` / ``data:`` href - from executing in the notebook DOM (``html.escape`` guards markup, not the - scheme). + response). Three checks, all of which a tampered/MITM'd response could + otherwise abuse to send the user somewhere other than the prompt suggests + (``html.escape`` guards markup, not any of these): + + * **scheme** — only ``http(s)``, so a ``javascript:`` / ``data:`` href can't + execute in the notebook DOM; + * **no userinfo** — ``https://login.questdb.io@evil.example/`` connects to + ``evil.example`` while *reading* as the trusted host; the device-flow + verification URL never legitimately carries credentials; + * **plain host** — a host with non-ASCII/confusable or control characters + (a homograph, a fullwidth solidus) can spoof the destination. + + A URL that fails these is still shown as inert, escaped text (visible and + copyable) — it is just never turned into a live link, opened in a browser, + or encoded into a QR. """ if not url or not isinstance(url, str): # A non-string has no scheme to vet and would make urlparse raise. return None try: - scheme = urllib.parse.urlparse(url).scheme.lower() + parts = urllib.parse.urlparse(url) + scheme = (parts.scheme or '').lower() + # `.username`/`.password`/`.hostname` parse the authority; `.port` (read + # indirectly via a malformed netloc) can raise ValueError — catch it. + userinfo = parts.username is not None or parts.password is not None + host = parts.hostname except (ValueError, TypeError): return None - return url if scheme in ('http', 'https') else None + if scheme not in ('http', 'https'): + return None + if userinfo: + return None + if not host or not _SAFE_HOST_RE.match(host): + return None + return url def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: @@ -146,9 +180,17 @@ def _strip_control(text: Optional[str]) -> str: ANSI escapes or bidi/zero-width/line-separator chars could spoof the prompt or hide the real sign-in URL. Needed on both paths — ``html.escape`` does not catch bidi/zero-width spoofing. + + Total by design: a truthy non-``str`` (e.g. a JSON object/number a hostile + IdP put in an ``error`` field) is coerced through ``str()`` rather than + raising. This sanitizer runs on untrusted input from several sites and must + never raise — a ``TypeError`` here would escape the module's typed-error + contract (see :class:`~questdb.auth._errors.OidcDeviceFlowError`). """ if not text: return '' + if not isinstance(text, str): + text = str(text) return ''.join( ch for ch in text if ch not in _STRIP_EXTRA diff --git a/test/test_auth.py b/test/test_auth.py index 3202303a..afcc315c 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -168,6 +168,35 @@ def now(self): return self.wall +class _ConcurrentClock: + """Like FakeClock but safe under real thread contention; sleep is instant. + + The deterministic FakeClock mutates plain attributes, which races when many + threads drive the flow at once (and tears on a free-threaded build). This + guards every read/write with a lock so a multi-thread stress test gets + instant, non-racing time. Only the lock-holding acquirer ever sleeps (the + lock-free fast path never does), so contention on this lock stays low. + """ + + def __init__(self): + self._lock = threading.Lock() + self.mono = 0.0 + self.wall = 1_000_000.0 + + def sleep(self, dt): + with self._lock: + self.mono += dt + self.wall += dt + + def monotonic(self): + with self._lock: + return self.mono + + def now(self): + with self._lock: + return self.wall + + class MockState: """Scriptable behaviour shared with the request handler.""" @@ -539,6 +568,18 @@ def test_idp_expired_token_error_raises_timeout(self): auth.token() self.assertEqual(cm.exception.error, 'expired_token') + def test_non_string_poll_error_field_raises_typed_error(self): + # M1 (end-to-end): a non-conformant/hostile IdP can answer the token poll + # with a non-string error / error_description (a JSON object/array). + # Building the terminal OidcDeviceFlowError from it must surface as a + # TYPED OidcError, never a raw TypeError that escapes token(). + self.state.token_script = [ + (400, {'error': {'nested': 'obj'}, + 'error_description': ['a', 'list']})] + auth = self.make_auth() + with self.assertRaises(OidcError): # typed, not TypeError + auth.token() + def test_nonpositive_expires_in_still_polls(self): # A non-positive expires_in in the device-auth response must be treated # as unknown, not as "already expired" — otherwise the flow times out @@ -1157,6 +1198,20 @@ def test_refresh_failure_falls_back_to_device_flow(self): self.assertEqual(self.state.refresh_requests, 1) self.assertEqual(self.state.device_requests, 1) # re-prompted + def test_non_string_refresh_error_falls_back_not_crashes(self): + # M1 (end-to-end): a refresh rejected with a NON-STRING error must still + # fall back to a fresh device flow, not crash. The terminal + # OidcDeviceFlowError _refresh raises is caught by _acquire's + # 'except OidcError'; before the fix a raw TypeError raised DURING that + # exception's construction slipped past the handler and aborted token(). + auth = self.make_auth() + self._seed_expired(auth) + self.state.refresh_response = (400, {'error': {'obj': 'denied'}}) + token = auth.token() # must not raise + self.assertEqual(token, ID_TOKEN) + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual(self.state.device_requests, 1) # fell back to sign-in + def test_refresh_token_preserved_when_not_rotated(self): auth = self.make_auth() self._seed_expired(auth) @@ -2016,6 +2071,74 @@ def on_prompt(self, resp): self.assertNotIn(a.cache_key, _MEMORY_GENERATION) # reclaimed on release self.assertNotIn(a.cache_key, _MEMORY_INFLIGHT) + def test_token_clear_stress(self): + # M3: drive the lock-free fast path and the generation/inflight CAS under + # REAL thread contention. The other concurrency tests exercise the CAS + # sequentially or via a same-thread synchronous clear(); this races many + # token() readers against a thread that periodically clear()s, the closest + # analogue to a SQLAlchemy/psycopg pool opening connections as the token is + # cycled. Asserts: no thread sees an exception (torn read / CAS bug / the + # M1 non-string crash would surface here), none deadlocks, the cache keeps + # serving so prompts stay far below the token() call count, and the + # process-global in-flight bookkeeping doesn't leak. + # + # Runs on a free-threaded (no-GIL) build too — that is where the lock-free + # fast-path read of self._tokens is genuinely concurrent with the locked + # writers, so the _ConcurrentClock (not the racy FakeClock) is used. + clock = _ConcurrentClock() + auth = self.make_auth(clock=clock, open_browser=False) + # Seed a valid token so the steady state is the lock-free fast path; a + # device flow then runs ONLY when a clear() has just emptied the cache. + seed = TokenSet( + access_token='a', id_token=ID_TOKEN, refresh_token='r', + issued_at=clock.now(), expires_at=clock.now() + 3600) + auth._cache.store(auth.cache_key, seed) + + n_workers = 7 + iters = 200 + n_clears = 40 + errors = [] + start = threading.Barrier(n_workers + 1 + 1) # workers + clearer + main + + def worker(): + start.wait() + try: + for _ in range(iters): + if auth.token() != ID_TOKEN: + errors.append('wrong token kind served') + return + except Exception as e: # noqa: BLE001 + errors.append(e) + + def clearer(): + start.wait() + try: + for _ in range(n_clears): + auth.clear() + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [threading.Thread(target=worker) for _ in range(n_workers)] + threads.append(threading.Thread(target=clearer)) + for t in threads: + t.start() + start.wait() # release everyone at once for maximum contention + for t in threads: + t.join(30) + for t in threads: + self.assertFalse(t.is_alive(), 'a thread deadlocked under contention') + self.assertEqual(errors, [], f'errors under contention: {errors[:3]}') + # A device flow runs only after a clear() empties the cache (the lock + # serializes the re-acquisition, so racing readers reuse it) — never per + # call. So prompts are bounded by the clears, far below n_workers*iters. + self.assertLessEqual(self.state.device_requests, n_clears + 1) + self.assertGreater(n_workers * iters, + max(1, self.state.device_requests) * 10) + # No leaked in-flight bookkeeping once the storm settles. + self.assertEqual(_MEMORY_INFLIGHT.get(auth.cache_key, 0), 0) + # The auth is still usable afterwards. + self.assertEqual(auth.token(), ID_TOKEN) + class TestAdapters(unittest.TestCase): """PG-wire connection adapters: tested via injected fake modules (the real @@ -2932,6 +3055,33 @@ def test_safe_link_url_allowlist(self): 'vbscript:x', 'file:///etc/passwd', '', None): self.assertIsNone(_safe_link_url(bad)) + def test_safe_link_url_rejects_userinfo_and_confusable_host(self): + # M2: an http(s) URL that could MISREPRESENT its destination host must + # not be made clickable / auto-opened — embedded userinfo (reads as the + # trusted host, connects past the '@'), or a non-ASCII confusable / + # control char in the authority. Such a URL is shown as inert text + # instead. Defeating a host-spoof the scheme allowlist alone misses. + from questdb.auth._render import _safe_link_url + spoofs = [ + 'https://login.questdb.io@evil.example/device', # userinfo + 'https://idp.example.com@evil/device?user_code=X', # userinfo + 'https://evil.example／login.questdb.io/auth', # fullwidth solidus + 'https://qоestdb.io/device', # Cyrillic homograph + 'https://idp.example.com\x00/device', # NUL in authority + ] + for url in spoofs: + self.assertIsNone(_safe_link_url(url), f'should reject {url!r}') + # Legitimate targets (DNS, explicit port, loopback, IPv6, punycode IDN) + # still pass — the host gate must not over-block the happy path. + for url in ( + 'https://idp.example.com/device', + 'https://idp.example.com:8443/device?user_code=WDJB-MJHT', + 'http://127.0.0.1:9000/device', + 'https://[::1]:8080/device', + 'https://xn--nxasmm1c.example/device', + 'https://accounts.google.com/o/oauth2/device/code'): + self.assertEqual(_safe_link_url(url), url, f'should accept {url!r}') + def test_render_link_inert_for_dangerous_scheme(self): from questdb.auth._render import _render_link safe = _render_link('https://idp/x') @@ -3315,6 +3465,58 @@ def __str__(self): self.assertIn('boom', str(e)) self.assertIn('spoof', str(e)) + def test_oidc_device_flow_error_tolerates_non_string_fields(self): + # M1: a hostile/non-conformant IdP can put a non-string into the + # error / error_description of a token or device-auth response. Building + # OidcDeviceFlowError from it must NOT raise a raw TypeError — that would + # escape the typed-error contract and, on the refresh path, slip past the + # 'except OidcError' fallback (the TypeError would be raised DURING the + # exception's construction, so it isn't an OidcError). The field is + # coerced through str() and sanitized, mirroring OidcError's args. + for bad in ({'code': 'denied'}, 12345, ['a', 'bb'], True): + e = OidcDeviceFlowError('failed', error=bad, error_description=bad) + self.assertIsInstance(e, OidcError) + self.assertIsInstance(e.error, str) + self.assertIsInstance(e.error_description, str) + + # A non-string whose text representation embeds an ANSI/bidi sequence is + # still stripped (same traceback-sink concern as OidcError's message). + class _Evil: + def __str__(self): + return 'denied \x1b[31m' + chr(0x202e) + 'spoof' + e = OidcDeviceFlowError('failed', error=_Evil(), + error_description=_Evil()) + for s in (e.error, e.error_description): + self.assertNotIn('\x1b', s) + self.assertNotIn(chr(0x202e), s) + self.assertIn('spoof', s) + # Absent stays None (not coerced to '' or 'None'). + self.assertIsNone(OidcDeviceFlowError('x').error) + self.assertIsNone(OidcDeviceFlowError('x').error_description) + + def test_userinfo_verification_url_not_auto_opened(self): + # M2: a tampered device response whose verification URL embeds userinfo + # (https://trusted@evil/) must NOT be auto-opened in the browser — it + # would navigate to `evil` while reading as `trusted`. The same + # _safe_link_url gate that makes it inert in the notebook also blocks the + # browser auto-open on a terminal. + auth = OidcDeviceAuth( + client_id='c', + device_authorization_endpoint='https://idp.example.com/device', + token_endpoint='https://idp.example.com/token', + open_browser=True) + with mock.patch('webbrowser.open') as wb, \ + mock.patch('questdb.auth._device.in_ipython_kernel', + return_value=False): + auth._maybe_open_browser({ + 'verification_uri': + 'https://login.questdb.io@evil.example/device'}) + wb.assert_not_called() + # A legitimate URL is still opened. + auth._maybe_open_browser( + {'verification_uri': 'https://idp.example.com/device'}) + wb.assert_called_once_with('https://idp.example.com/device') + def test_qr_helpers_degrade_without_qrcode(self): # The QR helpers must degrade gracefully (return None), never raise, # when `qrcode` is absent or the data is empty. See M4. From a0532333957b74511647ec1f68c1f77f5c5dba97 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 15:36:58 +0100 Subject: [PATCH 071/104] fix(auth): trim link whitespace and reject IPv6 zone-id host Two minor hardening tidy-ups from review: - _safe_link_url echoed the untrimmed original when a verification URL had leading/trailing whitespace (" https://idp/..."): urlparse ignores it when parsing the scheme, so the URL was accepted but returned with the spaces and handed to the href / webbrowser.open(). Strip first so the vetted value and the returned value match. Not a scheme bypass. - _ILLEGAL_HOST_CHARS now also rejects '%', so an IPv6 zone-id (fe80::1%eth0) no longer passes the PG-wire host guard. A zone-id is meaningful only for a link-local address on the local machine, never for reaching a remote QuestDB, so this keeps the guard a strict plain-host allowlist (defense-in-depth; '%' is not itself a libpq conninfo delimiter). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_adapters.py | 20 ++++++++++++-------- src/questdb/auth/_render.py | 5 +++++ test/test_auth.py | 14 +++++++++++--- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/questdb/auth/_adapters.py b/src/questdb/auth/_adapters.py index a5008be8..d3014464 100644 --- a/src/questdb/auth/_adapters.py +++ b/src/questdb/auth/_adapters.py @@ -43,12 +43,16 @@ _DEFAULT_PG_PORT = 8812 _DEFAULT_DATABASE = 'qdb' -# Reject connection-string delimiters (';', '=') and whitespace/control chars in -# the host: a real hostname/IP never has them, so their presence means a -# tampered URL trying to inject PG connection parameters (psycopg turns its -# kwargs into a libpq conninfo string). ':' is allowed — IPv6 literals contain -# it, and the PG drivers take host and port separately. -_ILLEGAL_HOST_CHARS = re.compile(r'[\x00-\x20\x7f;=]') +# Reject connection-string delimiters (';', '='), whitespace/control chars, and +# '%' in the host: a real hostname / IPv4 / IPv6-literal never has them, so their +# presence means a tampered URL trying to inject PG connection parameters +# (psycopg turns its kwargs into a libpq conninfo string). ':' is allowed — IPv6 +# literals contain it, and the PG drivers take host and port separately. '%' +# would only appear as an IPv6 zone-id (e.g. 'fe80::1%eth0'), meaningful only for +# a link-local address on the local machine and never for reaching a remote +# QuestDB; rejecting it keeps the guard a strict plain-host allowlist +# (defense-in-depth — '%' is not itself a conninfo delimiter). +_ILLEGAL_HOST_CHARS = re.compile(r'[\x00-\x20\x7f;=%]') def _pg_module(): @@ -87,8 +91,8 @@ def _require_host(url: str, host: Optional[str] = None) -> str: if _ILLEGAL_HOST_CHARS.search(resolved): raise OidcConfigError( f'The QuestDB host {resolved!r} contains an illegal character ' - "(';', '=', whitespace or a control character). A hostname or IP " - 'address never does; this indicates a malformed or tampered URL. ' + "(';', '=', '%', whitespace or a control character). A hostname or " + 'IP address never does; this indicates a malformed or tampered URL. ' '(Such a host could otherwise inject PG connection parameters.)') return resolved diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index a901199a..4cb1b049 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -121,6 +121,11 @@ def _safe_link_url(url: Optional[str]) -> Optional[str]: if not url or not isinstance(url, str): # A non-string has no scheme to vet and would make urlparse raise. return None + # urlparse() ignores surrounding whitespace when parsing the scheme, so + # " https://idp/..." parses as https; trim it so the value we vet is the + # value we return (and hand to the href / webbrowser.open()), not the + # untrimmed original. + url = url.strip() try: parts = urllib.parse.urlparse(url) scheme = (parts.scheme or '').lower() diff --git a/test/test_auth.py b/test/test_auth.py index afcc315c..097946fe 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2298,9 +2298,11 @@ def test_require_host_with_conf_metachars_rejected(self): with self.subTest(url=bad): with self.assertRaises(OidcConfigError): _require_host(bad) - # An explicit host= override goes through the same guard (incl. - # whitespace, which is never valid in a host). - for bad_host in ('evil;sslmode=disable', 'a=b', 'h ost'): + # An explicit host= override goes through the same guard: whitespace + # (never valid in a host) and an IPv6 zone-id '%' (meaningful only for a + # link-local address on the local machine, never a remote QuestDB) are + # rejected too, keeping the guard a strict plain-host allowlist. + for bad_host in ('evil;sslmode=disable', 'a=b', 'h ost', 'fe80::1%eth0'): with self.subTest(host=bad_host): with self.assertRaises(OidcConfigError): _require_host('https://db.example.com:9000', bad_host) @@ -3054,6 +3056,12 @@ def test_safe_link_url_allowlist(self): for bad in ('javascript:alert(1)', 'data:text/html,x', 'vbscript:x', 'file:///etc/passwd', '', None): self.assertIsNone(_safe_link_url(bad)) + # Surrounding whitespace is trimmed: urlparse ignores it when parsing the + # scheme, so the value we return (and hand to the href / browser) must be + # the trimmed one, not the untrimmed original. + self.assertEqual( + _safe_link_url(' https://idp.example.com/x '), + 'https://idp.example.com/x') def test_safe_link_url_rejects_userinfo_and_confusable_host(self): # M2: an http(s) URL that could MISREPRESENT its destination host must From 5d1421b1d2d04aa7084e4b2a41166007acdfbe9e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 15:53:43 +0100 Subject: [PATCH 072/104] fix(auth): show real link host and unify the open/QR target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two minor verification-link hardening items from review. - A homoglyph "dot" in the verification URL host (fullwidth U+FF0E, one-dot- leader U+2024, ideographic U+3002) IDNA-folds to a real '.', so the true registrable domain is evil.com while the raw string reads like a trusted host. The ASCII host-allowlist already blocks such a host from being clickable or opened, but the prompt still echoed the raw string. Add _display_url(), which renders the host in IDNA/punycode form and drops any userinfo, so the prompt (terminal and Jupyter) shows the host the browser would actually resolve; clickability stays governed by the allowlist. - webbrowser.open and the terminal QR vetted the RAW response value while the on-screen link was control-stripped, so a stripped char could survive into the opened/scanned target — the displayed link and the real target could diverge. Route every sink (display href, webbrowser.open, both QR encoders) through one _safe_target() = _safe_link_url(_strip_control(...)), so they can't diverge. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 10 ++-- src/questdb/auth/_render.py | 105 +++++++++++++++++++++++++++++------- test/test_auth.py | 53 ++++++++++++++++++ 3 files changed, 144 insertions(+), 24 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 10a63169..8f36d4a3 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -48,7 +48,7 @@ from ._http import build_ssl_context, post_form, safe_urlparse from ._render import ( Renderer, - _safe_link_url, + _safe_target, detect_interactive, in_ipython_kernel, make_renderer, @@ -906,9 +906,11 @@ def _maybe_open_browser(self, resp: Dict[str, Any]) -> None: # kernel host isn't the user's machine. Suppress with open_browser=False. if not self.open_browser or in_ipython_kernel(): return - # Only http(s) — never a javascript:/data: scheme from a malicious or - # MITM'd device response. - target = _safe_link_url( + # Open the SAME _strip_control'd, vetted target the prompt shows (via + # _safe_target) — not the raw response value — so a char stripped from the + # on-screen link can't survive into the opened URL, and a javascript:/ + # data: scheme (or userinfo / non-ASCII host) is never opened. + target = _safe_target( resp.get('verification_uri_complete') or resp.get('verification_uri') or resp.get('verification_url')) diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 4cb1b049..eca4ca40 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -144,16 +144,76 @@ def _safe_link_url(url: Optional[str]) -> Optional[str]: return url +def _safe_target(value: Optional[str]) -> Optional[str]: + """ + The single control-stripped, scheme/userinfo/host-vetted URL to click, open + in a browser, or encode as a QR — or ``None`` if it can't be trusted. + + One value feeds the displayed link's ``href``, :func:`webbrowser.open` and + both QR encoders, so a control / zero-width char stripped from the on-screen + link can never survive into the URL actually opened or scanned: the displayed + link and the real target cannot diverge. (Earlier the browser/QR paths vetted + the *raw* response value while the display was control-stripped.) + """ + return _safe_link_url(_strip_control(value)) + + +def _display_url(url: Optional[str]) -> str: + """ + A verification URL rendered safe to *show* as text. + + Control / bidi / zero-width chars are stripped, then the host is rebuilt in + its IDNA / punycode (ASCII) form and any userinfo is dropped, so neither a + homoglyph host (e.g. a fullwidth ``U+FF0E`` that IDNA folds to a real ``.``, + making the true registrable domain ``evil.com``) nor a ``user@host`` trick + can visually masquerade as a trusted host in the prompt — the user reads the + host the browser would actually resolve. Clickability is decided + independently by :func:`_safe_target` (which rejects a non-ASCII host + outright); this governs only the visible text. A non-``http(s)`` / hostless + value is returned control-stripped but otherwise unchanged. + """ + text = _strip_control(url) + if not text: + return '' + try: + parts = urllib.parse.urlparse(text) + scheme = (parts.scheme or '').lower() + host = parts.hostname + port = parts.port + except ValueError: + return text + if scheme not in ('http', 'https') or not host: + return text # nothing host-like to normalize (opaque / relative) + if host.isascii(): + ascii_host = host + else: + try: + # The stdlib idna codec splits on the homoglyph dots too + # (``. 。 ． ｡``) and ToASCII-encodes each label, so a fullwidth-dot + # host resolves to its real ASCII registrable domain here. + ascii_host = host.encode('idna').decode('ascii') + except (UnicodeError, ValueError): + # IDNA can't encode it (an illegal label); make the bytes visible + # rather than let an invisible homoglyph through unchanged. + ascii_host = host.encode('ascii', 'backslashreplace').decode('ascii') + host_part = f'[{ascii_host}]' if ':' in ascii_host else ascii_host # IPv6 + netloc = f'{host_part}:{port}' if port is not None else host_part + return urllib.parse.urlunparse(parts._replace(netloc=netloc)) + + def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: """ Render ``url`` as a clickable link, or as inert escaped text if its scheme is not ``http(s)``. - The label defaults to the URL itself. A rejected URL is shown as escaped - plain text (still visible/copyable) but never made clickable. + The label defaults to the URL with its host shown in IDNA/punycode form + (:func:`_display_url`) so a homoglyph / userinfo host can't masquerade as a + trusted one; a rejected URL is shown as that escaped plain text (still + visible/copyable) but never made clickable. The ``href`` is the single vetted + target (:func:`_safe_target`), so the link points exactly where it reads. """ - safe = _safe_link_url(url) - label = html.escape(text if text is not None else (url or '')) + safe = _safe_target(url) + label = html.escape(text if text is not None else _display_url(url)) if safe is None: return label return (f' str: def format_prompt(resp: Dict[str, Any]) -> str: """Plain-text sign-in prompt (also used as the notebook fallback).""" - uri = _strip_control(_verification_uri(resp)) + # _display_url shows the IDNA/punycode host (and drops userinfo) so a + # homoglyph / user@host can't spoof the host in the plain-text prompt either. + uri = _display_url(_verification_uri(resp)) code = _strip_control(str(resp.get('user_code', ''))) - complete = _strip_control(_verification_uri_complete(resp)) + complete = _display_url(_verification_uri_complete(resp)) lines = [ '🔐 Sign in to QuestDB', f' Open {uri} and enter code: {code}', @@ -269,11 +331,12 @@ def _write(self, text: str) -> None: def on_prompt(self, resp: Dict[str, Any]) -> None: self._write(format_prompt(resp) + '\n') if self._qr: - # Scheme-vet the target before encoding it, mirroring the Jupyter QR - # (_qr_img): never turn a javascript:/data: verification_uri from a - # hostile device response into a scannable QR. - target = (_safe_link_url(_verification_uri_complete(resp)) - or _safe_link_url(_verification_uri(resp))) + # Encode the SAME _strip_control'd, vetted target the prompt displays + # (via _safe_target), not the raw response value — so a char stripped + # from the on-screen URL can't survive into the scanned QR, and a + # javascript:/data: scheme is never encoded. + target = (_safe_target(_verification_uri_complete(resp)) + or _safe_target(_verification_uri(resp))) art = _qr_ascii(target) if target else None if art: self._write(art + '\n') @@ -335,27 +398,29 @@ def _prompt_head(self): URL. Returns ``(body, uri, complete)``. """ resp = self._resp - uri = _strip_control(_verification_uri(resp)) + # _render_link / _safe_target / _qr_img each strip + vet internally, so + # pass the raw fields and let the single canonical target drive the href, + # the QR and the displayed (IDNA-normalized) label uniformly. + raw_uri = _verification_uri(resp) + raw_complete = _verification_uri_complete(resp) code = html.escape(_strip_control(str(resp.get('user_code', '')))) - complete = _verification_uri_complete(resp) - complete = _strip_control(complete) if complete else None body = [ '
' '🔐 Sign in to QuestDB
', - f'
Open {_render_link(uri)} and enter code:
', + f'
Open {_render_link(raw_uri)} and enter code:
', f'
{code}
', ] - if _safe_link_url(complete): + if _safe_target(raw_complete): body.append( '
' + _render_link( - complete, text='Click here to authorize directly →') + raw_complete, text='Click here to authorize directly →') + '
') if self._qr: - qr_html = self._qr_img(complete, uri) + qr_html = self._qr_img(raw_complete, raw_uri) if qr_html: body.append(qr_html) - return body, uri, complete + return body, raw_uri, raw_complete def _qr_img(self, complete: Optional[str], uri: str) -> str: """The QR ```` for the verification URL, built once and cached. @@ -365,7 +430,7 @@ def _qr_img(self, complete: Optional[str], uri: str) -> str: the countdown re-renders neither drop the QR nor regenerate the PNG. """ if self._qr_html is None: - target = _safe_link_url(complete) or _safe_link_url(uri) + target = _safe_target(complete) or _safe_target(uri) data_uri = _qr_data_uri(target) if target else None self._qr_html = ( f'QR code Date: Thu, 25 Jun 2026 16:08:58 +0100 Subject: [PATCH 073/104] fix(auth): validate pg_port and attach status to get_json errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two minor consistency fixes from review. (A third finding — OidcError leaving a non-str arg un-stripped — was already handled: _strip_control wraps the whole isinstance conditional, so str(a) is stripped for a non-str arg, and test_oidc_error_sanitizes_non_string_arg already covers it.) - A non-integer pg_port (e.g. a port read from an env var without int()) reached URL.create(port=...) / driver.connect(port=...) and surfaced as a bare ValueError / driver error, escaping OidcConfigError. Add _coerce_port(), called up front in both adapters (before the driver import) so a bad port fails fast with the typed error. - get_json() raised OidcError without the HTTP status, unlike post_form() which attaches it on every non-conformant branch. Attach status= to both get_json raises so the terminal-vs-transient classifier stays uniform before any retry caller is added. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_adapters.py | 27 +++++++++++++++++++++++++++ src/questdb/auth/_http.py | 10 ++++++++-- test/test_auth.py | 27 +++++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/questdb/auth/_adapters.py b/src/questdb/auth/_adapters.py index d3014464..beb21deb 100644 --- a/src/questdb/auth/_adapters.py +++ b/src/questdb/auth/_adapters.py @@ -97,6 +97,31 @@ def _require_host(url: str, host: Optional[str] = None) -> str: return resolved +def _coerce_port(pg_port: Any) -> int: + """ + Coerce ``pg_port`` to an ``int`` within the module's typed-error contract. + + A non-integer ``pg_port`` (e.g. a port read from an env var without an + ``int()``) would otherwise reach ``URL.create(port=...)`` / + ``driver.connect(port=...)`` and surface as a bare ``ValueError`` / driver + error, escaping ``OidcConfigError``. ``bool`` is an ``int`` subclass but + ``True``/``False`` is never a meaningful port, so reject it explicitly — + mirroring the constructor's other up-front type checks. + """ + if isinstance(pg_port, bool): + raise OidcConfigError( + f'pg_port must be an integer port number, got {pg_port!r}.') + try: + port = int(pg_port) + except (TypeError, ValueError) as e: + raise OidcConfigError( + f'pg_port must be an integer port number, got {pg_port!r}.') from e + if not 1 <= port <= 65535: + raise OidcConfigError( + f'pg_port must be a valid TCP port (1-65535), got {port}.') + return port + + def sqlalchemy_engine( auth: OidcDeviceAuth, url: str, @@ -128,6 +153,7 @@ def sqlalchemy_engine( (v3) or ``postgresql+psycopg2`` depending on what is installed. :param engine_kwargs: Forwarded to ``create_engine``. """ + pg_port = _coerce_port(pg_port) try: from sqlalchemy import create_engine, event from sqlalchemy.engine import URL @@ -181,6 +207,7 @@ def psycopg_connect( ``host=`` is given. :param connect_kwargs: Forwarded to the driver's ``connect()``. """ + pg_port = _coerce_port(pg_port) mod = _pg_module() return mod.connect( host=_require_host(url, host), diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 8d95dea7..91985c7b 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -296,14 +296,20 @@ def get_json( 'GET', url, headers=headers, timeout=timeout, ctx=ctx, insecure=insecure) if not resp.ok: + # Attach the HTTP status (mirroring post_form) so a future retry caller + # can tell a terminal 4xx from a transient 5xx/429 the same way the poll + # loop / silent refresh do. Today's callers (fetch_settings, IdP + # discovery) are one-shot and ignore it; this keeps the contract uniform. raise OidcError( - f'HTTP {resp.status} from {url}: {resp.text()[:200]}') + f'HTTP {resp.status} from {url}: {resp.text()[:200]}', + status=resp.status) try: return resp.json() except (ValueError, UnicodeDecodeError, RecursionError) as e: # RecursionError (deeply-nested JSON) isn't a ValueError, so catch it # explicitly to keep the typed contract. - raise OidcError(f'Invalid JSON from {url}: {e}') from e + raise OidcError( + f'Invalid JSON from {url}: {e}', status=resp.status) from e def post_form( diff --git a/test/test_auth.py b/test/test_auth.py index d9308b5f..cdd5aa36 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2314,6 +2314,25 @@ def test_require_host_with_conf_metachars_rejected(self): _require_host('https://db.example.com:9000', 'questdb.example.com'), 'questdb.example.com') + def test_pg_port_validation(self): + # A non-integer pg_port (e.g. a port read from an env var without int()) + # must surface as OidcConfigError, not a bare ValueError / driver error + # from URL.create(port=...) / connect(port=...). The check runs before + # the driver import, so it holds even without sqlalchemy / psycopg. + from questdb.auth._adapters import _coerce_port + for bad in ('not-a-port', None, '88a2', True, 0, 70000): + with self.subTest(pg_port=bad): + with self.assertRaises(OidcConfigError): + _coerce_port(bad) + self.assertEqual(_coerce_port(8812), 8812) + self.assertEqual(_coerce_port('5432'), 5432) # str port coerced + # Both adapter entry points reject it up front (no driver required). + for fn in (sqlalchemy_engine, psycopg_connect): + with self.subTest(fn=fn.__name__): + with self.assertRaises(OidcConfigError): + fn(_FakeAuth(), 'https://db.example.com:9000', + pg_port='not-a-port') + @unittest.skipIf(importlib.util.find_spec('sqlalchemy') is not None, 'sqlalchemy installed') def test_sqlalchemy_engine_missing_dep_raises(self): @@ -3031,17 +3050,21 @@ def test_get_json_non_2xx_raises_oidc_error(self): # See M4. from questdb.auth import _http with _raw_response_server(500, 'text/plain', b'boom') as b: - with self.assertRaises(OidcError): + with self.assertRaises(OidcError) as cm: _http.get_json(b + '/settings', timeout=5) + # The HTTP status is attached (mirroring post_form) so a future retry + # caller can classify terminal-vs-transient the same way. + self.assertEqual(cm.exception.status, 500) def test_get_json_non_json_2xx_raises_oidc_error(self): # A 2xx /settings or discovery body that isn't JSON must surface as # OidcError, not a raw JSONDecodeError. See M4. from questdb.auth import _http with _raw_response_server(200, 'text/html', b'x') as b: - with self.assertRaises(OidcError): + with self.assertRaises(OidcError) as cm: _http.get_json( b + '/.well-known/openid-configuration', timeout=5) + self.assertEqual(cm.exception.status, 200) # status attached class TestRendererSecurity(unittest.TestCase): From 6806bdf51c9140464f4d6c5eb7fcf909f95f9760 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 16:46:31 +0100 Subject: [PATCH 074/104] fix(auth): guard verification_uri, real-lifetime message, Retry-After Four minor prompt / poll robustness items from review. - _request_device_code accepted a 200 device-auth response that omits the verification URI (only device_code/user_code were required), which then rendered a blank "Open and enter code" gap and polled pointlessly. Require a verification_uri (or the legacy verification_url) too; its absence is a non-conformant response. - The "expires in N min" sign-in message reported the cache's clamped expires_at (_MAX_EXPIRES_IN, 1h), under-stating a token that genuinely lives longer. Add _display_lifetime(): report the JWT exp claim (the real expiry) for the message, falling back to the clamped value for an opaque token; the cached expires_at stays clamped (re-validated at least hourly). - The poll back-off after a 429 / slow_down was a fixed +5s and ignored Retry-After. Surface the parsed Retry-After from post_form via a backward- compatible 2-tuple subclass (_PostResult) and honor it in the poll loop (clamped to [5, 60]); _backoff_interval keeps the RFC 8628 +5s step otherwise. - Pin (test-only) that _NoRedirect is wired into the HTTPS (ctx != None) opener -- the production path that carries the bearer/refresh token -- not only the plain-HTTP opener the existing end-to-end redirect test exercises. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 71 ++++++++++++++++----- src/questdb/auth/_http.py | 47 +++++++++++++- test/test_auth.py | 120 ++++++++++++++++++++++++++++++++++++ 3 files changed, 221 insertions(+), 17 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 8f36d4a3..739e3492 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -173,6 +173,19 @@ def _http_status_is_transient(status: Optional[int]) -> bool: return status is not None and (status >= 500 or status == 429) +def _backoff_interval(interval: int, retry_after: Optional[int]) -> int: + """ + The next poll interval after a 429 / ``slow_down``. + + Honors a server ``Retry-After`` (delta-seconds) when present, else the RFC + 8628 §3.5 +5s slow-down step. Clamped to ``[_MIN_POLL_INTERVAL, + _MAX_POLL_INTERVAL]`` so a hostile/huge value can't pin the polling thread — + the device-code deadline still bounds the total wait. + """ + target = retry_after if retry_after is not None else interval + 5 + return min(_MAX_POLL_INTERVAL, max(_MIN_POLL_INTERVAL, target)) + + def _validate_positive_number(value: Any, name: str) -> None: """ Require a duration argument to be a positive, finite number of seconds. @@ -737,10 +750,26 @@ def _run_device_flow(self) -> TokenSet: claims = (_decode_jwt_claims(tokens.id_token) or _decode_jwt_claims(tokens.access_token)) identity = _identity_from_claims(claims) - self._renderer.on_success( - identity, max(0.0, tokens.expires_at - self._now())) + self._renderer.on_success(identity, self._display_lifetime(tokens, claims)) return tokens + def _display_lifetime( + self, tokens: TokenSet, claims: Dict[str, Any]) -> float: + # Remaining lifetime to SHOW in the sign-in message. tokens.expires_at is + # deliberately clamped to _MAX_EXPIRES_IN so a cached token is + # re-validated at least hourly — reporting that would under-state a token + # that genuinely lives longer. Prefer the JWT `exp` claim (the + # authoritative token expiry); fall back to the clamped value for an + # opaque token with no exp. Bounded to ~1y so a hostile/garbage exp + # (incl. inf/nan, which fail the comparison) can't overflow on_success's + # int(round(...)). + exp = claims.get('exp') + if isinstance(exp, (int, float)) and not isinstance(exp, bool): + remaining = float(exp) - self._now() + if 0 < remaining <= 366 * 24 * 3600: + return remaining + return max(0.0, tokens.expires_at - self._now()) + def _request_device_code(self) -> Dict[str, Any]: form = { 'client_id': self.config.client_id, @@ -750,20 +779,27 @@ def _request_device_code(self) -> Dict[str, Any]: form['audience'] = self.config.audience status, body = self._idp_post( self.config.device_authorization_endpoint, form) + # RFC 8628 §3.2 requires device_code, user_code AND a verification URI + # (RFC spells it verification_uri; some IdPs, e.g. older Google, use + # verification_url). Require the URI too: without it the prompt would + # render a blank "Open and enter code" gap, so its absence is a + # non-conformant response, not a usable one. if (status == 200 and _str_or_none(body.get('device_code')) - and _str_or_none(body.get('user_code'))): + and _str_or_none(body.get('user_code')) + and (_str_or_none(body.get('verification_uri')) + or _str_or_none(body.get('verification_url')))): return body error = body.get('error') if status == 200: - # 200 but the guard above failed: device_code/user_code missing or + # 200 but the guard above failed: a required field is missing or # non-string (coerced via _str_or_none, so a JSON number/list reads - # as absent instead of being stringified into the poll request). - # A non-conformant body, not an HTTP failure — say so plainly rather - # than a contradictory "failed (HTTP 200)". + # as absent instead of being stringified into the prompt / poll + # request). A non-conformant body, not an HTTP failure — say so + # plainly rather than a contradictory "failed (HTTP 200)". raise OidcDeviceFlowError( - 'The IdP returned a 200 device-authorization response that is ' - 'missing the required "device_code"/"user_code" fields; cannot ' - 'start the device flow.', + 'The IdP returned a 200 device-authorization response missing a ' + 'required field (device_code, user_code, or verification_uri); ' + 'cannot start the device flow.', error=error, error_description=body.get('error_description')) if status in (400, 404, 405) or error in ( @@ -815,7 +851,7 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: self._sleep(min(interval, remaining)) try: - status, body = self._idp_post( + result = self._idp_post( self.config.token_endpoint, { 'grant_type': DEVICE_CODE_GRANT, @@ -844,9 +880,16 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: # poll again rather than discard the sign-in (the deadline bounds # the total wait; a genuine JSON rejection arrives below). if getattr(e, 'status', None) == 429: - interval = min(_MAX_POLL_INTERVAL, interval + 5) + # No JSON body here (a non-JSON 429 from a proxy/WAF), so no + # Retry-After is surfaced; fall back to the +5s step. + interval = _backoff_interval(interval, None) continue + status, body = result + # post_form surfaces Retry-After on the JSON path via _PostResult; a + # mocked _idp_post may return a plain 2-tuple, hence getattr/default. + retry_after = getattr(result, 'retry_after', None) + if status == 200: # The RFC 6749 §5.1 token response: the grant completed. Accept # it only if it carries the kind _select hands to QuestDB, using @@ -869,14 +912,14 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: # polling until the deadline, as above. if status >= 500 or status == 429: if status == 429: - interval = min(_MAX_POLL_INTERVAL, interval + 5) + interval = _backoff_interval(interval, retry_after) continue error = body.get('error') if error == 'authorization_pending': continue if error == 'slow_down': - interval = min(_MAX_POLL_INTERVAL, interval + 5) + interval = _backoff_interval(interval, retry_after) continue if error == 'expired_token': self._renderer.on_failure( diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 91985c7b..3d826fa7 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -312,6 +312,44 @@ def get_json( f'Invalid JSON from {url}: {e}', status=resp.status) from e +def _parse_retry_after(headers: Optional[Mapping[str, str]]) -> Optional[int]: + """ + A ``Retry-After`` header as a non-negative ``int`` of seconds, else ``None``. + + Honors the delta-seconds form (RFC 7231 §7.1.3); the HTTP-date form is + ignored (the caller's fixed back-off covers that rarer case, and parsing a + date pulls in tz handling for little gain). Case-insensitive, so an HTTP/2 / + proxy-lowercased header name is still matched. + """ + if not headers: + return None + value = None + for key, val in headers.items(): + if key.lower() == 'retry-after': + value = val + break + try: + secs = int(str(value).strip()) + except (TypeError, ValueError): + return None + return secs if secs >= 0 else None + + +class _PostResult(tuple): + """``(status, body)`` carrying an extra ``.retry_after`` (seconds or None). + + A 2-tuple subclass, so existing ``status, body = post_form(...)`` callers are + unchanged; the device-flow poll additionally reads ``.retry_after`` to honor + a 429 / 503 ``Retry-After`` header instead of its fixed +5s back-off. + """ + + def __new__(cls, status: int, body: Dict[str, Any], + retry_after: Optional[int]): + self = super().__new__(cls, (status, body)) + self.retry_after = retry_after + return self + + def post_form( url: str, form: Mapping[str, Any], @@ -323,12 +361,15 @@ def post_form( """ POST a form-url-encoded body and parse the JSON response. - Returns ``(status, parsed_json)``. Used for the device-authorization and - token endpoints, which return JSON on both success and error. + Returns ``(status, parsed_json)`` — a :class:`_PostResult`, a 2-tuple that + also carries ``.retry_after`` (the parsed ``Retry-After`` seconds, or + ``None``). Used for the device-authorization and token endpoints, which + return JSON on both success and error. """ resp = request( 'POST', url, form=form, headers=headers, timeout=timeout, ctx=ctx, insecure=insecure) + retry_after = _parse_retry_after(resp.headers) try: parsed = resp.json() except (ValueError, UnicodeDecodeError, RecursionError): @@ -349,4 +390,4 @@ def post_form( # — fails the poll loop fast instead of polling on to "code expired". raise OidcError( f'Unexpected JSON shape from {url}: {parsed!r}', status=resp.status) - return resp.status, parsed + return _PostResult(resp.status, parsed, retry_after) diff --git a/test/test_auth.py b/test/test_auth.py index cdd5aa36..46eeb71b 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -580,6 +580,73 @@ def test_non_string_poll_error_field_raises_typed_error(self): with self.assertRaises(OidcError): # typed, not TypeError auth.token() + def test_missing_verification_uri_is_rejected(self): + # Issue 6: a 200 device-auth response with device_code/user_code but NO + # verification URI (RFC 8628 §3.2 requires it) must be rejected with a + # typed error, not accepted into a prompt that renders a blank + # "Open and enter code" gap and then polls pointlessly. + self.state.device_response = { + 'device_code': 'DEV-CODE', 'user_code': 'WDJB-MJHT', + 'expires_in': 600, 'interval': 5} # no verification_uri / _url + auth = self.make_auth() + with self.assertRaises(OidcDeviceFlowError): + auth.token() + self.assertEqual(len(self.state.token_requests), 0) # never polled + # The legacy verification_url spelling (older Google) is accepted. + self.state.device_response = { + 'device_code': 'DEV-CODE', 'user_code': 'WDJB-MJHT', + 'verification_url': 'https://idp.example.com/device', + 'expires_in': 600, 'interval': 5} + self.state.token_script = [(200, None)] + self.assertEqual(self.make_auth().token(), ID_TOKEN) + + def test_success_message_reports_real_jwt_lifetime(self): + # Issue 7: the "expires in N min" message must report the token's REAL + # lifetime (JWT exp), not the cache's clamped expires_at (_MAX_EXPIRES_IN, + # 1h). An 8h token must not be reported as "60 min". + from questdb.auth._device import _MAX_EXPIRES_IN + + class _Rec(Renderer): + expires_in = None + + def on_success(self, identity, expires_in): + self.expires_in = expires_in + + auth = self.make_auth() # sets self._clock + auth._renderer = _Rec() + real_exp = self._clock.now() + 8 * 3600 + id_tok = _jwt({'sub': 'alice', 'exp': real_exp}) + self.state.token_script = [(200, { + 'access_token': 'a', 'id_token': id_tok, 'refresh_token': 'r', + 'expires_in': 8 * 3600, 'scope': 'openid groups'})] # clamped to 1h + auth.token() + # Reported lifetime reflects the 8h JWT exp, well beyond the 1h clamp... + self.assertGreater(auth._renderer.expires_in, 7 * 3600) + self.assertLessEqual(auth._renderer.expires_in, 8 * 3600) + # ...while the CACHED token is still clamped (re-validated at least hourly). + self.assertLessEqual(auth._tokens.expires_at - self._clock.now(), + _MAX_EXPIRES_IN) + + def test_poll_honors_retry_after(self): + # Issue 8: a slow_down poll response carrying Retry-After backs off by + # that many seconds (clamped), not the fixed +5s. + from questdb.auth._http import _PostResult + auth = self.make_auth() + real = auth._idp_post + n = {'tok': 0} + + def fake(url, form): + if url.endswith('/token'): + n['tok'] += 1 + if n['tok'] == 1: + return _PostResult(400, {'error': 'slow_down'}, 30) + return real(url, form) # then succeed + return real(url, form) + + auth._idp_post = fake + self.assertEqual(auth.token(), ID_TOKEN) + self.assertIn(30, self._clock.sleeps) # honored Retry-After, not +5 + def test_nonpositive_expires_in_still_polls(self): # A non-positive expires_in in the device-auth response must be treated # as unknown, not as "already expired" — otherwise the flow times out @@ -2871,6 +2938,59 @@ def do_GET(self): self.assertEqual(resp.status, 302) self.assertEqual(seen, [('/exec', 'Bearer SECRET')]) + def test_https_opener_refuses_redirects(self): + # Issue 9: the production path carries the bearer / refresh token over + # HTTPS (ctx is a real SSLContext); the end-to-end redirect test above + # exercises only the plain-HTTP (ctx=None) opener. Pin that _NoRedirect is + # wired into the HTTPS opener too — and that it REPLACES the default + # HTTPRedirectHandler (so a 30x is actually refused, not just shadowed). + import ssl + import urllib.request + from questdb.auth import _http + opener = _http._opener(ssl.create_default_context()) + redirect_handlers = [ + h for h in opener.handlers + if isinstance(h, urllib.request.HTTPRedirectHandler)] + self.assertEqual(len(redirect_handlers), 1, + 'expected exactly one redirect handler in the opener') + self.assertIsInstance( + redirect_handlers[0], _http._NoRedirect, + '_NoRedirect missing from the HTTPS opener — a 30x could re-send the ' + 'bearer/refresh token cross-origin') + + def test_parse_retry_after(self): + # Issue 8: Retry-After delta-seconds parsed (case-insensitive); the + # HTTP-date form and junk return None (caller falls back to its +5s step). + from questdb.auth._http import _parse_retry_after + self.assertEqual(_parse_retry_after({'Retry-After': '30'}), 30) + self.assertEqual(_parse_retry_after({'retry-after': ' 45 '}), 45) + self.assertEqual(_parse_retry_after({'Retry-After': '0'}), 0) + for bad in ({'Retry-After': 'Wed, 21 Oct 2015 07:28:00 GMT'}, + {'Retry-After': '-5'}, {'Retry-After': 'soon'}, + {'X-Other': '5'}, {}, None): + self.assertIsNone(_parse_retry_after(bad)) + + def test_backoff_interval(self): + # Issue 8: honor Retry-After (clamped to [5, 60]); else the RFC 8628 +5s. + from questdb.auth._device import ( + _backoff_interval, _MIN_POLL_INTERVAL, _MAX_POLL_INTERVAL) + self.assertEqual(_backoff_interval(5, None), 10) # +5s step + self.assertEqual(_backoff_interval(20, None), 25) + self.assertEqual(_backoff_interval(5, 30), 30) # honored + self.assertEqual(_backoff_interval(5, 120), _MAX_POLL_INTERVAL) # capped + self.assertEqual(_backoff_interval(5, 1), _MIN_POLL_INTERVAL) # floored + + def test_post_result_is_2tuple_with_retry_after(self): + # Issue 8: _PostResult is a 2-tuple (existing `status, body = ...` callers + # are unaffected) that also carries .retry_after. + from questdb.auth._http import _PostResult + r = _PostResult(429, {'error': 'slow_down'}, 30) + status, body = r + self.assertEqual((status, body), (429, {'error': 'slow_down'})) + self.assertEqual(r, (429, {'error': 'slow_down'})) # equals plain tuple + self.assertEqual(r.retry_after, 30) + self.assertIsNone(_PostResult(200, {}, None).retry_after) + def test_malformed_url_raises_config_error(self): # A non-integer port must surface as OidcConfigError, not a raw # http.client.InvalidURL escaping the typed-error contract — this is the From ff84625c94bb93de5405c4ceb07890b1203e7019 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 18:09:45 +0100 Subject: [PATCH 075/104] fix(auth): reject confusable endpoint authority and inf timeout Two hardening fixes from the PR review. Endpoint/issuer authority confusion: the origin pin, co-location check and cache key all derived the host from urlparse(url).hostname, which strips userinfo at the last '@', while urllib hands the FULL netloc to the connection. So "https://attacker.evil\@idp.good/token" validated as host "idp.good" (passing the issuer-origin pin) yet urllib would target "attacker.evil\@idp.good". Add _reject_confusable_authority() to refuse any credential URL whose authority carries userinfo, a backslash, whitespace or a control char, and call it on both endpoints (in validate_endpoint_origins, which every construction path reaches via __init__) and on the issuer (in resolve_config before the pin). This closes the validation-vs-connection mismatch by design and mirrors the host hygiene already enforced in _adapters and _render; IPv6 literals and Google-style cross-origin issuers are unaffected. Non-finite / over-large timeout: _validate_positive_number accepted inf (inf > 0 is True), which then crashed socket.settimeout with a bare OverflowError, escaping the typed-error contract the validator exists to enforce. Convert-and-catch instead so inf/-inf/NaN and a too-large int (also a settimeout crash; math.isfinite itself raises on it) all raise OidcConfigError. Add regression tests for both and extend the bad-typed-args cases. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 31 ++++++++++++++++------- src/questdb/auth/_discovery.py | 37 ++++++++++++++++++++++++++++ test/test_auth.py | 45 +++++++++++++++++++++++++++++++++- 3 files changed, 103 insertions(+), 10 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 739e3492..42168fdf 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -29,6 +29,7 @@ import base64 import binascii import json +import math import threading import time import webbrowser @@ -190,17 +191,29 @@ def _validate_positive_number(value: Any, name: str) -> None: """ Require a duration argument to be a positive, finite number of seconds. - Mirrors the constructor's other up-front type checks: a non-numeric value - would otherwise surface later as a bare TypeError from the poll-interval - clamp (``max(_MIN_POLL_INTERVAL, default_interval)``) or a urllib socket - call (``timeout``), escaping the module's typed-error contract. ``bool`` is - an ``int`` subclass, so reject it explicitly; ``NaN`` fails ``> 0`` and is - rejected too (``<= 0`` would let it through). + Mirrors the constructor's other up-front type checks: a value that is + non-numeric, non-finite, or too large for the platform clock would otherwise + surface later as a bare ``TypeError``/``OverflowError`` from the poll-interval + clamp (``max(_MIN_POLL_INTERVAL, default_interval)``) or a urllib socket call + (``timeout`` -> ``socket.settimeout``), escaping the module's typed-error + contract. ``bool`` is an ``int`` subclass, so reject it explicitly. ``NaN`` + fails ``> 0``; ``inf`` *passes* ``> 0`` yet overflows ``settimeout``, and a + too-large ``int`` does too — both are caught by the finite check below (which + is wrapped because ``math.isfinite`` itself raises ``OverflowError`` on an + ``int`` too large to convert to ``float``). """ - if (isinstance(value, bool) or not isinstance(value, (int, float)) - or not value > 0): + ok = isinstance(value, (int, float)) and not isinstance(value, bool) + if ok: + try: + ok = math.isfinite(value) and value > 0 + except (OverflowError, ValueError): + # A too-large int: finite in principle but unusable as a timeout + # (settimeout would raise its own OverflowError later). + ok = False + if not ok: raise OidcConfigError( - f'{name} must be a positive number of seconds, got {value!r}') + f'{name} must be a positive, finite number of seconds, ' + f'got {value!r}') class OidcDeviceAuth: diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index e3c93c65..2b2e6ecd 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -35,6 +35,7 @@ from __future__ import annotations +import re import ssl import urllib.parse from dataclasses import dataclass @@ -271,6 +272,31 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: return ep_segs[:len(base_segs)] == base_segs +# An endpoint authority that urllib's urlparse() splits differently than the +# transport (http.client) connects to. Every origin / issuer-pin / cache-key +# check derives the host from ``urlparse(url).hostname``, which strips userinfo +# at the LAST ``@`` (rpartition); urllib, however, hands the FULL netloc to the +# connection. So ``https://attacker.evil\@idp.good/token`` validates as host +# ``idp.good`` (passing the issuer-origin pin) while urllib connects to the whole +# ``attacker.evil\@idp.good``. A real credential-endpoint authority never carries +# userinfo, a backslash, whitespace or a control char, so reject them — fail +# closed — mirroring the host hygiene already enforced in +# ``_adapters._ILLEGAL_HOST_CHARS`` and ``_render._SAFE_HOST_RE``. +_UNSAFE_AUTHORITY_RE = re.compile(r'[\\\s\x00-\x1f\x7f]') + + +def _reject_confusable_authority(url: str, *, label: str) -> None: + """Reject a credential URL whose authority urllib may resolve unlike parsed.""" + netloc = safe_urlparse(url)[0].netloc + if '@' in netloc or _UNSAFE_AUTHORITY_RE.search(netloc): + raise OidcConfigError( + f'The OIDC {label} URL {url!r} has an unsafe authority (userinfo ' + "'@', a backslash, whitespace, or a control character). A real " + 'endpoint host never contains these, so this indicates a malformed ' + 'or tampered configuration; refusing to send credentials to a host ' + 'the HTTP transport may resolve differently than the one validated.') + + def validate_endpoint_origins( token_endpoint: str, device_authorization_endpoint: str) -> None: @@ -293,6 +319,14 @@ def validate_endpoint_origins( ``oauth2.googleapis.com``), so requiring issuer-origin equality there would reject a legitimate cross-origin IdP. """ + # Reject a confusable authority FIRST, so the origin comparison below (and + # every later check) can't validate a host different from the one urllib + # will connect to. Runs in OidcDeviceAuth.__init__, which every construction + # path goes through, so it covers caller-explicit and discovered endpoints. + _reject_confusable_authority(token_endpoint, label='token endpoint') + _reject_confusable_authority( + device_authorization_endpoint, + label='device-authorization endpoint') if _normalized_origin(token_endpoint) != _normalized_origin( device_authorization_endpoint): raise OidcConfigError( @@ -517,6 +551,9 @@ def resolve_config( # outside the issuer path), so pinning them would reject a legitimate IdP. # The co-location check in OidcDeviceAuth.__init__ still applies on top. if issuer: + # The pin compares each /settings endpoint's origin against the issuer's; + # a confusable issuer authority would pin to the wrong host, so vet it too. + _reject_confusable_authority(issuer, label='issuer') issuer_origin = _normalized_origin(issuer) for label, url, from_settings, confirmed_by_idp in ( ('token endpoint', token_endpoint, token_from_settings, diff --git a/test/test_auth.py b/test/test_auth.py index 46eeb71b..be7ce28c 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -838,8 +838,14 @@ def test_constructor_rejects_bad_typed_args(self): {'default_interval': 'soon'}, {'default_interval': 0}, {'default_interval': -1}, {'default_interval': True}, {'default_interval': float('nan')}, + {'default_interval': float('inf')}, {'timeout': 'slow'}, {'timeout': 0}, {'timeout': -5}, - {'timeout': True}, {'timeout': float('nan')}): + {'timeout': True}, {'timeout': float('nan')}, + # M2: inf passes ``> 0`` but crashes socket.settimeout with a + # bare OverflowError, and a too-large int does the same; both + # must raise the typed error up front, not escape from urllib. + {'timeout': float('inf')}, {'timeout': float('-inf')}, + {'timeout': 10 ** 1000}, {'default_interval': 10 ** 1000}): with self.assertRaises(OidcConfigError): OidcDeviceAuth(**{**good, **bad}) # A float interval/timeout is fine (clamped / passed to the socket). @@ -2668,6 +2674,43 @@ def test_explicit_constructor_enforces_co_location(self): token_endpoint='https://attacker.example/token', renderer=Renderer()) + def test_confusable_authority_endpoint_rejected(self): + # M1: urlparse(url).hostname (what co-location, the issuer-origin pin and + # the cache key all derive the host from) can report a DIFFERENT host + # than urllib connects to when the authority carries userinfo. E.g. + # 'https://attacker.evil\\@idp.good/token' parses with hostname + # 'idp.good' (passing the origin pin) while urllib connects to the full + # 'attacker.evil\\@idp.good'. Such an endpoint must be rejected on every + # construction path, fail-closed. + from questdb.auth._discovery import _reject_confusable_authority + for url in ('https://attacker.evil\\@idp.good/token', + 'https://idp.good@attacker.evil/token'): + with self.assertRaises(OidcConfigError): + _reject_confusable_authority(url, label='token endpoint') + with self.assertRaises(OidcConfigError): # public co-location entry + self._validate(url, url) + with self.assertRaises(OidcConfigError): # and the full constructor + OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint=url, + token_endpoint=url, + renderer=Renderer()) + # A legitimate IPv6-literal authority is NOT flagged as confusable. + self._validate('https://[::1]:443/token', 'https://[::1]:443/device') + + def test_confusable_issuer_rejected(self): + # M1 (defense-in-depth): a confusable issuer authority would make the + # issuer-origin pin compare against the wrong host. With explicit + # (caller-trusted) endpoints the pin loop is skipped, but the issuer is + # still vetted up front, so this raises. + from questdb.auth._discovery import resolve_config + with self.assertRaises(OidcConfigError): + resolve_config( + client_id='questdb', + token_endpoint='https://idp.good/token', + device_authorization_endpoint='https://idp.good/device', + issuer='https://idp.good@attacker.evil') + def test_endpoint_path_under_issuer(self): # M1: segment-aware path containment used to isolate path-based realms. from questdb.auth._discovery import _endpoint_path_under_issuer as under From 6581ad51284882782bb3f69747835d00fe7230ec Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 25 Jun 2026 21:15:45 +0100 Subject: [PATCH 076/104] fix(auth): apply minor review fixes (m1-m7) m1 cache_key docstring: list the pinned issuer among the keyed components -- it was already in the key, only the docstring omitted it. m2 MemoryCache.release(): floor the in-flight count at zero (elif remaining == 0) so a stray double-release can't prematurely reclaim a key's clear()-generation while an acquisition still holds a captured value. No live double-release exists today; guards a future caller. m3 _valid_cached(): correct the lock-free-read rationale -- the reference read's atomicity and the object's lifetime come from the CPython memory model (atomic load + free-threaded QSBR), not from TokenSet being frozen, which only covers the pointed-to object. m4 _SAFE_HOST_RE: drop '%' so a percent-encoded / zone-id verification host renders inert instead of clickable, matching the host hygiene in _adapters._ILLEGAL_HOST_CHARS. m5 _settings_url(): require an explicit http(s):// scheme and raise a clear OidcConfigError, instead of letting a bare "host:port" mis-parse into a confusing "insecure URL (scheme 'host')" much later. m6 add a SQLAlchemy IPv6 adapter test asserting the host is passed unbracketed ('::1'), matching the (already tested) psycopg path. m7 tighten test_non_string_poll_error_field_raises_typed_error to assert the specific OidcDeviceFlowError (not the base OidcError), and exercise clear() -> token() re-sign-in end-to-end with a stateful renderer. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_cache.py | 9 +++++- src/questdb/auth/_device.py | 20 ++++++++----- src/questdb/auth/_discovery.py | 8 +++++ src/questdb/auth/_render.py | 14 +++++---- test/test_auth.py | 53 +++++++++++++++++++++++++++++++--- 5 files changed, 86 insertions(+), 18 deletions(-) diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index 04cf9a61..3fcdb34f 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -164,9 +164,16 @@ def release(self, key: str) -> None: remaining = _MEMORY_INFLIGHT.get(key, 0) - 1 if remaining > 0: _MEMORY_INFLIGHT[key] = remaining - else: + elif remaining == 0: _MEMORY_INFLIGHT.pop(key, None) _MEMORY_GENERATION.pop(key, None) + # remaining < 0 means more release()s than generation() captures (a + # double-release): floor at zero and do NOT reclaim the generation — + # a concurrent acquisition may still hold a captured value to compare + # against, so reclaiming here could drop the clear()-defense. Today + # every generation() is paired with exactly one release() (the + # finally in _obtain_tokens), so this guards a future caller, not a + # path reached now. def store_if_current( self, key: str, tokens: TokenSet, generation: int) -> bool: diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 42168fdf..c96b0f0a 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -456,11 +456,11 @@ def cache_key(self) -> str: Identifies the token's security context for caching. Two sessions share a cached token only when they'd accept the same one: - same IdP token endpoint (**path included**, so multi-tenant realms on one - host don't collide), client id, scope *set* (order-insensitive), - audience, and token-kind mode (``groups_in_token`` — id_token vs - access_token). The QuestDB URL is excluded — the same IdP token is valid - against any QuestDB that trusts it. + same pinned ``issuer`` (when one is set), IdP token endpoint (**path + included**, so multi-tenant realms on one host don't collide), client id, + scope *set* (order-insensitive), audience, and token-kind mode + (``groups_in_token`` — id_token vs access_token). The QuestDB URL is + excluded — the same IdP token is valid against any QuestDB that trusts it. ``groups_in_token`` is keyed because it selects the token kind :meth:`_select` returns; otherwise two sessions differing only in that @@ -589,8 +589,14 @@ def _valid_cached(self) -> Optional[TokenSet]: # (b) self.config / self.cache_key are set once in __init__ and never # reassigned, so a token published by another thread is always for # THIS instance's security context (never a wrong-context token). - # If either is broken, this read must move under self._lock. Exercised - # under real contention by TestConcurrency.test_token_clear_stress. + # (a)/(b) concern the pointed-TO object. The atomicity of the reference + # READ itself, and the guarantee the object isn't freed between the load + # below and its use, come from the CPython memory model (an atomic pointer + # load, plus free-threaded QSBR / deferred ref-counting on a no-GIL build) + # — NOT from frozen-ness; a non-CPython runtime lacking those would need + # the lock. If any of this is broken, this read must move under + # self._lock. Exercised under real contention by + # TestConcurrency.test_token_clear_stress. tokens = self._tokens if tokens is None: tokens = self._cache.load(self.cache_key) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 2b2e6ecd..4e4f27f0 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -125,6 +125,14 @@ def _settings_url(questdb_url: str) -> str: # ".../?x=1/settings". The path is rstrip('/')-ed to avoid a double slash. # safe_urlparse maps a malformed URL to OidcConfigError, not a bare ValueError. parts, _ = safe_urlparse(questdb_url) + # Require an explicit http(s):// scheme. Without one urllib mis-parses a bare + # "host:port" — "questdb.example.com:9000" parses with scheme + # "questdb.example.com" — which would otherwise surface much later as a + # confusing "insecure URL (scheme 'questdb.example.com')" from _require_secure. + if (parts.scheme or '').lower() not in ('http', 'https'): + raise OidcConfigError( + f'The QuestDB URL {questdb_url!r} needs an explicit http(s):// ' + 'scheme, e.g. "https://questdb.example.com:9000".') path = (parts.path or '').rstrip('/') + '/settings' return urllib.parse.urlunparse( (parts.scheme, parts.netloc, path, '', '', '')) diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index eca4ca40..c8da87f8 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -88,12 +88,14 @@ def _verification_uri_complete(resp: Dict[str, Any]) -> Optional[str]: # A host safe to make clickable / auto-open: plain ASCII letters-digits-hyphen -# (a DNS name or punycode ``xn--`` label), dots, and the ``:``/``%`` an IPv6 -# literal carries once urlparse has stripped its brackets. Anything else — a -# non-ASCII confusable (e.g. a Cyrillic look-alike, or the fullwidth solidus -# ``U+FF0F``) or a stray control char in the authority — can misrepresent the -# real destination host, so such a URL is never made clickable/auto-opened. -_SAFE_HOST_RE = re.compile(r'\A[a-z0-9._%:-]+\Z') +# (a DNS name or punycode ``xn--`` label), dots, and the ``:`` an IPv6 literal +# carries once urlparse has stripped its brackets. Anything else — a non-ASCII +# confusable (e.g. a Cyrillic look-alike, or the fullwidth solidus ``U+FF0F``), +# a stray control char, or a ``%`` (percent-encoding, or an IPv6 zone-id — +# neither of which a remote verification host legitimately needs, matching the +# hygiene in ``_adapters._ILLEGAL_HOST_CHARS``) — can misrepresent the real +# destination host, so such a URL is never made clickable/auto-opened. +_SAFE_HOST_RE = re.compile(r'\A[a-z0-9._:-]+\Z') def _safe_link_url(url: Optional[str]) -> Optional[str]: diff --git a/test/test_auth.py b/test/test_auth.py index be7ce28c..df91e820 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -331,7 +331,7 @@ def tearDown(self): self.thread.join(timeout=5) def make_auth(self, *, clock=None, groups_in_token=True, - interactive=True, **kw): + interactive=True, renderer=None, **kw): clock = clock or FakeClock() self._clock = clock return OidcDeviceAuth( @@ -342,7 +342,7 @@ def make_auth(self, *, clock=None, groups_in_token=True, groups_in_token=groups_in_token, insecure=True, interactive=interactive, - renderer=Renderer(), + renderer=renderer if renderer is not None else Renderer(), _clock=clock, **kw) @@ -577,7 +577,7 @@ def test_non_string_poll_error_field_raises_typed_error(self): (400, {'error': {'nested': 'obj'}, 'error_description': ['a', 'list']})] auth = self.make_auth() - with self.assertRaises(OidcError): # typed, not TypeError + with self.assertRaises(OidcDeviceFlowError): # the specific typed error auth.token() def test_missing_verification_uri_is_rejected(self): @@ -795,12 +795,23 @@ def test_access_token_headers(self): {'Authorization': 'Bearer ' + ACCESS_TOKEN}) def test_clear_forces_resignin(self): - auth = self.make_auth() + # A stateful renderer confirms the prompt is drawn END-TO-END on the + # second sign-in (not merely that the device endpoint is hit again): + # clear() then token() must re-run on_prompt, exercising the renderer's + # own re-sign-in reset path too. + prompts = [] + + class _CountingRenderer(Renderer): + def on_prompt(self, resp): + prompts.append(resp.get('user_code')) + + auth = self.make_auth(renderer=_CountingRenderer()) auth.token() self.assertEqual(self.state.device_requests, 1) auth.clear() auth.token() self.assertEqual(self.state.device_requests, 2) # prompted again + self.assertEqual(len(prompts), 2) # renderer saw both def test_openid_scope_auto_added_for_groups_in_token(self): # groups-in-token requires an id_token, which needs the openid scope. @@ -2314,6 +2325,40 @@ def create(**kw): self.assertEqual(cparams['password'], 'TKN') self.assertEqual(auth.calls - before, 2) + def test_sqlalchemy_engine_uses_bare_ipv6_host(self): + # m6: SQLAlchemy's URL.create takes host and port separately and hands + # the host to the driver as a connect kwarg (not a "host:port" DSN + # string), so the host must be the UNBRACKETED IPv6 literal '::1' — + # exactly as the (separately tested) psycopg path passes it. Asserted at + # the adapter boundary; real SQLAlchemy is not a test dependency. + created = {} + fake_sa = types.ModuleType('sqlalchemy') + fake_sa.__path__ = [] + fake_sa.create_engine = lambda url, **kw: object() + + class _Event: + @staticmethod + def listens_for(target, name): + return lambda fn: fn + + fake_sa.event = _Event + fake_eng = types.ModuleType('sqlalchemy.engine') + + class _URL: + @staticmethod + def create(**kw): + created.update(kw) + return 'URL' + + fake_eng.URL = _URL + with mock.patch.dict(sys.modules, { + 'sqlalchemy': fake_sa, + 'sqlalchemy.engine': fake_eng, + 'psycopg': types.ModuleType('psycopg')}): + sqlalchemy_engine(_FakeAuth(), 'https://[::1]:9000') + self.assertEqual(created['host'], '::1') # unbracketed, like psycopg + self.assertEqual(created['port'], 8812) + def test_sqlalchemy_engine_uses_psycopg2_drivername(self): # When only psycopg2 (v2) is importable, the SQLAlchemy driver name is # postgresql+psycopg2, not +psycopg (the v3 branch). From 623b983fa447e73b55e69406f61783cbc4452d2f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 26 Jun 2026 00:14:46 +0100 Subject: [PATCH 077/104] fix(auth): detect non-interactive notebook executors; review nits Auto-detect papermill / nbclient / nbconvert --execute via the kernel's allow_stdin (they execute with allow_stdin=False), so token() fails fast with OidcInteractionRequired instead of polling to the device-code deadline. papermill sets no environment variable, so the kernel stdin flag is the authoritative signal; default to interactive when it can't be read so a present human is never wrongly refused. Plus minor review fixes: - _coerce_port: catch OverflowError (non-finite pg_port) -> OidcConfigError - poll loop: honor Retry-After on a transient 5xx, not just 429 - poll loop: treat a JSON-bodied 3xx as terminal, not a live poll state - _maybe_open_browser: per-field _safe_target fallback so the opened URL can't diverge from the displayed link / QR - _endpoint_path_under_issuer: reject non-ASCII (homoglyph-dot) segments - freeze OidcConfig (the lock-free fast path relies on its immutability) - OidcDeviceFlowError now carries .status - harden on_success against inf/nan; correct stale comments/docstrings - tests: assert the mock-server thread shuts down; cover all of the above Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_adapters.py | 5 +- src/questdb/auth/_cache.py | 4 +- src/questdb/auth/_device.py | 69 ++++++++++++---- src/questdb/auth/_discovery.py | 19 ++++- src/questdb/auth/_errors.py | 8 +- src/questdb/auth/_render.py | 60 ++++++++++++-- test/test_auth.py | 146 ++++++++++++++++++++++++++++++++- 7 files changed, 278 insertions(+), 33 deletions(-) diff --git a/src/questdb/auth/_adapters.py b/src/questdb/auth/_adapters.py index beb21deb..4dccf7bd 100644 --- a/src/questdb/auth/_adapters.py +++ b/src/questdb/auth/_adapters.py @@ -113,7 +113,10 @@ def _coerce_port(pg_port: Any) -> int: f'pg_port must be an integer port number, got {pg_port!r}.') try: port = int(pg_port) - except (TypeError, ValueError) as e: + except (TypeError, ValueError, OverflowError) as e: + # int(float('inf')) / int(1e400) raise OverflowError (not ValueError), + # so catch it too — else a non-finite pg_port escapes the typed-error + # contract as a bare OverflowError (mirrors _validate_positive_number). raise OidcConfigError( f'pg_port must be an integer port number, got {pg_port!r}.') from e if not 1 <= port <= 65535: diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index 3fcdb34f..baad8cb1 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -95,7 +95,9 @@ class MemoryCache: """ def load(self, key: str) -> Optional[TokenSet]: - # Return a copy so callers can't mutate the cached entry in place. + # TokenSet is frozen, so a caller can't mutate it anyway; the replace() + # copy is defensive — it hands back a distinct instance and keeps the + # contract correct should the dataclass ever lose frozen. with _MEMORY_LOCK: tokens = _MEMORY_STORE.get(key) return replace(tokens) if tokens is not None else None diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index c96b0f0a..9ca639e8 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -50,6 +50,8 @@ from ._render import ( Renderer, _safe_target, + _verification_uri, + _verification_uri_complete, detect_interactive, in_ipython_kernel, make_renderer, @@ -235,7 +237,9 @@ class OidcDeviceAuth: never blocks, but one whose token is missing/expired waits behind the signer. So when threads share an auth object (e.g. a SQLAlchemy/psycopg pool), sign in once up front — call :meth:`token` once on the main thread - before the pool opens connections. + before the pool opens connections. (A custom ``renderer``'s callbacks run + while this lock is held, so they must not call back into the same instance's + :meth:`token` / :meth:`clear` — the lock is not reentrant.) .. code-block:: python @@ -346,9 +350,10 @@ def __init__( self._interactive = interactive self._default_interval = default_interval # Per-request network timeout for every IdP call (device-code, each poll, - # refresh). Bounds how long one network leg pins the acquisition lock if - # the IdP stalls; the total poll duration is separately capped by - # _MAX_DEVICE_CODE_LIFETIME. + # refresh). Applied to connect+headers and then again to the body read, + # so one network leg can pin the acquisition lock for up to ~2x this + # value if the IdP stalls; the total poll duration is separately capped + # by _MAX_DEVICE_CODE_LIFETIME. self._timeout = timeout self._cache = MemoryCache() self._ctx = build_ssl_context(ca_bundle) @@ -396,6 +401,13 @@ def from_questdb( groups mode, falling back to the IdP ``.well-known`` document for the device-authorization endpoint when QuestDB doesn't advertise it. Any explicit keyword overrides discovery. + + When the server does not advertise the device-authorization endpoint (so + it must be discovered from the IdP), ``issuer=`` is **required** to pin + the identity provider — the helper refuses to derive the discovery origin + from a server-supplied endpoint, so a tampered ``/settings`` cannot + redirect the device-code / refresh-token POSTs. See :ref:`oidc_auth`. + Raises :class:`OidcConfigError` if the configuration can't be resolved. """ # Validate before resolve_config consumes `timeout` on its /settings and # discovery HTTP calls (which run before cls() would validate it), so a @@ -749,6 +761,7 @@ def _refresh(self, tokens: TokenSet) -> TokenSet: 'the refresh token is still valid — retry later.') raise OidcDeviceFlowError( f"Token refresh failed: {body.get('error', 'unknown error')}", + status=status, error=body.get('error'), error_description=body.get('error_description')) @@ -819,6 +832,7 @@ def _request_device_code(self) -> Dict[str, Any]: 'The IdP returned a 200 device-authorization response missing a ' 'required field (device_code, user_code, or verification_uri); ' 'cannot start the device flow.', + status=status, error=error, error_description=body.get('error_description')) if status in (400, 404, 405) or error in ( @@ -830,11 +844,13 @@ def _request_device_code(self) -> Dict[str, Any]: f'{self.config.client_id!r} has the device grant ' "('urn:ietf:params:oauth:grant-type:device_code') enabled and " 'is registered as a public client.', + status=status, error=error, error_description=body.get('error_description')) raise OidcDeviceFlowError( f'Device authorization request failed (HTTP {status}): ' f'{body.get("error_description") or error or body}', + status=status, error=error, error_description=body.get('error_description')) @@ -891,7 +907,8 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: 'request.') raise OidcDeviceFlowError( f'Device flow failed: the IdP rejected the token ' - f'request ({e}).') from e + f'request ({e}).', + status=getattr(e, 'status', None)) from e # Otherwise transient: a dropped connection / DNS blip / timeout # (OidcNetworkError) or a non-JSON 5xx/429 from a proxy (bare # OidcError). The user may already have authorized, and RFC 8628 @@ -927,13 +944,30 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: raise self._missing_required_token_error() # A 5xx/429 with a JSON body is also transient (server error or - # rate-limit), not a terminal rejection: back off on 429 and keep - # polling until the deadline, as above. + # rate-limit), not a terminal rejection: keep polling until the + # deadline. Honor a Retry-After on either a 429 or a transient 5xx + # (as _PostResult documents); apply the RFC 8628 §3.5 +5s slow-down + # step only to a 429 with no header (a generic 5xx is a server error, + # not a rate-limit, so its cadence is unchanged absent a Retry-After). if status >= 500 or status == 429: - if status == 429: + if status == 429 or retry_after is not None: interval = _backoff_interval(interval, retry_after) continue + # A 3xx with a JSON body is still a redirect these endpoints never + # legitimately return (and _NoRedirect won't follow): treat it as + # terminal, like the non-JSON 3xx the exception path above rejects, so + # a proxy/WAF returning a JSON-bodied redirect that happens to carry + # an OAuth error field can't be mistaken for a live poll state and + # polled on to a misleading "code expired". + if 300 <= status < 400: + self._renderer.on_failure( + 'Sign-in failed: the identity provider rejected the request.') + raise OidcDeviceFlowError( + 'Device flow failed: the IdP returned an unexpected ' + f'redirect (HTTP {status}).', + status=status) + error = body.get('error') if error == 'authorization_pending': continue @@ -952,6 +986,7 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: self._renderer.on_failure(f'Sign-in failed: {description}') raise OidcDeviceFlowError( f'Device flow failed: {description}', + status=status, error=error, error_description=body.get('error_description')) @@ -968,14 +1003,16 @@ def _maybe_open_browser(self, resp: Dict[str, Any]) -> None: # kernel host isn't the user's machine. Suppress with open_browser=False. if not self.open_browser or in_ipython_kernel(): return - # Open the SAME _strip_control'd, vetted target the prompt shows (via - # _safe_target) — not the raw response value — so a char stripped from the - # on-screen link can't survive into the opened URL, and a javascript:/ - # data: scheme (or userinfo / non-ASCII host) is never opened. - target = _safe_target( - resp.get('verification_uri_complete') - or resp.get('verification_uri') - or resp.get('verification_url')) + # Open the SAME _strip_control'd, vetted target the prompt shows — not the + # raw response value — so a char stripped from the on-screen link can't + # survive into the opened URL, and a javascript:/data: scheme (or + # userinfo / non-ASCII host) is never opened. Vet each field + # independently and fall back, exactly as the renderers do (_safe_target + # per field, complete-then-plain), so a truthy-but-unsafe + # verification_uri_complete can't shadow a usable verification_uri and + # make the browser diverge from the displayed link / QR. + target = (_safe_target(_verification_uri_complete(resp)) + or _safe_target(_verification_uri(resp))) if target: try: webbrowser.open(target) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 4e4f27f0..87a3c09c 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -54,9 +54,15 @@ _K_AUDIENCE = 'acl.oidc.audience' -@dataclass +@dataclass(frozen=True) class OidcConfig: - """Resolved OIDC parameters needed to run the device flow.""" + """Resolved OIDC parameters needed to run the device flow. + + ``frozen`` because :class:`~questdb.auth._device.OidcDeviceAuth` reads + ``self.config`` (and the ``cache_key`` derived from it) on the lock-free + fast path; the fields are set once at construction and never reassigned, so + freezing makes that immutability structural rather than convention. + """ client_id: str token_endpoint: str @@ -271,10 +277,15 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: # A residual '%' means the path did not fully decode within the bounded loop # (an over-deeply multiply-encoded escape) or is a malformed escape; a server # may yet decode it further to a dot-segment, so fail closed rather than - # prefix-match a value that could still resolve to a different path. - # Legitimate credential-endpoint paths are plain ASCII with no encoding. + # prefix-match a value that could still resolve to a different path. A + # non-ASCII segment is rejected for the same reason: a homoglyph dot + # (e.g. fullwidth U+FF0E '．．') is not literally '..' here, yet a server that + # NFKC-normalizes the path before dot-segment removal could fold it to a real + # '..' and traverse to a different tenant. Legitimate credential-endpoint + # paths are plain ASCII with no encoding. if ('.' in ep_segs or '..' in ep_segs or any('%' in s for s in ep_segs) + or any(not s.isascii() for s in ep_segs) or any(_has_control_char(s) for s in ep_segs)): return False return ep_segs[:len(base_segs)] == base_segs diff --git a/src/questdb/auth/_errors.py b/src/questdb/auth/_errors.py index dcb67689..8c091edf 100644 --- a/src/questdb/auth/_errors.py +++ b/src/questdb/auth/_errors.py @@ -87,8 +87,12 @@ def __init__( message: str, *, error: Optional[str] = None, - error_description: Optional[str] = None): - super().__init__(message) + error_description: Optional[str] = None, + status: Optional[int] = None): + # Forward status to OidcError so a device-flow error raised in response + # to a known HTTP status carries it (e.g. for a caller inspecting + # err.status), rather than always reporting None. + super().__init__(message, status=status) # error / error_description come straight from the untrusted IdP # response and are exposed as attributes (a caller may re-display them), # so strip them too — same rationale as the message in OidcError. Coerce diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index c8da87f8..a5208400 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -56,15 +56,54 @@ def in_ipython_kernel() -> bool: 'ZMQInteractiveShell', 'TerminalInteractiveShell') +def _kernel_allows_stdin() -> bool: + """ + True if the live IPython kernel can prompt a human for input. + + A real Jupyter frontend (Lab / Notebook / VS Code / qtconsole / jupyter + console) issues each ``execute_request`` with ``allow_stdin=True``. A + notebook *executor* — papermill, ``nbclient``, ``jupyter nbconvert + --execute`` / ``jupyter run`` — runs the same kind of kernel (so + :func:`in_ipython_kernel` is ``True``) but sends ``allow_stdin=False``: there + is no human to authorize, and an input request would raise + ``StdinNotImplementedError``. ipykernel records the current request's value + on the kernel as ``_allow_stdin``; read it so the device flow fails fast with + :class:`~questdb.auth._errors.OidcInteractionRequired` instead of polling to + the device-code deadline. papermill sets no environment variable (its + ``PAPERMILL_*_PATH`` values are opt-in *notebook parameters*, not + ``os.environ`` entries), so the kernel's stdin flag — not an env var — is the + authoritative signal. + + Defaults to ``True`` (assume a human is present) whenever the signal can't be + read: a terminal IPython shell has no ``kernel`` attribute, and on an + unexpected ipykernel layout it is safer to let a present user sign in than to + wrongly refuse one. + """ + try: + from IPython import get_ipython # type: ignore + kernel = getattr(get_ipython(), 'kernel', None) + if kernel is None: + return True # e.g. TerminalInteractiveShell — a human at the REPL + allow = getattr(kernel, '_allow_stdin', None) + return True if allow is None else bool(allow) + except Exception: + return True + + def detect_interactive() -> bool: """ Best-effort detection of whether a human can complete the sign-in. - Interactive when attached to a TTY or an interactive IPython shell; guards - against hanging forever in a non-interactive context (papermill/cron/CI). + Interactive when attached to a TTY, or inside an IPython kernel whose + frontend accepts stdin. A notebook executor (papermill / ``nbclient`` / + ``nbconvert --execute``) runs a real kernel — so :func:`in_ipython_kernel` + is ``True`` — but with no human to authorize; it executes with + ``allow_stdin=False``, which :func:`_kernel_allows_stdin` detects, so the + device flow fails fast in those contexts instead of hanging until the + device code expires. """ if in_ipython_kernel(): - return True + return _kernel_allows_stdin() try: return bool(sys.stdin and sys.stdin.isatty() and sys.stdout and sys.stdout.isatty()) @@ -290,6 +329,17 @@ def _fmt_mmss(seconds: float) -> str: return f'{seconds // 60}:{seconds % 60:02d}' +def _fmt_minutes(seconds: float) -> int: + # Minutes for the "expires in N min" success line, with the same non-finite + # guard as _fmt_mmss: a hostile/garbage lifetime (inf/nan) would otherwise + # make int(round(...)) raise (OverflowError/ValueError) inside on_success and + # break the sign-in at the last step. Callers pass a clamped, finite value + # today (see _device._display_lifetime), so this is defense-in-depth. + if not math.isfinite(seconds): + return 1 + return max(1, int(round(seconds / 60))) + + class Renderer: """No-op renderer interface; subclasses present the prompt to the user.""" @@ -352,7 +402,7 @@ def on_success(self, identity: Optional[str], expires_in: float) -> None: self._write('\n') self._countdown_active = False who = f' as {_strip_control(identity)}' if identity else '' - mins = max(1, int(round(expires_in / 60))) + mins = _fmt_minutes(expires_in) self._write(f'✅ Signed in{who} — token cached, expires in {mins} min\n') def on_failure(self, message: str) -> None: @@ -467,7 +517,7 @@ def on_waiting(self, seconds_left: float) -> None: def on_success(self, identity: Optional[str], expires_in: float) -> None: # identity comes from untrusted JWT claims: strip then html-escape. who = html.escape(_strip_control(identity)) if identity else '' - mins = max(1, int(round(expires_in / 60))) + mins = _fmt_minutes(expires_in) suffix = f' as {who}' if who else '' self._render_with_status( f'✅ Signed in{suffix} — token cached, expires in {mins} min', diff --git a/test/test_auth.py b/test/test_auth.py index df91e820..562b947b 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -46,6 +46,7 @@ import unittest import http.server import urllib.parse +from dataclasses import replace from unittest import mock sys.dont_write_bytecode = True @@ -329,6 +330,11 @@ def tearDown(self): self.server.shutdown() self.server.server_close() self.thread.join(timeout=5) + # Assert the server thread actually terminated: a join() that times out + # silently leaks a thread, so a future deadlock regression would still + # "pass" instead of failing here. + self.assertFalse(self.thread.is_alive(), + 'mock server thread did not shut down within 5s') def make_auth(self, *, clock=None, groups_in_token=True, interactive=True, renderer=None, **kw): @@ -474,8 +480,9 @@ def test_non_json_4xx_during_poll_is_terminal(self): # Point only the poll (token) endpoint at the non-JSON 403; the # device-code request still hits the JSON mock IdP. Set it post- # construction so the (already-satisfied) co-location check isn't - # re-run against the throwaway origin. - auth.config.token_endpoint = raw + '/token' + # re-run against the throwaway origin. OidcConfig is frozen, so + # rebuild it via replace() rather than mutating in place. + auth.config = replace(auth.config, token_endpoint=raw + '/token') with self.assertRaises(OidcDeviceFlowError) as cm: auth.token() # Terminal on the first poll: not a timeout, and it did not keep polling @@ -492,7 +499,26 @@ def test_non_json_3xx_during_poll_is_terminal(self): auth = self.make_auth() with _raw_response_server( 302, 'text/html', b'see /login') as raw: - auth.config.token_endpoint = raw + '/token' # post-construction + # OidcConfig is frozen; rebuild it rather than mutating in place. + auth.config = replace(auth.config, token_endpoint=raw + '/token') + with self.assertRaises(OidcDeviceFlowError) as cm: + auth.token() + self.assertNotIsInstance(cm.exception, OidcTimeoutError) + self.assertLessEqual(len(self._clock.sleeps), 1) + + def test_json_3xx_during_poll_is_terminal(self): + # A 3xx whose body IS valid JSON (a proxy/WAF redirect that happens to + # carry an OAuth-looking error field) must fail fast too: _NoRedirect + # refuses the redirect and post_form returns (3xx, {...}), which the poll + # loop must classify terminal rather than mistake the embedded + # authorization_pending for a live poll state and poll on to "code + # expired". (The non-JSON 3xx is covered above via the exception path; + # this exercises the JSON-body path inside the loop.) + auth = self.make_auth() + with _raw_response_server( + 302, 'application/json', + b'{"error": "authorization_pending"}') as raw: + auth.config = replace(auth.config, token_endpoint=raw + '/token') with self.assertRaises(OidcDeviceFlowError) as cm: auth.token() self.assertNotIsInstance(cm.exception, OidcTimeoutError) @@ -568,6 +594,17 @@ def test_idp_expired_token_error_raises_timeout(self): auth.token() self.assertEqual(cm.exception.error, 'expired_token') + def test_device_flow_error_carries_http_status(self): + # An OidcDeviceFlowError raised in response to a known HTTP status now + # carries it on .status (forwarded to the OidcError base) instead of + # always reporting None, so a caller can inspect err.status. + self.state.device_status = 400 + self.state.device_response = {'error': 'invalid_client'} + auth = self.make_auth() + with self.assertRaises(OidcDeviceFlowError) as cm: + auth.token() + self.assertEqual(cm.exception.status, 400) + def test_non_string_poll_error_field_raises_typed_error(self): # M1 (end-to-end): a non-conformant/hostile IdP can answer the token poll # with a non-string error / error_description (a JSON object/array). @@ -647,6 +684,28 @@ def fake(url, form): self.assertEqual(auth.token(), ID_TOKEN) self.assertIn(30, self._clock.sleeps) # honored Retry-After, not +5 + def test_poll_honors_retry_after_on_5xx(self): + # A transient 5xx poll response carrying Retry-After backs off by that + # many seconds (clamped) — as _PostResult documents for 429/503, not just + # 429. (The +5s slow-down step stays 429/slow_down-only; a 5xx without a + # Retry-After keeps its cadence.) + from questdb.auth._http import _PostResult + auth = self.make_auth() + real = auth._idp_post + n = {'tok': 0} + + def fake(url, form): + if url.endswith('/token'): + n['tok'] += 1 + if n['tok'] == 1: + return _PostResult(503, {'error': 'server_error'}, 30) + return real(url, form) # then succeed + return real(url, form) + + auth._idp_post = fake + self.assertEqual(auth.token(), ID_TOKEN) + self.assertIn(30, self._clock.sleeps) # honored Retry-After on a 5xx + def test_nonpositive_expires_in_still_polls(self): # A non-positive expires_in in the device-auth response must be treated # as unknown, not as "already expired" — otherwise the flow times out @@ -1116,6 +1175,18 @@ def test_open_browser_rejects_dangerous_scheme(self): {'verification_uri': 'https://idp.example.com/device'}) opener.assert_called_once_with('https://idp.example.com/device') + def test_open_browser_falls_back_past_unsafe_complete(self): + # A truthy-but-unsafe verification_uri_complete must not shadow a usable + # verification_uri: each field is vetted independently (complete-then- + # plain), so the browser opens the SAME safe target the prompt and QR + # show, instead of opening nothing — the link/browser/QR can't diverge. + auth = self.make_auth(open_browser=True) + with mock.patch('webbrowser.open') as opener: + auth._maybe_open_browser({ + 'verification_uri_complete': 'javascript:alert(1)', + 'verification_uri': 'https://idp.example.com/device'}) + opener.assert_called_once_with('https://idp.example.com/device') + def test_open_browser_default_is_true(self): # We try to open the browser by default ("always when possible"), via # both the explicit constructor and discovery. @@ -1223,6 +1294,23 @@ def test_non_interactive_raises_without_polling(self): auth.token() self.assertEqual(self.state.device_requests, 0) + def test_papermill_kernel_fails_fast(self): + # End-to-end auto-detection (no explicit interactive= override): a + # papermill-style kernel (a real kernel, but allow_stdin=False) makes + # token() raise OidcInteractionRequired immediately — no device request, + # no poll — rather than hanging until the device code expires. + from questdb.auth import _render + auth = self.make_auth(interactive=None) # fall through to auto-detection + fake_ip = types.ModuleType('IPython') + fake_ip.get_ipython = lambda: types.SimpleNamespace( + kernel=types.SimpleNamespace(_allow_stdin=False)) + with mock.patch.object(_render, 'in_ipython_kernel', return_value=True), \ + mock.patch.dict(sys.modules, {'IPython': fake_ip}): + with self.assertRaises(OidcInteractionRequired): + auth.token() + self.assertEqual(self.state.device_requests, 0) + self.assertEqual(self.state.token_requests, []) + class TestRefresh(AuthTestBase): def _seed_expired(self, auth, refresh_token='REFRESH-1'): @@ -2438,7 +2526,10 @@ def test_pg_port_validation(self): # from URL.create(port=...) / connect(port=...). The check runs before # the driver import, so it holds even without sqlalchemy / psycopg. from questdb.auth._adapters import _coerce_port - for bad in ('not-a-port', None, '88a2', True, 0, 70000): + # float('inf')/1e400 raise OverflowError (not ValueError) from int(); + # float('nan') raises ValueError. Both must map to OidcConfigError. + for bad in ('not-a-port', None, '88a2', True, 0, 70000, -1, + float('inf'), float('-inf'), float('nan'), 1e400): with self.subTest(pg_port=bad): with self.assertRaises(OidcConfigError): _coerce_port(bad) @@ -2814,6 +2905,17 @@ def test_endpoint_path_under_issuer(self): self.assertFalse(under(iss + '/..%7f/EVIL/token', iss)) # ..DEL # A printable-ASCII segment with an internal space (%20) is still fine. self.assertTrue(under(iss + '/ok%20name/token', iss)) + # A non-ASCII homoglyph dot segment is rejected too: a fullwidth U+FF0E + # '．．' (literal, its %-encoded UTF-8, or the ideographic U+3002) is not + # literally '..' here, yet a server that NFKC-normalizes the path before + # dot-segment removal could fold it to a real '..' and reach a different + # realm. Legitimate credential-endpoint paths are plain ASCII (chr() keeps + # the confusables out of the test source). + fw_dot = chr(0xff0e) * 2 # fullwidth '．．' + self.assertFalse(under(iss + '/' + fw_dot + '/EVIL/token', iss)) + self.assertFalse(under(iss + '/%ef%bc%8e%ef%bc%8e/EVIL/token', iss)) + self.assertFalse( # ideographic full stop + under(iss + '/' + chr(0x3002) * 2 + '/EVIL/token', iss)) class TestCacheKey(unittest.TestCase): @@ -3535,6 +3637,42 @@ def test_detect_interactive_requires_tty(self): so.isatty.return_value = False # stdout not a tty self.assertFalse(_render.detect_interactive()) + def test_kernel_without_stdin_is_noninteractive(self): + # papermill / nbclient / nbconvert --execute run a real kernel + # (in_ipython_kernel True) but execute with allow_stdin=False — there is + # no human to authorize. detect_interactive must report non-interactive + # so the device flow fails fast instead of polling to the device-code + # deadline; a real Jupyter frontend sends allow_stdin=True (interactive). + # papermill sets no env var, so the kernel stdin flag is the signal. + from questdb.auth import _render + + def fake_ipython(kernel): + mod = types.ModuleType('IPython') + mod.get_ipython = lambda: types.SimpleNamespace(kernel=kernel) + return mod + + with mock.patch.object(_render, 'in_ipython_kernel', return_value=True): + # papermill / nbclient / nbconvert: allow_stdin False -> fail fast. + with mock.patch.dict(sys.modules, {'IPython': fake_ipython( + types.SimpleNamespace(_allow_stdin=False))}): + self.assertFalse(_render._kernel_allows_stdin()) + self.assertFalse(_render.detect_interactive()) + # Real Jupyter frontend: allow_stdin True -> interactive. + with mock.patch.dict(sys.modules, {'IPython': fake_ipython( + types.SimpleNamespace(_allow_stdin=True))}): + self.assertTrue(_render._kernel_allows_stdin()) + self.assertTrue(_render.detect_interactive()) + # Signal unreadable -> assume a human is present (never wrongly refuse + # one): terminal IPython has no .kernel; a kernel may lack the attr; + # and IPython may fail to import entirely. + with mock.patch.dict(sys.modules, {'IPython': fake_ipython(None)}): + self.assertTrue(_render._kernel_allows_stdin()) + with mock.patch.dict(sys.modules, {'IPython': fake_ipython( + types.SimpleNamespace())}): + self.assertTrue(_render._kernel_allows_stdin()) + with mock.patch.dict(sys.modules, {'IPython': None}): + self.assertTrue(_render._kernel_allows_stdin()) + def test_in_ipython_kernel_false_without_ipython(self): # When IPython can't be imported (plain CPython), it's not a kernel. from questdb.auth import _render From e78de248d4c5a73d653662b1e3c985c737b3a94a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 26 Jun 2026 13:04:00 +0100 Subject: [PATCH 078/104] fix(auth): bound chunked reads and keep errors typed Four review fixes to the questdb.auth device-flow client: - C1: _read_body could hang ~indefinitely on a chunked Transfer-Encoding slow-loris. read1() parks inside http.client's readline() on the chunk-size line, so the between-reads wall-clock deadline never ran and the per-socket timeout kept resetting -- pinning the calling thread (and the acquisition lock) for hours. Add a watchdog that shuts the socket down at the deadline; a post-deadline empty read is now treated as a timeout, not a clean EOF (which would silently truncate the body). - M1/M2: a lone surrogate in a form field, or a non-ASCII URL host, escaped request() as a raw UnicodeEncodeError. Move request-building inside the try and map UnicodeError to OidcConfigError; also reject a non-ASCII authority up front in _reject_confusable_authority (closing the homoglyph gap and giving a clear "use xn-- punycode" message). - M3: a hostile JWT exp (a huge int overflowing float()) or a raising custom renderer could abort an already-completed sign-in, discarding the token and re-prompting on every later call. Make the cosmetic success-message rendering best-effort and guard float(exp). Adds regression tests for each (chunked dribble, unencodable request, non-ASCII authority, hostile exp, raising renderer); full questdb.auth suite green (213 tests). Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 31 +++++-- src/questdb/auth/_discovery.py | 27 ++++-- src/questdb/auth/_http.py | 156 ++++++++++++++++++++++++++------- test/test_auth.py | 137 +++++++++++++++++++++++++++++ 4 files changed, 304 insertions(+), 47 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 9ca639e8..74b928ab 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -779,10 +779,21 @@ def _run_device_flow(self) -> TokenSet: self._renderer.on_prompt(resp) self._maybe_open_browser(resp) tokens = self._poll_for_token(resp) - claims = (_decode_jwt_claims(tokens.id_token) - or _decode_jwt_claims(tokens.access_token)) - identity = _identity_from_claims(claims) - self._renderer.on_success(identity, self._display_lifetime(tokens, claims)) + # The grant has completed and `tokens` is valid. Rendering the success + # message (identity + remaining lifetime) is purely cosmetic and must + # NEVER abort an authorized sign-in: a hostile JWT (e.g. a non-finite or + # huge `exp`) or a custom renderer that raises would otherwise discard a + # token the user already authorized — and, since _store runs only after + # this returns, force a fresh prompt (and re-crash) on every later + # token() call. Best-effort, mirroring _maybe_open_browser. + try: + claims = (_decode_jwt_claims(tokens.id_token) + or _decode_jwt_claims(tokens.access_token)) + identity = _identity_from_claims(claims) + self._renderer.on_success( + identity, self._display_lifetime(tokens, claims)) + except Exception: + pass return tokens def _display_lifetime( @@ -792,12 +803,16 @@ def _display_lifetime( # re-validated at least hourly — reporting that would under-state a token # that genuinely lives longer. Prefer the JWT `exp` claim (the # authoritative token expiry); fall back to the clamped value for an - # opaque token with no exp. Bounded to ~1y so a hostile/garbage exp - # (incl. inf/nan, which fail the comparison) can't overflow on_success's - # int(round(...)). + # opaque token with no exp. Bounded to ~1y, with the float() conversion + # guarded, so a hostile/garbage exp can't break on_success's + # int(round(...)): inf/nan fail the bound below, and a huge int (e.g. + # 10**400) that overflows float() is caught. exp = claims.get('exp') if isinstance(exp, (int, float)) and not isinstance(exp, bool): - remaining = float(exp) - self._now() + try: + remaining = float(exp) - self._now() + except (OverflowError, ValueError): + remaining = -1.0 # fall through to the clamped token expiry if 0 < remaining <= 366 * 24 * 3600: return remaining return max(0.0, tokens.expires_at - self._now()) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 87a3c09c..fe3c3ab8 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -298,22 +298,33 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: # connection. So ``https://attacker.evil\@idp.good/token`` validates as host # ``idp.good`` (passing the issuer-origin pin) while urllib connects to the whole # ``attacker.evil\@idp.good``. A real credential-endpoint authority never carries -# userinfo, a backslash, whitespace or a control char, so reject them — fail -# closed — mirroring the host hygiene already enforced in -# ``_adapters._ILLEGAL_HOST_CHARS`` and ``_render._SAFE_HOST_RE``. +# userinfo, a non-ASCII character, a backslash, whitespace or a control char, so +# reject them — fail closed — mirroring the host hygiene already enforced in +# ``_adapters._ILLEGAL_HOST_CHARS`` and ``_render._SAFE_HOST_RE``. (Non-ASCII is +# checked with ``str.isascii`` in the function, not this regex.) _UNSAFE_AUTHORITY_RE = re.compile(r'[\\\s\x00-\x1f\x7f]') def _reject_confusable_authority(url: str, *, label: str) -> None: """Reject a credential URL whose authority urllib may resolve unlike parsed.""" netloc = safe_urlparse(url)[0].netloc - if '@' in netloc or _UNSAFE_AUTHORITY_RE.search(netloc): + # A non-ASCII authority is rejected too: a real endpoint host is plain ASCII + # (a DNS name, an xn-- punycode label, or an IP literal), a raw non-ASCII host + # is a homoglyph/confusable spoofing vector (the renderer already distrusts + # it for display), AND http.client can't even encode it — it would otherwise + # raise a raw UnicodeEncodeError from the transport rather than a typed error. + if ('@' in netloc or not netloc.isascii() + or _UNSAFE_AUTHORITY_RE.search(netloc)): raise OidcConfigError( f'The OIDC {label} URL {url!r} has an unsafe authority (userinfo ' - "'@', a backslash, whitespace, or a control character). A real " - 'endpoint host never contains these, so this indicates a malformed ' - 'or tampered configuration; refusing to send credentials to a host ' - 'the HTTP transport may resolve differently than the one validated.') + "'@', a non-ASCII character, a backslash, whitespace, or a control " + 'character). A real endpoint host is plain ASCII (a DNS name, an ' + 'xn-- punycode label, or an IP literal) and never contains these, so ' + 'this indicates a malformed or tampered configuration; refusing to ' + 'send credentials to a host the HTTP transport may resolve ' + 'differently than the one validated, or that a confusable/homoglyph ' + 'host could misrepresent. Pass the punycode (xn--) form for an ' + 'internationalized domain.') def validate_endpoint_origins( diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 3d826fa7..59314a73 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -41,7 +41,9 @@ import ipaddress import json import os +import socket import ssl +import threading import time import urllib.error import urllib.parse @@ -182,6 +184,39 @@ def _opener(ctx: Optional[ssl.SSLContext]) -> urllib.request.OpenerDirector: return urllib.request.build_opener(*handlers) +def _underlying_socket(resp: Any): + """ + Best-effort: the raw socket behind an http.client response / ``HTTPError``. + + The deadline watchdog in :func:`_read_body` uses it to break a read that is + blocked past the deadline. Returns ``None`` if the socket can't be located + (a non-socket stream, or an unexpected stdlib layout) — the caller then + relies on the between-reads deadline check alone (the pre-watchdog + behavior), so a layout change degrades safely rather than crashing. + """ + obj = resp + # HTTPError -> HTTPResponse -> BufferedReader (.raw is a SocketIO) -> socket. + for _ in range(5): + if obj is None: + break + sock = (getattr(getattr(obj, 'raw', None), '_sock', None) + or getattr(obj, '_sock', None)) + if sock is not None: + return sock + obj = getattr(obj, 'fp', None) + return None + + +def _shutdown_socket(sock: Any) -> None: + # Force a read blocked past the deadline to return/raise. shutdown() (not + # close()) is what actually unblocks a thread parked in recv(); an error here + # (socket already closed, or not connected) is irrelevant to that goal. + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + def _read_body(resp: Any, *, max_bytes: int, deadline: float) -> bytes: """ Read a response body bounded by a total byte cap and a wall-clock deadline. @@ -190,31 +225,72 @@ def _read_body(resp: Any, *, max_bytes: int, deadline: float) -> bytes: past the caller's timeout (urllib's timeout is per-socket-read, not a whole-read bound) nor exhaust memory with an unbounded body. """ - # Read via read1(): it returns after a SINGLE underlying socket read, so the - # deadline check below actually runs between reads. resp.read(n) on an - # http.client response instead blocks until it has buffered the full n bytes - # (or hits EOF), so a server dribbling one byte per socket-timeout window - # would keep one read(_READ_CHUNK) blocked indefinitely — the per-socket - # timeout keeps resetting and the deadline is never reached. read1 is - # provided by http.client.HTTPResponse and (by delegation) urllib's - # HTTPError; fall back to read() for any stream that lacks it. + # read1() returns after a SINGLE underlying socket read on a Content-Length + # body, so the deadline check below runs between reads. But read1() alone is + # NOT sufficient: for a *chunked* body it calls http.client's readline() to + # parse each chunk-size line, and readline() loops over socket reads until it + # sees a newline — so a server that dribbles the size line one byte per + # socket-timeout window (never terminating it) keeps a single read1() blocked + # for up to _MAXLINE (~hours), and this loop's deadline check never runs. The + # per-leg socket timeout doesn't fire either (each dribbled byte resets it). + # Guard that with a watchdog that shuts the socket down at the deadline: the + # blocked read then returns/raises and is mapped to a typed OidcNetworkError + # below, instead of hanging the calling thread (which holds the acquisition + # lock). read1 is provided by http.client.HTTPResponse and (by delegation) + # urllib's HTTPError; fall back to read() for any stream that lacks it. read = getattr(resp, 'read1', None) or resp.read + sock = _underlying_socket(resp) + timer = None + if sock is not None: + timer = threading.Timer( + max(0.0, deadline - _monotonic()), _shutdown_socket, (sock,)) + timer.daemon = True + timer.start() chunks = [] total = 0 - while True: - if _monotonic() > deadline: - raise OidcNetworkError( - 'Timed out reading the response body; the server is too slow ' - 'or is dribbling data.') - chunk = read(_READ_CHUNK) - if not chunk: - return b''.join(chunks) - total += len(chunk) - if total > max_bytes: - raise OidcNetworkError( - f'Response body exceeded the {max_bytes}-byte limit; refusing ' - 'to buffer an unbounded response.') - chunks.append(chunk) + try: + while True: + if _monotonic() > deadline: + raise OidcNetworkError( + 'Timed out reading the response body; the server is too ' + 'slow or is dribbling data.') + try: + chunk = read(_READ_CHUNK) + except OidcNetworkError: + raise + except (OSError, http.client.HTTPException, ValueError) as e: + # The watchdog shut the socket down at the deadline to break a + # stalled read (a chunked size-line dribble surfaces here as an + # IncompleteRead / a bad chunk size / a socket error); or a + # genuine transport failure occurred mid-body. Either way it is a + # network problem, not a usable response — keep the typed-error + # contract rather than leak a raw socket/decode exception. + if _monotonic() > deadline: + raise OidcNetworkError( + 'Timed out reading the response body; the server is too ' + 'slow or is dribbling data.') from e + raise OidcNetworkError( + f'Failed while reading the response body: {e}') from e + if not chunk: + # An empty read is a clean end-of-body — UNLESS the watchdog + # tore the socket down at the deadline, which on a Content-Length + # body surfaces as EOF (not an exception). Treat a post-deadline + # EOF as the timeout it is, so a dribbled body isn't mistaken for + # a complete one and returned silently truncated. + if _monotonic() > deadline: + raise OidcNetworkError( + 'Timed out reading the response body; the server is too ' + 'slow or is dribbling data.') + return b''.join(chunks) + total += len(chunk) + if total > max_bytes: + raise OidcNetworkError( + f'Response body exceeded the {max_bytes}-byte limit; ' + 'refusing to buffer an unbounded response.') + chunks.append(chunk) + finally: + if timer is not None: + timer.cancel() def request( @@ -239,22 +315,40 @@ def request( _require_secure(url, insecure) body: Optional[bytes] = data req_headers = {'User-Agent': _USER_AGENT, 'Accept': 'application/json'} - if form is not None: - body = urllib.parse.urlencode( - {k: v for k, v in form.items() if v is not None}).encode('utf-8') - req_headers['Content-Type'] = 'application/x-www-form-urlencoded' - if headers: - req_headers.update(headers) - - req = urllib.request.Request( - url, data=body, headers=req_headers, method=method.upper()) try: + # Build the request INSIDE the try. urlencode(...).encode('utf-8') on a + # form value carrying a lone surrogate — a JSON string a hostile IdP can + # return as a device_code / refresh_token / scope, which passes the + # isinstance(str) coercion guards — and http.client's encode of a + # non-ASCII URL host both raise a raw UnicodeEncodeError. Previously the + # encode and Request() ran before this try, so that escaped the + # typed-error contract. A non-ASCII credential-endpoint authority is also + # rejected up-front by _reject_confusable_authority; catching here is the + # backstop covering every other path (the /settings and IdP-discovery + # URLs, whose hosts that check never sees). + if form is not None: + body = urllib.parse.urlencode( + {k: v for k, v in form.items() if v is not None}).encode('utf-8') + req_headers['Content-Type'] = 'application/x-www-form-urlencoded' + if headers: + req_headers.update(headers) + req = urllib.request.Request( + url, data=body, headers=req_headers, method=method.upper()) with _opener(ctx).open(req, timeout=timeout) as resp: return HttpResponse( getattr(resp, 'status', resp.getcode()), _read_body(resp, max_bytes=_MAX_RESPONSE_BYTES, deadline=_monotonic() + timeout), resp.headers) + except UnicodeError as e: + # A non-ASCII URL host or an unencodable request field (e.g. a lone + # surrogate) — keep it within the typed-error contract instead of leaking + # a raw UnicodeEncodeError from urlencode().encode() / http.client. + raise OidcConfigError( + f'Could not encode the request to {url!r}: {e}. The URL host or a ' + 'request field contains a non-ASCII or unencodable character (e.g. a ' + 'lone surrogate), indicating a malformed or tampered configuration ' + 'or server response.') from e except urllib.error.HTTPError as e: # 4xx/5xx still carry a (possibly JSON) body to inspect. Bound the read # (same cap/deadline), map a mid-body read failure to a network error, diff --git a/test/test_auth.py b/test/test_auth.py index 562b947b..3492557f 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -664,6 +664,31 @@ def on_success(self, identity, expires_in): self.assertLessEqual(auth._tokens.expires_at - self._clock.now(), _MAX_EXPIRES_IN) + def test_hostile_jwt_exp_does_not_abort_signin(self): + # M3: _display_lifetime ("expires in N min") is cosmetic but runs on the + # success path. A hostile JWT exp — a huge int that overflows float() — + # must NOT abort an already-completed sign-in; otherwise the token the + # user authorized is discarded (the cache store runs only after the flow + # returns) and every later token() re-prompts and re-crashes. token() + # must still return the token. + auth = self.make_auth() + id_tok = _jwt({'sub': 'alice', 'exp': 10 ** 400}) + self.state.token_script = [(200, { + 'access_token': 'a', 'id_token': id_tok, 'refresh_token': 'r', + 'expires_in': 3600, 'scope': 'openid groups'})] + self.assertEqual(auth.token(), id_tok) # groups mode -> id_token + + def test_raising_success_renderer_does_not_abort_signin(self): + # M3: rendering the success message is best-effort — a custom renderer + # whose on_success raises must NOT discard a token the user already + # authorized. The sign-in completes and token() returns. + class _Boom(Renderer): + def on_success(self, identity, expires_in): + raise RuntimeError('renderer blew up') + + auth = self.make_auth(renderer=_Boom()) + self.assertEqual(auth.token(), ID_TOKEN) + def test_poll_honors_retry_after(self): # Issue 8: a slow_down poll response carrying Retry-After backs off by # that many seconds (clamped), not the fixed +5s. @@ -2834,6 +2859,31 @@ def test_confusable_authority_endpoint_rejected(self): # A legitimate IPv6-literal authority is NOT flagged as confusable. self._validate('https://[::1]:443/token', 'https://[::1]:443/device') + def test_non_ascii_authority_endpoint_rejected(self): + # M2: a non-ASCII endpoint host is rejected fail-closed on every + # construction path. It is a homoglyph/confusable spoofing vector (the + # renderer already distrusts it for display), and http.client cannot even + # encode it — it would otherwise reach the transport and raise a raw + # UnicodeEncodeError instead of a typed error. An IDN must be given in its + # ASCII xn-- punycode form. + from questdb.auth._discovery import _reject_confusable_authority + for url in ('https://bаd.example/token', # Cyrillic 'а' + 'https://idp.exämple.com/token'): # 'ä' + with self.assertRaises(OidcConfigError): + _reject_confusable_authority(url, label='token endpoint') + with self.assertRaises(OidcConfigError): # public co-location entry + self._validate(url, url) + with self.assertRaises(OidcConfigError): # and the full constructor + OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint=url, + token_endpoint=url, + renderer=Renderer()) + # An ASCII xn-- punycode authority (how an IDN must be supplied) is NOT + # flagged — isascii() admits it. + self._validate('https://xn--bcher-kva.example/token', + 'https://xn--bcher-kva.example/device') + def test_confusable_issuer_rejected(self): # M1 (defense-in-depth): a confusable issuer authority would make the # issuer-origin pin compare against the wrong host. With explicit @@ -3190,6 +3240,21 @@ def test_malformed_url_raises_config_error(self): request('GET', 'https://questdb.example.com:notaport/settings', timeout=5) + def test_unencodable_request_maps_to_config_error(self): + # M1/M2: a lone surrogate in a form field (a JSON string a hostile IdP + # can return as a device_code / refresh_token / scope — it passes the + # isinstance(str) coercion guards) makes urlencode().encode('utf-8') + # raise; a non-ASCII URL host makes http.client's encode raise. Both must + # surface as a typed OidcConfigError, not a raw UnicodeEncodeError + # escaping the contract (the encode/Request now run inside request()'s + # try). Neither reaches the network, so no server is needed. + from questdb.auth._http import request + with self.assertRaises(OidcConfigError): # surrogate in the form body + request('POST', 'https://idp.example/token', + form={'device_code': '\ud800'}, timeout=5) + with self.assertRaises(OidcConfigError): # non-ASCII host (backstop path) + request('GET', 'https://bаd.example/settings', timeout=5) + def test_require_secure_rejects_malformed_ipv6(self): # _require_secure routes through safe_urlparse, so a malformed IPv6 # endpoint raises OidcConfigError instead of a bare ValueError (urlparse @@ -3296,6 +3361,78 @@ def call(): 'read1()).') self.assertIsInstance(result.get('error'), OidcNetworkError) + def test_read_body_aborts_real_socket_chunked_dribble(self): + # Regression for C1, against the REAL socket stack. read1() returns after + # one socket read on a Content-Length body (the test above), but on a + # CHUNKED body it calls http.client's readline() to parse each chunk-size + # line, and readline() loops over socket reads until it sees a newline. + # A server that dribbles the size line one byte at a time, never + # terminating it, keeps a single read1() blocked for up to _MAXLINE + # (~hours) — the between-reads deadline never runs and the per-socket + # timeout keeps resetting — hanging the calling thread (which holds the + # acquisition lock). _read_body's deadline watchdog must shut the socket + # down at the deadline and surface a typed OidcNetworkError. + import socket + from questdb.auth import _http + + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(('127.0.0.1', 0)) + srv.listen(1) + port = srv.getsockname()[1] + stop = threading.Event() + + def serve(): + try: + conn, _ = srv.accept() + except OSError: + return + try: + conn.recv(65536) # consume the request line/headers + # Announce chunked, then dribble the chunk-SIZE line one hex + # digit at a time with NO terminating CRLF, each well inside the + # per-socket timeout window so urllib's per-read timeout never + # fires. + conn.sendall( + b'HTTP/1.1 200 OK\r\n' + b'Content-Type: application/json\r\n' + b'Transfer-Encoding: chunked\r\n\r\n') + while not stop.is_set(): + try: + conn.sendall(b'a') + except OSError: + break + stop.wait(0.1) + finally: + conn.close() + + server_thread = threading.Thread(target=serve, daemon=True) + server_thread.start() + + result = {} + + def call(): + try: + _http.request( + 'GET', f'http://127.0.0.1:{port}/x', timeout=1.0) + result['returned'] = True + except Exception as e: # noqa: BLE001 - record for the assert below + result['error'] = e + + t = threading.Thread(target=call, daemon=True) + t.start() + t.join(8.0) + hung = t.is_alive() # capture before cleanup unblocks it + stop.set() + srv.close() + server_thread.join(2.0) + + self.assertFalse( + hung, + 'request() hung on a chunked size-line dribble: the deadline ' + 'watchdog never fired (C1 regression).') + self.assertIsInstance(result.get('error'), OidcNetworkError) + def test_bad_ca_bundle_raises_config_error(self): # A missing or invalid CA bundle path (explicit or via env) must surface # as OidcConfigError, not a raw FileNotFoundError / ssl.SSLError. See M1. From 81f72a53f0a52dec6a79fd69aabe01f91b9aab40 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 26 Jun 2026 13:17:16 +0100 Subject: [PATCH 079/104] fix(auth): refuse pool-thread sign-in; floor slow_down backoff Three more review fixes to questdb.auth: - M5: sqlalchemy_engine's do_connect listener ran auth.token() on pool threads, so a first-time interactive sign-in could block the whole pool (or storm OidcInteractionRequired in a headless pool). Thread an internal allow_interactive flag: the pool callback now reuses / silently refreshes the cached token but refuses to START the device flow, raising a clear "sign in up front" error instead. Public token() is unchanged, and the silent refresh still runs on pool threads. - M6: a slow_down poll response carrying a Retry-After lower than the current interval reset the interval to the floor -- polling FASTER right after the IdP told it to slow down (RFC 8628 section 3.5 requires an increase). slow_down now never drops below current + 5; a plain 429/5xx still honors Retry-After verbatim. - M4: the concurrency stress test used a single instance, so the cross-instance generation/inflight CAS (the pool case) was never raced. Add a cross-instance stress test driving several instances that share one cache_key under real contention. Full questdb.auth suite green (217 tests). Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/auth.rst | 6 +- src/questdb/auth/_adapters.py | 13 +++- src/questdb/auth/_device.py | 53 ++++++++++++-- test/test_auth.py | 129 +++++++++++++++++++++++++++++++++- 4 files changed, 188 insertions(+), 13 deletions(-) diff --git a/docs/auth.rst b/docs/auth.rst index 5b452789..878d46cb 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -180,7 +180,11 @@ Two helpers wire the auto-refreshed token into PG-wire as the ``_sso`` password * :func:`~questdb.auth.sqlalchemy_engine` — a SQLAlchemy ``Engine`` that injects a fresh token for every new connection, so a pool keeps working as the token - rotates. + rotates. The per-connection injection is non-interactive (it reuses and + silently refreshes the up-front token); sign in once with ``auth.token()`` + before opening connections, or it raises + :class:`~questdb.auth.OidcInteractionRequired` rather than launching a browser + prompt from a pool thread. * :func:`~questdb.auth.psycopg_connect` — a raw psycopg / psycopg2 connection (token captured at connect time). diff --git a/src/questdb/auth/_adapters.py b/src/questdb/auth/_adapters.py index 4dccf7bd..3ebd33cd 100644 --- a/src/questdb/auth/_adapters.py +++ b/src/questdb/auth/_adapters.py @@ -143,8 +143,12 @@ def sqlalchemy_engine( always authenticate with a valid, auto-refreshed token. Requires ``acl.oidc.pg.token.as.password.enabled=true`` on the server. - Sign in once up front (``auth.token()``) before the pool opens connections, - so threads don't race the interactive prompt. + Sign in once up front (``auth.token()``) before the pool opens connections. + The per-connection injection is **non-interactive**: it reuses and silently + refreshes the cached token, but never launches a browser prompt from a pool + thread. If no token has been acquired yet it raises + :class:`OidcInteractionRequired` rather than blocking the pool on an + interactive sign-in. :param auth: An :class:`OidcDeviceAuth`, e.g. from :meth:`OidcDeviceAuth.from_questdb`. @@ -183,7 +187,10 @@ def sqlalchemy_engine( @event.listens_for(engine, 'do_connect') def _provide_token(dialect, conn_rec, cargs, cparams): # noqa: ANN001 - cparams['password'] = auth.token() + # Non-interactive: reuse / silently refresh the up-front token, but never + # run an interactive device flow from a pool thread (it would block the + # pool). Raises OidcInteractionRequired if no token was acquired first. + cparams['password'] = auth._token(allow_interactive=False) return engine diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 74b928ab..9f32e922 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -176,7 +176,9 @@ def _http_status_is_transient(status: Optional[int]) -> bool: return status is not None and (status >= 500 or status == 429) -def _backoff_interval(interval: int, retry_after: Optional[int]) -> int: +def _backoff_interval( + interval: int, retry_after: Optional[int], + *, at_least_increment: bool = False) -> int: """ The next poll interval after a 429 / ``slow_down``. @@ -184,8 +186,16 @@ def _backoff_interval(interval: int, retry_after: Optional[int]) -> int: 8628 §3.5 +5s slow-down step. Clamped to ``[_MIN_POLL_INTERVAL, _MAX_POLL_INTERVAL]`` so a hostile/huge value can't pin the polling thread — the device-code deadline still bounds the total wait. + + ``at_least_increment`` enforces RFC 8628 §3.5 for ``slow_down``: the interval + MUST increase by at least 5, so a contradictory ``Retry-After`` *lower* than + the current interval can't make the client poll faster right after the IdP + told it to slow down. (A plain ``429``/``5xx`` keeps honoring ``Retry-After`` + verbatim — there the server's value is authoritative, not a slow-down step.) """ target = retry_after if retry_after is not None else interval + 5 + if at_least_increment: + target = max(target, interval + 5) return min(_MAX_POLL_INTERVAL, max(_MIN_POLL_INTERVAL, target)) @@ -456,7 +466,18 @@ def token(self) -> str: token (``acl.oidc.groups.encoded.in.token=true``), else the ``access_token`` — mirroring QuestDB's own selection logic. """ - return self._select(self._obtain_tokens()) + return self._token() + + def _token(self, *, allow_interactive: bool = True) -> str: + # Internal token accessor. allow_interactive=False does the fast-path / + # silent-refresh work but refuses to START the interactive device flow, + # raising OidcInteractionRequired instead. Used by the SQLAlchemy pool + # callback (see _adapters.sqlalchemy_engine), where running a browser + # prompt on a pool thread would block the pool — the user is expected to + # sign in once up front, and the pool then reuses / silently refreshes + # that token. + return self._select( + self._obtain_tokens(allow_interactive=allow_interactive)) def headers(self) -> Dict[str, str]: """Return ``{"Authorization": "Bearer "}``.""" @@ -549,7 +570,7 @@ def _missing_required_token_error(self) -> OidcDeviceFlowError: 'Device authorization completed but the IdP returned no ' 'access_token.') - def _obtain_tokens(self) -> TokenSet: + def _obtain_tokens(self, *, allow_interactive: bool = True) -> TokenSet: # Fast path: return a valid token without the lock, so a caller with a # usable token never blocks behind another thread's refresh/sign-in. # READ-ONLY — never writes self._tokens; every write to that field is @@ -586,7 +607,8 @@ def _obtain_tokens(self) -> TokenSet: tokens = self._valid_cached() if tokens is not None: return tokens - return self._acquire(generation) + return self._acquire( + generation, allow_interactive=allow_interactive) finally: self._cache.release(self.cache_key) @@ -617,10 +639,14 @@ def _valid_cached(self) -> Optional[TokenSet]: return tokens return None - def _acquire(self, generation: int) -> TokenSet: + def _acquire( + self, generation: int, *, + allow_interactive: bool = True) -> TokenSet: # Holds self._lock. Try a silent refresh, else run the device flow. # `generation` was captured before the cache read in _obtain_tokens; # _store drops its write if a concurrent clear() bumped it since. + # A silent refresh runs regardless of allow_interactive; only the + # interactive device-flow fallback below is gated by it. tokens = self._tokens if tokens is not None and tokens.refresh_token: try: @@ -653,6 +679,18 @@ def _acquire(self, generation: int) -> TokenSet: self._tokens = None self._cache.evict(self.cache_key) + if not allow_interactive: + # A fresh interactive sign-in would be required, but this caller + # forbids it — it runs on a connection-pool thread, where a browser + # prompt would block the pool (see _adapters.sqlalchemy_engine). + raise OidcInteractionRequired( + 'A token must be acquired by an interactive sign-in, but this ' + 'request disallows it (it runs on a connection-pool thread, ' + 'where a browser prompt would block the pool). Call ' + 'auth.token() once on the main thread before opening pooled ' + 'connections; the pool then reuses and silently refreshes that ' + 'token.') + fresh = self._run_device_flow() self._store(fresh, generation) return fresh @@ -987,7 +1025,10 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: if error == 'authorization_pending': continue if error == 'slow_down': - interval = _backoff_interval(interval, retry_after) + # RFC 8628 §3.5: slow_down MUST raise the interval. Never let a + # contradictory low Retry-After reduce it below current + 5. + interval = _backoff_interval( + interval, retry_after, at_least_increment=True) continue if error == 'expired_token': self._renderer.on_failure( diff --git a/test/test_auth.py b/test/test_auth.py index 3492557f..7ec18d96 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -80,15 +80,23 @@ class _FakeAuth: _ctx = None def __init__(self, token='TKN'): - self._token = token + self._value = token self.calls = 0 def token(self): self.calls += 1 - return self._token + return self._value + + def _token(self, *, allow_interactive=True): + # The SQLAlchemy adapter fetches the per-connection token through this + # internal accessor with allow_interactive=False (it runs on a pool + # thread, where an interactive prompt would block the pool). Mirror + # OidcDeviceAuth and count it like token(). + self.calls += 1 + return self._value def headers(self): - return {'Authorization': f'Bearer {self._token}'} + return {'Authorization': f'Bearer {self._value}'} class _ChunkStream: @@ -400,6 +408,33 @@ def test_slow_down_backs_off(self): # interval starts at 5, +5 after slow_down. self.assertEqual(self._clock.sleeps, [5, 10]) + def test_slow_down_retry_after_never_decreases_interval(self): + # M6 / RFC 8628 §3.5: slow_down MUST raise the poll interval. A + # contradictory LOW Retry-After on a slow_down must not reduce it below + # current + 5 (which would make the client poll faster right after the IdP + # told it to slow down). A plain 429/5xx still honors Retry-After verbatim + # (test_poll_honors_retry_after); this is the slow_down-specific rule. + from questdb.auth._http import _PostResult + auth = self.make_auth() + real = auth._idp_post + n = {'tok': 0} + + def fake(url, form): + if url.endswith('/token'): + n['tok'] += 1 + if n['tok'] == 1: + return _PostResult(400, {'error': 'slow_down'}, None) # 5->10 + if n['tok'] == 2: + return _PostResult(400, {'error': 'slow_down'}, 1) # low RA + return real(url, form) # success + return real(url, form) + + auth._idp_post = fake + self.assertEqual(auth.token(), ID_TOKEN) + # 5 -> (+5) 10 -> slow_down w/ Retry-After:1 stays >= 10+5 = 15, never + # dropping back to the 5s floor. + self.assertEqual(self._clock.sleeps, [5, 10, 15]) + def test_transient_network_error_during_poll_keeps_polling(self): # A dropped connection / DNS blip / timeout on a single poll must not # abort a sign-in the user may already have completed in the browser: @@ -1353,6 +1388,27 @@ def test_silent_refresh(self): self.assertEqual(self.state.refresh_requests, 1) self.assertEqual(self.state.device_requests, 0) # no re-prompt + def test_noninteractive_token_refuses_device_flow(self): + # M5: the SQLAlchemy pool callback fetches the token with + # allow_interactive=False so a browser prompt never blocks a pool thread. + # With nothing cached and no refresh token, it must raise a clear + # OidcInteractionRequired WITHOUT starting the device flow. + auth = self.make_auth() + with self.assertRaises(OidcInteractionRequired): + auth._token(allow_interactive=False) + self.assertEqual(self.state.device_requests, 0) # flow never ran + self.assertEqual(len(self.state.token_requests), 0) + + def test_noninteractive_token_still_silently_refreshes(self): + # M5: a non-interactive caller (a pool thread) still performs a SILENT + # refresh of an expired token — only the interactive device flow is + # refused, never the refresh. No prompt. + auth = self.make_auth() + self._seed_expired(auth) + self.assertEqual(auth._token(allow_interactive=False), ID_TOKEN) + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual(self.state.device_requests, 0) + def test_skew_window_triggers_proactive_refresh(self): # The point of the cache+skew design: a token still within its lifetime # but inside the 30s clock-skew margin is refreshed PROACTIVELY (silently @@ -2336,6 +2392,73 @@ def clearer(): # The auth is still usable afterwards. self.assertEqual(auth.token(), ID_TOKEN) + def test_cross_instance_clear_stress(self): + # M4: the single-instance stress test above cannot exercise the + # cross-instance generation/inflight CAS — one instance's clear() and its + # own acquisition serialize on the same self._lock. The race the CAS + # actually guards is a clear() on ONE instance bumping the generation + # while ANOTHER instance's store_if_current is in flight (separate locks, + # one shared process-global store) — the SQLAlchemy/psycopg pool case. + # Drive several instances sharing one cache_key under real contention: + # workers acquire on every instance while a clearer clears them. Asserts + # no torn read / exception / deadlock, every served token is the right + # kind, the cache serves the steady state, and the process-global + # in-flight & generation bookkeeping does not leak. (Lock order is always + # instance-lock -> _MEMORY_LOCK on every path, so there is no inversion to + # deadlock on.) + clock = _ConcurrentClock() + n_inst = 4 + insts = [self.make_auth(clock=clock, open_browser=False) + for _ in range(n_inst)] + key = insts[0].cache_key + self.assertTrue(all(a.cache_key == key for a in insts)) + + n_workers = 6 + iters = 80 + n_clears = 40 + errors = [] + start = threading.Barrier(n_workers + 1 + 1) # workers + clearer + main + + def worker(wid): + start.wait() + try: + for i in range(iters): + if insts[(wid + i) % n_inst].token() != ID_TOKEN: + errors.append('wrong token kind served') + return + except Exception as e: # noqa: BLE001 + errors.append(e) + + def clearer(): + start.wait() + try: + for i in range(n_clears): + insts[i % n_inst].clear() # clears under a DIFFERENT lock + except Exception as e: # noqa: BLE001 + errors.append(e) + + threads = [threading.Thread(target=worker, args=(w,)) + for w in range(n_workers)] + threads.append(threading.Thread(target=clearer)) + for t in threads: + t.start() + start.wait() # release everyone at once for maximum contention + for t in threads: + t.join(30) + for t in threads: + self.assertFalse(t.is_alive(), 'a thread deadlocked under contention') + self.assertEqual(errors, [], f'errors under contention: {errors[:3]}') + # The shared cache + each instance's fast path serve the steady state, so + # device flows stay far below the total token() calls. (Cross-instance CAN + # double-prompt across separate locks, so this is looser than the + # single-instance "exactly once".) + self.assertLess(self.state.device_requests, n_workers * iters // 2) + # No leaked process-global bookkeeping once the storm settles. + self.assertEqual(_MEMORY_INFLIGHT.get(key, 0), 0) + self.assertNotIn(key, _MEMORY_GENERATION) + # Still usable. + self.assertEqual(insts[0].token(), ID_TOKEN) + class TestAdapters(unittest.TestCase): """PG-wire connection adapters: tested via injected fake modules (the real From 1e5a5f8adcadf7880abca40bee8978dce59fe553 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 26 Jun 2026 13:32:17 +0100 Subject: [PATCH 080/104] fix(auth): adopt fresh cached token; honor non-JSON Retry-After Remaining minor review fixes (the no-GIL CI gap is intentionally left out): - Cross-instance cache: when an instance held a stale (non-None, expired) self._tokens while another instance sharing the process-global cache had stored a fresh valid token, the locked slow path reloaded the shared store only when self._tokens was None -- so the stale local token shadowed the fresher cached one and forced a redundant refresh. Reload and adopt a valid cached token whenever the local one is stale. - Retry-After on non-JSON errors: post_form parsed Retry-After off the response headers but dropped it when the body was non-JSON, so a non-JSON 429/503 from a proxy/WAF fell back to the fixed +5s step. Carry it on OidcError.retry_after and honor it in the poll loop's exception arm. - _display_url: parts.port raises ValueError on a junk port, which aborted host normalization and showed a homoglyph host raw. Read the port defensively so the host is still IDNA/punycode-normalized (junk port dropped) -- the spoof is still revealed. - Docs/infra: drop stale Python 3.8 references (RELEASING.rst, a dead proj.py comment); narrow the mock server's handle_error suppression to the specific client-disconnect errors (not all ConnectionError); group the exception autodoc by hierarchy and document OidcConfig's fields. Adds regression tests for the first three (all teeth-checked by mutation). Full questdb.auth suite green (221 tests). Co-Authored-By: Claude Opus 4.8 (1M context) --- RELEASING.rst | 3 +- docs/api.rst | 6 +-- proj.py | 8 ---- src/questdb/auth/_device.py | 29 ++++++++----- src/questdb/auth/_discovery.py | 9 ++++ src/questdb/auth/_errors.py | 7 +++- src/questdb/auth/_http.py | 11 +++-- src/questdb/auth/_render.py | 10 ++++- test/mock_server.py | 10 ++++- test/test_auth.py | 75 +++++++++++++++++++++++++++++++++- 10 files changed, 136 insertions(+), 32 deletions(-) diff --git a/RELEASING.rst b/RELEASING.rst index c5444cdd..ba3b6c79 100644 --- a/RELEASING.rst +++ b/RELEASING.rst @@ -59,10 +59,9 @@ From a MacOS ARM computer install UTM. * Install MacOS X 12.4 (Monterey). See https://docs.getutm.app/guest-support/macos/ * Install Rust from https://rustup.rs/ * Install Firefox -* Install *all* OFFICIAL Python Releases from Python 3.8 onwards. Use the latest patch version for each minor release. +* Install *all* OFFICIAL Python Releases from Python 3.10 onwards. Use the latest patch version for each minor release. * https://www.python.org/downloads/macos/ * Do NOT use Homebrew to install Python. - * Python 3.8.10 requires Rosetta, install it when prompted to do so. * Optionally install VS Code diff --git a/docs/api.rst b/docs/api.rst index a9cc14aa..3aec0901 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -98,6 +98,9 @@ See the :ref:`oidc_auth` guide for an overview. .. autoexception:: questdb.auth.OidcConfigError :show-inheritance: +.. autoexception:: questdb.auth.OidcNetworkError + :show-inheritance: + .. autoexception:: questdb.auth.OidcInteractionRequired :show-inheritance: @@ -106,6 +109,3 @@ See the :ref:`oidc_auth` guide for an overview. .. autoexception:: questdb.auth.OidcTimeoutError :show-inheritance: - -.. autoexception:: questdb.auth.OidcNetworkError - :show-inheritance: diff --git a/proj.py b/proj.py index 2f27c966..6b55f75b 100755 --- a/proj.py +++ b/proj.py @@ -208,14 +208,6 @@ def cibuildwheel(*args): 'darwin': 'macos', 'linux': 'linux'}[sys.platform] python = 'python3' - # if sys.platform == 'darwin': - # # Launching with version other than 3.8 will - # # fail saying the 3.8 wheel is unsupported. - # # This is because the 3.8 wheel ends up getting loaded with another - # # Python version. - # # - # # NB: Make sure to update `cibuildwheel` on py3.8 too before running! - # python = '/Library/Frameworks/Python.framework/Versions/3.8/bin/python3' _run(python, '-m', 'cibuildwheel', '--platform', plat, diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 9f32e922..08409a54 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -597,13 +597,20 @@ def _obtain_tokens(self, *, allow_interactive: bool = True) -> TokenSet: # for it (bounds the process-global maps; see MemoryCache.release). generation = self._cache_generation() try: - # Promote a cached token under the lock (even expired, so - # _acquire can reuse its refresh_token). Here, not on the fast - # path, so every write to self._tokens stays serialized. - if self._tokens is None: - cached = self._cache.load(self.cache_key) - if cached is not None: - self._tokens = cached + # Promote a cached token under the lock, consulting the shared + # store even when self._tokens is already set: another instance + # sharing the process-global cache may have acquired or refreshed + # a token since this one's self._tokens went stale, so adopt that + # fresh one instead of running a redundant refresh / sign-in. When + # we have nothing, adopt whatever is cached (even expired) so + # _acquire can reuse its refresh_token. Not on the fast path, so + # every write to self._tokens stays serialized. + cached = self._cache.load(self.cache_key) + if cached is not None and ( + self._tokens is None + or (cached.is_valid(self._now()) + and self._has_required_token(cached))): + self._tokens = cached tokens = self._valid_cached() if tokens is not None: return tokens @@ -969,9 +976,11 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: # poll again rather than discard the sign-in (the deadline bounds # the total wait; a genuine JSON rejection arrives below). if getattr(e, 'status', None) == 429: - # No JSON body here (a non-JSON 429 from a proxy/WAF), so no - # Retry-After is surfaced; fall back to the +5s step. - interval = _backoff_interval(interval, None) + # A non-JSON 429 (proxy/WAF). Honor a Retry-After header if + # post_form parsed one off the error response, else the +5s + # step (clamped to the poll-interval bounds either way). + interval = _backoff_interval( + interval, getattr(e, 'retry_after', None)) continue status, body = result diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index fe3c3ab8..dc66d190 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -65,12 +65,21 @@ class OidcConfig: """ client_id: str + """The OIDC public-client id registered with the identity provider.""" token_endpoint: str + """IdP token endpoint — where the device-code and refresh grants are POSTed.""" device_authorization_endpoint: str + """IdP device-authorization endpoint (RFC 8628 §3.1).""" scope: str = 'openid' + """Space-separated scopes; ``openid`` is added automatically in groups mode.""" groups_in_token: bool = False + """When true, present the ``id_token`` (groups encoded in it) rather than the + ``access_token`` — mirroring QuestDB's own selection.""" audience: Optional[str] = None + """Optional ``audience`` sent on the device-code / refresh requests (some IdPs, + e.g. Auth0, require it to mint a token QuestDB accepts).""" issuer: Optional[str] = None + """Optional out-of-band IdP pin (its origin / issuer path); see :ref:`oidc_auth`.""" def _as_bool(value: Any, default: Optional[bool] = None) -> Optional[bool]: diff --git a/src/questdb/auth/_errors.py b/src/questdb/auth/_errors.py index 8c091edf..1766b692 100644 --- a/src/questdb/auth/_errors.py +++ b/src/questdb/auth/_errors.py @@ -35,7 +35,8 @@ class OidcError(Exception): """Base class for every error raised by :mod:`questdb.auth`.""" - def __init__(self, *args, status: Optional[int] = None): + def __init__(self, *args, status: Optional[int] = None, + retry_after: Optional[int] = None): # Strip terminal/bidi/zero-width control characters from every string # message argument before it can reach a display sink. Error messages # routinely interpolate untrusted IdP fields (error_description, response @@ -54,6 +55,10 @@ def __init__(self, *args, status: Optional[int] = None): # loop and silent refresh can tell a terminal 4xx (e.g. a WAF error # page) from a transient 5xx/429/network blip. self.status = status + # Parsed Retry-After (delta-seconds) off a non-JSON 429/503 error body, + # so the poll loop can honor it the same way the JSON path does (via + # _PostResult.retry_after). None when absent / not applicable. + self.retry_after = retry_after class OidcConfigError(OidcError): diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 59314a73..72f8f31a 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -472,16 +472,19 @@ def post_form( if resp.ok: raise OidcError( f'Expected JSON from {url}, got: {resp.text()[:200]}', - status=resp.status) + status=resp.status, retry_after=retry_after) # Non-JSON error body: attach the HTTP status so callers (poll loop / - # silent refresh) can tell a terminal 4xx from a transient 5xx/429. + # silent refresh) can tell a terminal 4xx from a transient 5xx/429, and + # the parsed Retry-After so a non-JSON 429/503 backs off by the server's + # value rather than the fixed +5s step. raise OidcError( f'HTTP {resp.status} from {url}: {resp.text()[:200]}', - status=resp.status) + status=resp.status, retry_after=retry_after) if not isinstance(parsed, dict): # Attach the status (mirroring the non-JSON branches) so a non-object # body on a terminal 4xx — e.g. a JSON array from a non-conformant IdP # — fails the poll loop fast instead of polling on to "code expired". raise OidcError( - f'Unexpected JSON shape from {url}: {parsed!r}', status=resp.status) + f'Unexpected JSON shape from {url}: {parsed!r}', + status=resp.status, retry_after=retry_after) return _PostResult(resp.status, parsed, retry_after) diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index a5208400..dc77d763 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -220,7 +220,6 @@ def _display_url(url: Optional[str]) -> str: parts = urllib.parse.urlparse(text) scheme = (parts.scheme or '').lower() host = parts.hostname - port = parts.port except ValueError: return text if scheme not in ('http', 'https') or not host: @@ -238,6 +237,15 @@ def _display_url(url: Optional[str]) -> str: # rather than let an invisible homoglyph through unchanged. ascii_host = host.encode('ascii', 'backslashreplace').decode('ascii') host_part = f'[{ascii_host}]' if ':' in ascii_host else ascii_host # IPv6 + # Read the port separately and defensively: parts.port raises ValueError for + # a malformed (non-integer / out-of-range) port. That must NOT abort host + # normalization — otherwise a homoglyph host paired with a junk port would be + # shown raw (the very spoof this reveals). A junk port can't be rendered, so + # omit it; the host (what matters for spoofing) is still IDNA-normalized. + try: + port = parts.port + except ValueError: + port = None netloc = f'{host_part}:{port}' if port is not None else host_part return urllib.parse.urlunparse(parts._replace(netloc=netloc)) diff --git a/test/mock_server.py b/test/mock_server.py index 708be7f4..c2c4761b 100644 --- a/test/mock_server.py +++ b/test/mock_server.py @@ -132,7 +132,15 @@ class _QuietHTTPServer(hs.HTTPServer): raises ConnectionResetError outside of any request handler's try/except. """ def handle_error(self, request, client_address): - if isinstance(sys.exc_info()[1], ConnectionError): + # Suppress ONLY the noise from a client that went away mid-request + # (BrokenPipeError on Unix; ConnectionResetError / ConnectionAbortedError + # on Windows, incl. the keep-alive read of the next request line, which + # raises outside any handler's try/except). Any other error -- including + # a ConnectionRefusedError or a genuine handler bug -- still prints, so a + # real failure isn't hidden. + if isinstance(sys.exc_info()[1], + (BrokenPipeError, ConnectionResetError, + ConnectionAbortedError)): return super().handle_error(request, client_address) diff --git a/test/test_auth.py b/test/test_auth.py index 7ec18d96..68bb9354 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -123,12 +123,13 @@ def b64(obj): @contextlib.contextmanager -def _raw_response_server(status, content_type, body): +def _raw_response_server(status, content_type, body, extra_headers=None): """A throwaway HTTP server that returns one fixed (status, type, body). Used to exercise the transport's handling of responses the scripted mock IdP can't produce (non-JSON 2xx, non-dict JSON, non-2xx) on the token / - device / settings / discovery endpoints. Yields the base URL. + device / settings / discovery endpoints. ``extra_headers`` adds response + headers (e.g. ``Retry-After``). Yields the base URL. """ class _H(http.server.BaseHTTPRequestHandler): def log_message(self, *a): @@ -138,6 +139,8 @@ def _send(self): self.send_response(status) self.send_header('Content-Type', content_type) self.send_header('Content-Length', str(len(body))) + for _k, _v in (extra_headers or {}).items(): + self.send_header(_k, _v) self.end_headers() self.wfile.write(body) @@ -435,6 +438,26 @@ def fake(url, form): # dropping back to the 5s floor. self.assertEqual(self._clock.sleeps, [5, 10, 15]) + def test_non_json_429_retry_after_honored_in_poll(self): + # m2: a non-JSON 429 from a proxy/WAF now carries its Retry-After + # (post_form attaches it to the OidcError), so the poll loop's exception + # arm backs off by that value rather than the fixed +5s step. + self.state.token_script = [(200, None)] # success once actually polled + auth = self.make_auth() + real = auth._idp_post + polls = {'n': 0} + + def flaky(url, form): + if url == auth.config.token_endpoint: + polls['n'] += 1 + if polls['n'] == 1: + raise OidcError('proxy 429', status=429, retry_after=30) + return real(url, form) + + auth._idp_post = flaky + self.assertEqual(auth.token(), ID_TOKEN) + self.assertIn(30, self._clock.sleeps) # honored Retry-After, not +5 + def test_transient_network_error_during_poll_keeps_polling(self): # A dropped connection / DNS blip / timeout on a single poll must not # abort a sign-in the user may already have completed in the browser: @@ -2459,6 +2482,31 @@ def clearer(): # Still usable. self.assertEqual(insts[0].token(), ID_TOKEN) + def test_stale_local_token_adopts_fresh_shared_cache_token(self): + # m1: when this instance holds a STALE (non-None, expired) self._tokens + # while ANOTHER instance sharing the process-global cache has stored a + # fresh valid token, the slow path must adopt the cached fresh token + # rather than run a redundant refresh / sign-in. (Before the fix the + # promotion reloaded the shared cache only when self._tokens was None, so + # a stale local token shadowed the fresher cached one.) + clock = FakeClock() + a = self.make_auth(clock=clock) + b = self.make_auth(clock=clock) + self.assertEqual(a.cache_key, b.cache_key) + now = clock.now() + # a has a stale local token (with a refresh_token it would otherwise use): + a._tokens = TokenSet( + access_token='old', id_token='old-id', refresh_token='r-old', + expires_at=now - 10) + # b stored a fresh valid token in the shared cache meanwhile: + b._cache.store(b.cache_key, TokenSet( + access_token='fresh-access', id_token=ID_TOKEN, + refresh_token='r-new', issued_at=now, expires_at=now + 3600)) + # a.token() adopts the fresh cached token: no refresh, no device flow. + self.assertEqual(a.token(), ID_TOKEN) + self.assertEqual(self.state.refresh_requests, 0) + self.assertEqual(self.state.device_requests, 0) + class TestAdapters(unittest.TestCase): """PG-wire connection adapters: tested via injected fake modules (the real @@ -3206,6 +3254,18 @@ def test_post_form_attaches_status_to_non_json_error(self): post_form(raw + '/token', {'grant_type': 'x'}) self.assertEqual(cm.exception.status, 503) + def test_post_form_attaches_retry_after_to_non_json_error(self): + # m2: a non-JSON 429/503 carrying a Retry-After header surfaces that value + # on the raised OidcError (mirroring the JSON path's _PostResult), so a + # poll backs off by the server's value rather than the fixed +5s step. + from questdb.auth._http import post_form + with _raw_response_server(429, 'text/plain', b'slow down', + {'Retry-After': '30'}) as raw: + with self.assertRaises(OidcError) as cm: + post_form(raw + '/token', {'grant_type': 'x'}) + self.assertEqual(cm.exception.status, 429) + self.assertEqual(cm.exception.retry_after, 30) + def test_incomplete_error_body_maps_to_network_error(self): # A 4xx/5xx with a truncated CHUNKED body (the server announces a chunk, # sends fewer bytes, then closes) makes the error-body read raise @@ -4183,6 +4243,17 @@ def test_homoglyph_host_revealed_not_clickable(self): self.assertIn(' Date: Mon, 29 Jun 2026 12:56:15 +0100 Subject: [PATCH 081/104] fix(auth): escape confusable host on display fail-open _display_url rebuilds an untrusted verification host in IDNA/punycode to defeat homoglyph spoofing, but a confusable that NFKC-folds to a URL delimiter (fullwidth solidus U+FF0F -> '/', U+FF20 -> '@', U+FF03 -> '#', U+FF1F -> '?') makes urllib.parse.urlparse raise ValueError, and the except branch returned the raw string. So a value like "https://login.questdb.io@evil.example/device" was shown verbatim in the terminal and Jupyter prompts -- reading as host login.questdb.io while a browser resolves evil.example after the fold. Confined to inert text (never clickable/opened/QR-encoded -- _safe_target already rejected these), but that is exactly the manual copy/retype path the IDNA-display defense exists to stop. Route both fail-open branches (and the existing IDNA-failure branch) through a shared _ascii_visible() that escapes non-ASCII to a visible \uXXXX, so a confusable can't masquerade as a trusted host even when the host can't be normalized. Add a TestRendererSecurity regression covering U+FF0F/FF20/FF03/FF1F. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_render.py | 32 ++++++++++++++++++++++++++++---- test/test_auth.py | 17 +++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index dc77d763..3cbd5911 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -199,6 +199,19 @@ def _safe_target(value: Optional[str]) -> Optional[str]: return _safe_link_url(_strip_control(value)) +def _ascii_visible(text: str) -> str: + """ + Escape every non-ASCII char to a visible ``\\uXXXX`` so a confusable / + homoglyph can't slip through a display path unchanged. ASCII is left intact. + + Used wherever :func:`_display_url` can't normalize the host — a netloc urllib + refuses to parse (a confusable that NFKC-folds to a URL delimiter), a + non-``http(s)`` / hostless value, or an IDNA-unencodable label — so the raw + value is never echoed verbatim. + """ + return text.encode('ascii', 'backslashreplace').decode('ascii') + + def _display_url(url: Optional[str]) -> str: """ A verification URL rendered safe to *show* as text. @@ -211,7 +224,11 @@ def _display_url(url: Optional[str]) -> str: host the browser would actually resolve. Clickability is decided independently by :func:`_safe_target` (which rejects a non-ASCII host outright); this governs only the visible text. A non-``http(s)`` / hostless - value is returned control-stripped but otherwise unchanged. + value — or one whose authority urllib refuses to parse (a confusable that + NFKC-folds to a URL delimiter, e.g. a fullwidth solidus ``U+FF0F``) — is + returned control-stripped with any non-ASCII escaped to a visible + ``\\uXXXX`` (:func:`_ascii_visible`), so a homoglyph can't masquerade as a + trusted host even on this fail-open path. """ text = _strip_control(url) if not text: @@ -221,9 +238,16 @@ def _display_url(url: Optional[str]) -> str: scheme = (parts.scheme or '').lower() host = parts.hostname except ValueError: - return text + # The authority carries a confusable that NFKC-folds to a URL delimiter + # (fullwidth solidus U+FF0F -> '/', U+FF20 -> '@', ...), so urlparse + # refuses it and the host can't be normalized. Echoing it raw would show + # a host that reads as trusted while a browser resolves the real one + # after the fold; make the non-ASCII visible instead. + return _ascii_visible(text) if scheme not in ('http', 'https') or not host: - return text # nothing host-like to normalize (opaque / relative) + # Nothing host-like to normalize (opaque / relative); still neutralize any + # non-ASCII so a confusable can't pass through this path unchanged. + return _ascii_visible(text) if host.isascii(): ascii_host = host else: @@ -235,7 +259,7 @@ def _display_url(url: Optional[str]) -> str: except (UnicodeError, ValueError): # IDNA can't encode it (an illegal label); make the bytes visible # rather than let an invisible homoglyph through unchanged. - ascii_host = host.encode('ascii', 'backslashreplace').decode('ascii') + ascii_host = _ascii_visible(host) host_part = f'[{ascii_host}]' if ':' in ascii_host else ascii_host # IPv6 # Read the port separately and defensively: parts.port raises ValueError for # a malformed (non-integer / out-of-range) port. That must NOT abort host diff --git a/test/test_auth.py b/test/test_auth.py index 68bb9354..571cc354 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -4254,6 +4254,23 @@ def test_display_url_normalizes_host_with_malformed_port(self): self.assertIn('xn--', out) # shown in punycode self.assertIn('/device', out) # path preserved + def test_display_url_neutralizes_unparseable_confusable_host(self): + # M2: a confusable that NFKC-folds to a URL delimiter (fullwidth solidus + # U+FF0F -> '/', '@' U+FF20, '#' U+FF03, '?' U+FF1F) makes urlparse raise, + # so the host can't be normalized. The fail-open path must NOT echo the + # raw confusable (it would read as the trusted host 'login.questdb.io' + # while a browser resolves 'evil.example' after the fold); it escapes the + # non-ASCII to a visible \uXXXX, and the URL is never clickable / opened. + from questdb.auth._render import _display_url, _safe_target, _render_link + for cp in (0xFF0F, 0xFF20, 0xFF03, 0xFF1F): + raw = f'https://login.questdb.io{chr(cp)}@evil.example/device' + shown = _display_url(raw) + self.assertNotIn(chr(cp), shown) # confusable not echoed raw + self.assertIn(f'\\u{cp:04x}', shown) # made visible instead + self.assertIn('evil.example', shown) # real authority legible + self.assertIsNone(_safe_target(raw)) # never clickable / opened + self.assertNotIn(' Date: Mon, 29 Jun 2026 13:11:29 +0100 Subject: [PATCH 082/104] fix(auth): apply minor review fixes (5xx backoff, cache key, re-entry) Three minor findings from the device-flow review: - Poll loop: a non-JSON 5xx from a proxy/WAF carrying a Retry-After header was ignored by the exception arm (only a 429 backed off), while the JSON-body path honored it. Mirror the JSON arm so a transient 5xx with a Retry-After backs off by that value; a 429 still backs off without a header, and a 5xx without one keeps its cadence. - cache_key: the issuer was trailing-slash-normalized but the token endpoint was not, so ".../token" and ".../token/" produced different keys and forced an avoidable re-prompt. rstrip the normalized token endpoint too; the realm path still distinguishes multi-tenant keys. - Renderer re-entrancy: the acquisition lock is held across the whole sign-in, including renderer callbacks, so a custom renderer calling back into the same instance's token()/clear() silently deadlocked for up to the device-code lifetime. Track the lock-owning thread and raise a typed OidcError on same-thread re-entry instead of hanging. Adds a regression test for each. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 138 +++++++++++++++++++++++++----------- test/test_auth.py | 72 +++++++++++++++++++ 2 files changed, 167 insertions(+), 43 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 08409a54..1dd557bd 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -249,7 +249,8 @@ class OidcDeviceAuth: pool), sign in once up front — call :meth:`token` once on the main thread before the pool opens connections. (A custom ``renderer``'s callbacks run while this lock is held, so they must not call back into the same instance's - :meth:`token` / :meth:`clear` — the lock is not reentrant.) + :meth:`token` / :meth:`clear`; doing so raises :class:`OidcError` rather than + deadlocking, since the lock is not reentrant.) .. code-block:: python @@ -375,6 +376,11 @@ def __init__( # on the fast path, so a valid cached token never blocks behind a # sign-in. self._lock = threading.Lock() + # Thread id holding self._lock during an acquisition (set under the lock, + # cleared in the finally). Read lock-free by _guard_reentrancy to turn a + # same-thread re-entry — a custom renderer callback calling back into this + # instance's token()/clear() — into a clear error instead of a deadlock. + self._lock_owner: Optional[int] = None self._tokens: Optional[TokenSet] = None clock = _clock or _SYSTEM_CLOCK self._sleep = clock.sleep @@ -502,15 +508,17 @@ def cache_key(self) -> str: """ c = self.config scope = ' '.join(sorted(c.scope.split())) if c.scope else '' - # Normalize the issuer like the token endpoint (lower-case scheme/host, + # Normalize the issuer and token endpoint alike (lower-case scheme/host, # drop a default port) and strip a trailing slash, so a discovered - # "https://idp/" and an explicit "https://idp" — or a stray :443 / case - # difference — don't yield different keys and force an avoidable - # re-prompt. The realm path is kept (multi-tenant issuers differ by it). + # "https://idp/token/" and an explicit "https://idp/token" — or a stray + # :443 / case difference — don't yield different keys and force an + # avoidable re-prompt. The path is otherwise kept (multi-tenant realms + # differ by it); only a trailing slash, which never distinguishes an + # endpoint, is dropped. issuer = _normalize_url(c.issuer).rstrip('/') if c.issuer else '' return '\x1f'.join([ issuer, - _normalize_url(c.token_endpoint), + _normalize_url(c.token_endpoint).rstrip('/'), c.client_id, scope, c.audience or '', @@ -523,9 +531,16 @@ def clear(self) -> None: # ANOTHER instance sharing the process-global store can't repopulate the # entry (its _store sees the bumped generation and drops the write). # Resets the local/process cache only — does not revoke at the IdP. + # Refuse a same-thread re-entry (a renderer callback calling clear() on + # the instance whose sign-in it is rendering) rather than deadlock. + self._guard_reentrancy() with self._lock: - self._tokens = None - self._cache.clear(self.cache_key) + self._lock_owner = threading.get_ident() + try: + self._tokens = None + self._cache.clear(self.cache_key) + finally: + self._lock_owner = None # -- token lifecycle ---------------------------------------------------- @@ -570,6 +585,26 @@ def _missing_required_token_error(self) -> OidcDeviceFlowError: 'Device authorization completed but the IdP returned no ' 'access_token.') + def _guard_reentrancy(self) -> None: + # self._lock is non-reentrant and is held for the WHOLE acquisition, + # including the renderer callbacks invoked during the device flow + # (on_prompt / on_waiting / on_success / on_failure). A custom renderer + # whose callback calls back into THIS instance's token() / headers() / + # clear() would deadlock the calling thread for up to the device-code + # lifetime. Detect that same-thread re-entry (we are already the lock + # owner) and fail fast with a clear typed error instead of hanging. A + # different thread is unaffected — it legitimately waits behind the lock + # and can never observe its own id as the owner. The read is lock-free + # but safe: only the owning thread ever writes its own id, so the + # comparison against get_ident() is true only for a genuine re-entry. + if self._lock_owner == threading.get_ident(): + raise OidcError( + 'A renderer callback called back into the same OidcDeviceAuth ' + 'instance (token()/headers()/clear()) while its own sign-in was ' + 'in progress. The acquisition lock is not reentrant, so this ' + 'would deadlock; a renderer must not re-enter the instance whose ' + 'sign-in it is rendering.') + def _obtain_tokens(self, *, allow_interactive: bool = True) -> TokenSet: # Fast path: return a valid token without the lock, so a caller with a # usable token never blocks behind another thread's refresh/sign-in. @@ -586,38 +621,50 @@ def _obtain_tokens(self, *, allow_interactive: bool = True) -> TokenSet: return tokens # Slow path: serialize acquisition so concurrent callers don't overlap # refreshes or double-prompt; the loser re-checks and reuses the - # winner's token. + # winner's token. Refuse a same-thread re-entry (a renderer callback + # calling back into this instance) BEFORE blocking on the lock, so it + # raises instead of deadlocking on the non-reentrant lock. + self._guard_reentrancy() with self._lock: - # Capture the generation before reading/acquiring, so a racing - # clear() — including on another instance sharing the process-global - # MemoryCache (whose per-instance lock doesn't serialize against - # ours) — invalidates the store below instead of resurrecting the - # cleared entry. Paired with release() in the finally so the cache - # reclaims the per-key generation once no acquisition is in flight - # for it (bounds the process-global maps; see MemoryCache.release). - generation = self._cache_generation() + # Record ourselves as the lock owner so a re-entrant callback is + # detected (see _guard_reentrancy); the outer finally always clears + # it, even if the generation capture below raises. + self._lock_owner = threading.get_ident() try: - # Promote a cached token under the lock, consulting the shared - # store even when self._tokens is already set: another instance - # sharing the process-global cache may have acquired or refreshed - # a token since this one's self._tokens went stale, so adopt that - # fresh one instead of running a redundant refresh / sign-in. When - # we have nothing, adopt whatever is cached (even expired) so - # _acquire can reuse its refresh_token. Not on the fast path, so - # every write to self._tokens stays serialized. - cached = self._cache.load(self.cache_key) - if cached is not None and ( - self._tokens is None - or (cached.is_valid(self._now()) - and self._has_required_token(cached))): - self._tokens = cached - tokens = self._valid_cached() - if tokens is not None: - return tokens - return self._acquire( - generation, allow_interactive=allow_interactive) + # Capture the generation before reading/acquiring, so a racing + # clear() — including on another instance sharing the process- + # global MemoryCache (whose per-instance lock doesn't serialize + # against ours) — invalidates the store below instead of + # resurrecting the cleared entry. Paired with release() in the + # inner finally so the cache reclaims the per-key generation once + # no acquisition is in flight (bounds the maps; see + # MemoryCache.release). + generation = self._cache_generation() + try: + # Promote a cached token under the lock, consulting the + # shared store even when self._tokens is already set: another + # instance sharing the process-global cache may have acquired + # or refreshed a token since this one's self._tokens went + # stale, so adopt that fresh one instead of running a + # redundant refresh / sign-in. When we have nothing, adopt + # whatever is cached (even expired) so _acquire can reuse its + # refresh_token. Not on the fast path, so every write to + # self._tokens stays serialized. + cached = self._cache.load(self.cache_key) + if cached is not None and ( + self._tokens is None + or (cached.is_valid(self._now()) + and self._has_required_token(cached))): + self._tokens = cached + tokens = self._valid_cached() + if tokens is not None: + return tokens + return self._acquire( + generation, allow_interactive=allow_interactive) + finally: + self._cache.release(self.cache_key) finally: - self._cache.release(self.cache_key) + self._lock_owner = None def _valid_cached(self) -> Optional[TokenSet]: # Read-only: reads the published field, falling back to the shared cache @@ -975,12 +1022,17 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: # §3.4 expects polling to continue until the code expires, so # poll again rather than discard the sign-in (the deadline bounds # the total wait; a genuine JSON rejection arrives below). - if getattr(e, 'status', None) == 429: - # A non-JSON 429 (proxy/WAF). Honor a Retry-After header if - # post_form parsed one off the error response, else the +5s - # step (clamped to the poll-interval bounds either way). - interval = _backoff_interval( - interval, getattr(e, 'retry_after', None)) + err_status = getattr(e, 'status', None) + err_retry_after = getattr(e, 'retry_after', None) + if err_status == 429 or err_retry_after is not None: + # A non-JSON 429/5xx (proxy/WAF). Honor a Retry-After header + # if post_form parsed one off the error response, else (for a + # 429) the RFC 8628 +5s step; clamped to the poll-interval + # bounds either way. Mirrors the JSON-body arm below: a 429 + # backs off even without a header, while a transient 5xx + # honors a Retry-After but keeps its cadence without one (a + # server error isn't a rate-limit). + interval = _backoff_interval(interval, err_retry_after) continue status, body = result diff --git a/test/test_auth.py b/test/test_auth.py index 571cc354..93495bb9 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -526,6 +526,30 @@ def flaky(url, form): self.assertEqual(len(self.state.token_requests), 1) self.assertEqual(self._clock.sleeps, [5, 5, 10]) + def test_non_json_5xx_retry_after_honored_in_poll(self): + # m1: a non-JSON 5xx from a proxy/WAF carries its Retry-After (post_form + # attaches it to the OidcError), so the poll loop's exception arm now + # backs off by that value -- matching the JSON-body 5xx path + # (test_poll_honors_retry_after_on_5xx) and the non-JSON 429 path. + # Previously only a non-JSON 429 honored it, so a 5xx carrying a + # Retry-After kept polling at the base interval. + self.state.token_script = [(200, None)] # success once actually polled + auth = self.make_auth() + real = auth._idp_post + polls = {'n': 0} + + def flaky(url, form): + if url == auth.config.token_endpoint: + polls['n'] += 1 + if polls['n'] == 1: + raise OidcError('proxy 503', status=503, + retry_after=30) + return real(url, form) + + auth._idp_post = flaky + self.assertEqual(auth.token(), ID_TOKEN) + self.assertIn(30, self._clock.sleeps) # honored Retry-After, not base 5 + def test_non_json_4xx_during_poll_is_terminal(self): # A non-JSON 4xx during polling (an HTML/plain error page from a WAF or # reverse proxy in front of the IdP, or a non-conformant IdP) is a @@ -2347,6 +2371,44 @@ def on_prompt(self, resp): self.assertNotIn(a.cache_key, _MEMORY_GENERATION) # reclaimed on release self.assertNotIn(a.cache_key, _MEMORY_INFLIGHT) + def test_renderer_reentrant_call_raises_not_deadlocks(self): + # m3: self._lock is held across the WHOLE sign-in, including the renderer + # callbacks. A custom renderer whose callback calls back into the SAME + # instance (here clear()) must fail fast with a typed OidcError, not + # deadlock on the non-reentrant lock. Run token() on a worker thread with + # a join timeout so a regression that re-introduces the deadlock fails + # the test instead of hanging the whole suite. + auth = self.make_auth() + outcome = {} + + class _Reentrant(Renderer): + def on_prompt(self, resp): + auth.clear() # re-enter the same instance mid-sign-in + + auth._renderer = _Reentrant() + + def run(): + try: + auth.token() + outcome['result'] = 'returned' + except OidcError as e: + outcome['result'] = 'raised' + outcome['err'] = e + except BaseException as e: # noqa: BLE001 + outcome['result'] = 'other' + outcome['err'] = e + + t = threading.Thread(target=run) + t.start() + t.join(10) + self.assertFalse(t.is_alive(), 'reentrant renderer deadlocked the lock') + self.assertEqual(outcome.get('result'), 'raised', + f'expected OidcError, got {outcome!r}') + self.assertIn('reentrant', str(outcome['err']).lower()) + # The lock owner is cleared after the aborted acquisition, so the + # instance is left usable (no leaked owner / held lock). + self.assertIsNone(auth._lock_owner) + def test_token_clear_stress(self): # M3: drive the lock-free fast path and the generation/inflight CAS under # REAL thread contention. The other concurrency tests exercise the CAS @@ -3205,6 +3267,16 @@ def test_issuer_trailing_slash_and_case_do_not_change_key(self): self.assertEqual(base.cache_key, self._auth(issuer=variant).cache_key, variant) + def test_token_endpoint_trailing_slash_does_not_change_key(self): + # m2: like the issuer, the token endpoint is trailing-slash-normalized in + # the cache key. A discovered "https://idp/token/" and an explicit + # "https://idp/token" are the same endpoint, so they must not split the + # key and force an avoidable re-prompt. (A different realm PATH still + # stays distinct -- see test_realm_path_distinguishes_key.) + base = self._auth(token_endpoint='https://idp.example.com/token') + slashed = self._auth(token_endpoint='https://idp.example.com/token/') + self.assertEqual(base.cache_key, slashed.cache_key) + def test_issuer_realm_path_distinguishes_key(self): # A different realm PATH on the same host is a different issuer and must # stay a distinct key — origin-only normalization would wrongly collide From 1ce95998f2f0dc2b9ea7147ee3c205e253d6ac1e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 29 Jun 2026 14:02:11 +0100 Subject: [PATCH 083/104] fix(auth): reject tab/newline/CR in endpoint host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit urllib.parse.urlparse() silently removes tab/newline/CR from the URL before producing .netloc, but the transport (http.client via Request.host) keeps them. So _reject_confusable_authority validated a host that could diverge from the one urllib connects to — e.g. "https://idp\t.good/token" parses with hostname "idp.good" while the transport targets the raw "idp\t.good". Its _UNSAFE_AUTHORITY_RE lists \s/\x00-\x1f but never saw these bytes because urlparse stripped them first, leaving the guard's stated invariant to two incidental backstops (the issuer/origin pin and http.client._validate_host). Check the raw URL for these bytes up front so the dedicated guard enforces its own contract. Add a regression test covering both the label-merge and host-split cases across every construction path. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 20 +++++++++++++++++++- test/test_auth.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index dc66d190..30b682b4 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -313,6 +313,18 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: # checked with ``str.isascii`` in the function, not this regex.) _UNSAFE_AUTHORITY_RE = re.compile(r'[\\\s\x00-\x1f\x7f]') +# Tab / newline / CR are SILENTLY REMOVED by urllib.parse.urlparse() from +# anywhere in the URL before it produces ``.netloc`` (CPython's +# ``_UNSAFE_URL_BYTES_TO_REMOVE``, implementing the WHATWG "tab/newline removal" +# rule). The transport — ``http.client`` via ``urllib.request.Request(url).host`` +# — KEEPS them, so the netloc we validate diverges from the host urllib connects +# to: ``https://idp\tevil/token`` parses with host ``idpevil`` (or a *trusted* +# host, if the byte splits one a check relies on) while urllib targets the raw +# ``idp\tevil``. Because urlparse drops these before _UNSAFE_AUTHORITY_RE (which +# does list ``\s``/``\x00-\x1f``) ever sees them, they must be caught on the RAW +# url instead. A legitimate credential endpoint never contains them anywhere. +_URL_STRIPPED_BYTES = ('\t', '\n', '\r') + def _reject_confusable_authority(url: str, *, label: str) -> None: """Reject a credential URL whose authority urllib may resolve unlike parsed.""" @@ -322,7 +334,13 @@ def _reject_confusable_authority(url: str, *, label: str) -> None: # is a homoglyph/confusable spoofing vector (the renderer already distrusts # it for display), AND http.client can't even encode it — it would otherwise # raise a raw UnicodeEncodeError from the transport rather than a typed error. - if ('@' in netloc or not netloc.isascii() + # The _URL_STRIPPED_BYTES check runs on the RAW url, not netloc, because + # urlparse has already removed those bytes from netloc (see above) — without + # it, a tab/newline/CR in the authority would slip this guard yet still reach + # the transport, the exact validated-vs-connected divergence this exists to + # stop. + if (any(b in url for b in _URL_STRIPPED_BYTES) + or '@' in netloc or not netloc.isascii() or _UNSAFE_AUTHORITY_RE.search(netloc)): raise OidcConfigError( f'The OIDC {label} URL {url!r} has an unsafe authority (userinfo ' diff --git a/test/test_auth.py b/test/test_auth.py index 93495bb9..47533672 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -3117,6 +3117,35 @@ def test_non_ascii_authority_endpoint_rejected(self): self._validate('https://xn--bcher-kva.example/token', 'https://xn--bcher-kva.example/device') + def test_tab_newline_cr_authority_rejected(self): + # M1: urllib.parse.urlparse() SILENTLY REMOVES tab/newline/CR from the URL + # before producing .netloc, while the transport (http.client via + # urllib.request.Request.host) keeps them — so the host validated diverges + # from the host connected to. The dangerous case is a byte that SPLITS a + # trusted host: 'https://idp\tgood/token' parses with hostname 'idpgood' + # via .netloc, but 'https://idp.goo\td/token' can leave the validated host + # equal to a trusted name while urllib targets the raw bytes. These must be + # rejected fail-closed on every construction path; _UNSAFE_AUTHORITY_RE + # never sees them (urlparse stripped them first), so the guard checks the + # RAW url. + from questdb.auth._discovery import _reject_confusable_authority + for url in ('https://idp.good\tevil.example/token', # tab merges labels + 'https://idp\t.good/token', # tab splits a host + 'https://idp.good\nevil.example/token', # newline + 'https://idp.good\revil.example/token'): # CR + with self.assertRaises(OidcConfigError): + _reject_confusable_authority(url, label='token endpoint') + with self.assertRaises(OidcConfigError): # public co-location entry + self._validate(url, url) + with self.assertRaises(OidcConfigError): # and the full constructor + OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint=url, + token_endpoint=url, + renderer=Renderer()) + # A clean ASCII authority is NOT flagged. + self._validate('https://idp.good/token', 'https://idp.good/device') + def test_confusable_issuer_rejected(self): # M1 (defense-in-depth): a confusable issuer authority would make the # issuer-origin pin compare against the wrong host. With explicit From 99c2d688cb98a979ae18badeb0c4328ed2370b46 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 29 Jun 2026 14:07:23 +0100 Subject: [PATCH 084/104] fix(auth): minor review fixes (link, host, return type) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit m1: _safe_link_url rejected only surrounding whitespace, but urlparse silently removes tab/newline/CR from anywhere in the URL before parsing — so it would vet the stripped form yet return (→ href / browser / QR) the original with the bytes intact. Reject a URL carrying them so the value vetted equals the value returned. Production already strips via _safe_target; this closes the standalone footgun. m2: an explicit host="[::1]" override reached the PG driver bracketed (→ a confusing connection failure on a copy-pasted IPv6 literal), while the URL-derived path is unbracketed by urlparse. Strip a surrounding [...] so both paths hand the driver a bare address, matching the docstring. m3: post_form's return annotation said tuple[int, Dict] but it returns _PostResult (a tuple subclass carrying .retry_after); correct it. Add regression tests for m1 and m2. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_adapters.py | 8 ++++++++ src/questdb/auth/_http.py | 2 +- src/questdb/auth/_render.py | 9 +++++++++ test/test_auth.py | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/questdb/auth/_adapters.py b/src/questdb/auth/_adapters.py index 3ebd33cd..c10f9371 100644 --- a/src/questdb/auth/_adapters.py +++ b/src/questdb/auth/_adapters.py @@ -88,6 +88,14 @@ def _require_host(url: str, host: Optional[str] = None) -> str: f'The QuestDB URL {url!r} has no host. Use a URL with an explicit ' 'host (e.g. "https://questdb.example.com:9000"), or pass host=... ' 'to the adapter.') + # An explicit host="[::1]" override arrives bracketed; the URL-derived path is + # already unbracketed (urlparse strips the brackets off an IPv6 literal). The + # drivers take a BARE address, so strip a single surrounding [...] here too, + # keeping the "returned host is unbracketed" contract for both paths. Done + # before the illegal-char check so it validates the bare host handed to the + # driver (and any junk inside the brackets is still caught). + if resolved.startswith('[') and resolved.endswith(']') and len(resolved) > 2: + resolved = resolved[1:-1] if _ILLEGAL_HOST_CHARS.search(resolved): raise OidcConfigError( f'The QuestDB host {resolved!r} contains an illegal character ' diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 72f8f31a..69073eca 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -451,7 +451,7 @@ def post_form( headers: Optional[Mapping[str, str]] = None, timeout: float = _DEFAULT_TIMEOUT, ctx: Optional[ssl.SSLContext] = None, - insecure: bool = False) -> tuple[int, Dict[str, Any]]: + insecure: bool = False) -> '_PostResult': """ POST a form-url-encoded body and parse the JSON response. diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 3cbd5911..7af0bb83 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -167,6 +167,15 @@ def _safe_link_url(url: Optional[str]) -> Optional[str]: # value we return (and hand to the href / webbrowser.open()), not the # untrimmed original. url = url.strip() + # urlparse() also silently REMOVES tab/newline/CR from anywhere in the URL + # before parsing, so a value carrying them would be vetted as its stripped + # form yet returned (→ the href / webbrowser.open() / QR) with them intact — + # the value vetted would not equal the value returned. Reject such a URL so + # the invariant holds even when this is called directly. (Production always + # passes a _strip_control'd value via _safe_target, which removes these + # already, so this never fires there; it closes the standalone footgun.) + if any(c in url for c in '\t\n\r'): + return None try: parts = urllib.parse.urlparse(url) scheme = (parts.scheme or '').lower() diff --git a/test/test_auth.py b/test/test_auth.py index 47533672..cf599b21 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2778,6 +2778,22 @@ def test_require_host_with_conf_metachars_rejected(self): _require_host('https://db.example.com:9000', 'questdb.example.com'), 'questdb.example.com') + def test_require_host_unbrackets_explicit_ipv6(self): + # m2: psycopg / SQLAlchemy take a BARE address. The URL-derived path is + # already unbracketed by urlparse, but an explicit host="[::1]" override + # used to reach the driver bracketed (→ a confusing connection failure on + # a copy-pasted IPv6 literal). Both paths must yield the bare address. + self.assertEqual(_require_host('https://[::1]:9000'), '::1') # URL + self.assertEqual( + _require_host('https://db:9000', '[::1]'), '::1') # override + self.assertEqual( + _require_host('https://db:9000', '[2001:db8::1]'), '2001:db8::1') + # A bare (unbracketed) literal is unchanged, and junk inside brackets is + # still caught by the illegal-char guard after stripping. + self.assertEqual(_require_host('https://db:9000', '::1'), '::1') + with self.assertRaises(OidcConfigError): + _require_host('https://db:9000', '[evil;sslmode=disable]') + def test_pg_port_validation(self): # A non-integer pg_port (e.g. a port read from an env var without int()) # must surface as OidcConfigError, not a bare ValueError / driver error @@ -3844,6 +3860,25 @@ def test_safe_link_url_rejects_userinfo_and_confusable_host(self): 'https://accounts.google.com/o/oauth2/device/code'): self.assertEqual(_safe_link_url(url), url, f'should accept {url!r}') + def test_safe_link_url_rejects_interior_tab_newline_cr(self): + # m1: urlparse() silently REMOVES tab/newline/CR from the URL before + # parsing, so _safe_link_url would otherwise VET the stripped form yet + # RETURN the original (→ href / browser / QR) with the bytes intact — the + # value vetted would not equal the value returned/clicked. It must return + # None so that invariant holds when called directly. + from questdb.auth._render import _safe_link_url, _safe_target + for url in ('https://idp.example\t.com/device', # tab splits host label + 'https://idp.example.com/de\tvice', # tab in path + 'https://idp.example.com\n/device', # newline + 'https://idp.example.com\r/device'): # CR + self.assertIsNone(_safe_link_url(url), f'should reject {url!r}') + # The production entry point (_safe_target) strips control chars first, so + # a raw response value carrying them is sanitized to a single vetted target + # rather than rejected — display, browser and QR all see the same clean URL. + self.assertEqual( + _safe_target('https://idp.example.com\n/device'), + 'https://idp.example.com/device') + def test_render_link_inert_for_dangerous_scheme(self): from questdb.auth._render import _render_link safe = _render_link('https://idp/x') From b37fe54493f956334150751f054bbf21eb087290 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 29 Jun 2026 16:00:33 +0100 Subject: [PATCH 085/104] fix(auth): public doc xrefs; non-object JWT test - Point docstring :class: refs at the public questdb.auth.* targets that api.rst documents, instead of the private _device/_errors module paths (unresolved under the -nW nitpicky doc build). - Add a test that a well-formed JWT whose payload decodes to non-object JSON (list/str/number) reads as no-claims and doesn't crash the success-path identity decode, guarding _decode_jwt_claims. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_cache.py | 2 +- src/questdb/auth/_discovery.py | 2 +- src/questdb/auth/_render.py | 4 ++-- test/test_auth.py | 24 ++++++++++++++++++++++++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index baad8cb1..71522b61 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -40,7 +40,7 @@ class TokenSet: IdP tokens plus their expiry. ``frozen`` because the lock-free fast path in - :class:`~questdb.auth._device.OidcDeviceAuth` reads a published ``TokenSet`` + :class:`~questdb.auth.OidcDeviceAuth` reads a published ``TokenSet`` without a lock, which is safe only if its fields never change; use :func:`dataclasses.replace` for a modified copy. The secret fields are excluded from ``repr`` so a token can't leak into a log or traceback. diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 30b682b4..e3387617 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -58,7 +58,7 @@ class OidcConfig: """Resolved OIDC parameters needed to run the device flow. - ``frozen`` because :class:`~questdb.auth._device.OidcDeviceAuth` reads + ``frozen`` because :class:`~questdb.auth.OidcDeviceAuth` reads ``self.config`` (and the ``cache_key`` derived from it) on the lock-free fast path; the fields are set once at construction and never reassigned, so freezing makes that immutability structural rather than convention. diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 7af0bb83..b9976f12 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -68,7 +68,7 @@ def _kernel_allows_stdin() -> bool: is no human to authorize, and an input request would raise ``StdinNotImplementedError``. ipykernel records the current request's value on the kernel as ``_allow_stdin``; read it so the device flow fails fast with - :class:`~questdb.auth._errors.OidcInteractionRequired` instead of polling to + :class:`~questdb.auth.OidcInteractionRequired` instead of polling to the device-code deadline. papermill sets no environment variable (its ``PAPERMILL_*_PATH`` values are opt-in *notebook parameters*, not ``os.environ`` entries), so the kernel's stdin flag — not an env var — is the @@ -332,7 +332,7 @@ def _strip_control(text: Optional[str]) -> str: IdP put in an ``error`` field) is coerced through ``str()`` rather than raising. This sanitizer runs on untrusted input from several sites and must never raise — a ``TypeError`` here would escape the module's typed-error - contract (see :class:`~questdb.auth._errors.OidcDeviceFlowError`). + contract (see :class:`~questdb.auth.OidcDeviceFlowError`). """ if not text: return '' diff --git a/test/test_auth.py b/test/test_auth.py index cf599b21..3515d2c7 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1188,6 +1188,30 @@ def test_deeply_nested_jwt_payload_does_not_crash(self): auth = self.make_auth() self.assertEqual(auth.token(), nested) + def test_non_object_jwt_payload_does_not_crash(self): + # A well-formed (3-part) JWT whose base64 payload decodes to valid JSON + # that is NOT an object — a list / string / number from a buggy or + # hostile IdP — must read as no-claims, not crash. _decode_jwt_claims + # guards this with `isinstance(claims, dict)`; without it, claims.get() + # in _tokenset_from_response (sub) and _identity_from_claims would raise + # AttributeError on the SUCCESS path, discarding an already-authorized + # token and re-prompting on every later token() call. See + # _decode_jwt_claims. + from questdb.auth._device import _decode_jwt_claims + for payload in ([1, 2, 3], 'a-string', 42, 3.5): + self.assertEqual(_decode_jwt_claims(_jwt(payload)), {}) + # End-to-end: the IdP returns an id_token whose payload is a JSON array; + # the success-path identity decode must degrade to no-identity and the + # token must still be returned and cached. + array_token = _jwt([1, 2, 3]) + self.state.token_script = [(200, { + 'id_token': array_token, 'token_type': 'Bearer', + 'expires_in': 3600})] + auth = self.make_auth() + self.assertEqual(auth.token(), array_token) + self.assertEqual(auth._tokens.id_token, array_token) + self.assertIsNone(auth._tokens.sub) + def test_non_string_token_fields_do_not_crash(self): # A buggy/hostile IdP returning a non-string access_token / id_token (a # JSON number/bool/object) must not crash token() with a raw From 1431df32c894235ec4f694fded5899b4a3759a51 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 29 Jun 2026 17:19:11 +0100 Subject: [PATCH 086/104] fix(auth): isolate renderer errors; reject % host - Isolate custom-renderer exceptions in the device flow: route on_prompt/on_waiting/on_failure through a best-effort _render_safe so a buggy renderer can't mask the authoritative OidcDeviceFlowError / OidcTimeoutError. An OidcError is re-raised, not swallowed, so the reentrancy guard's deadlock signal still reaches the caller; on_success stays inline-wrapped (it must never discard an already-authorized token). - Reject '%' (IPv6 zone-id / percent-encoding) in a credential-endpoint authority, matching the host hygiene in _adapters._ILLEGAL_HOST_CHARS and _render._SAFE_HOST_RE that the comment already claimed to mirror; update the comment and error message to match. - Correct the OidcConfig.scope docstring: the openid scope is added by OidcDeviceAuth in groups mode, not by the dataclass. - Add regression tests for the renderer-masking and %-authority cases. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 51 ++++++++++++++++++++++++++++------ src/questdb/auth/_discovery.py | 22 +++++++++------ test/test_auth.py | 50 +++++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 16 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 1dd557bd..f05793d0 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -859,6 +859,35 @@ def _refresh(self, tokens: TokenSet) -> TokenSet: # -- device flow (RFC 8628) --------------------------------------------- + def _render_safe(self, callback, *args) -> None: + """Invoke a renderer callback best-effort, swallowing renderer bugs. + + The on_prompt / on_waiting / on_failure callbacks are cosmetic and must + never abort the flow or mask the authoritative typed error. A custom + ``renderer`` is user code: were one of its callbacks to raise an ordinary + exception (a buggy display backend), it would otherwise replace the + OidcDeviceFlowError / OidcTimeoutError describing the real sign-in outcome + with its own exception, breaking the module's typed-error contract (every + failure path raises an OidcError subclass). The built-in renderers already + swallow their own I/O errors (TerminalRenderer._write); this extends the + same guarantee to every callback, including a user-supplied one. + + An ``OidcError`` is deliberately NOT swallowed: a renderer callback that + re-enters this instance's token()/clear() trips _guard_reentrancy, which + raises OidcError to signal the (otherwise deadlocking) misuse — that + signal must reach the caller, not be silently dropped. (on_success is + wrapped inline instead, and swallows even an OidcError, because nothing + cosmetic may discard the token the user already authorized — and its guard + must also cover the JWT-claim / lifetime computation evaluated before the + call.) + """ + try: + callback(*args) + except OidcError: + raise + except Exception: + pass + def _run_device_flow(self) -> TokenSet: if not self._is_interactive(): raise OidcInteractionRequired( @@ -868,7 +897,7 @@ def _run_device_flow(self) -> TokenSet: 'client-credentials grant for non-interactive contexts.') resp = self._request_device_code() - self._renderer.on_prompt(resp) + self._render_safe(self._renderer.on_prompt, resp) self._maybe_open_browser(resp) tokens = self._poll_for_token(resp) # The grant has completed and `tokens` is valid. Rendering the success @@ -982,13 +1011,14 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: while True: remaining = deadline - self._monotonic() if remaining <= 0: - self._renderer.on_failure( + self._render_safe( + self._renderer.on_failure, 'Code expired — run the cell again to retry.') raise OidcTimeoutError( 'The device code expired before authorization completed. ' 'Run the sign-in again.', error='expired_token') - self._renderer.on_waiting(remaining) + self._render_safe(self._renderer.on_waiting, remaining) # Never sleep past the deadline (remaining > 0 here). self._sleep(min(interval, remaining)) @@ -1009,7 +1039,8 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: # return and _NoRedirect won't follow — instead of polling on to # a misleading "code expired". if _http_status_is_terminal(getattr(e, 'status', None)): - self._renderer.on_failure( + self._render_safe( + self._renderer.on_failure, 'Sign-in failed: the identity provider rejected the ' 'request.') raise OidcDeviceFlowError( @@ -1052,7 +1083,8 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: # misconfiguration, not a transient poll state. Raise a terminal # error rather than cache an unusable token and silently re-run # the whole flow on every later token() call. - self._renderer.on_failure( + self._render_safe( + self._renderer.on_failure, 'Sign-in failed: the identity provider did not return the ' 'token this server requires.') raise self._missing_required_token_error() @@ -1075,7 +1107,8 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: # an OAuth error field can't be mistaken for a live poll state and # polled on to a misleading "code expired". if 300 <= status < 400: - self._renderer.on_failure( + self._render_safe( + self._renderer.on_failure, 'Sign-in failed: the identity provider rejected the request.') raise OidcDeviceFlowError( 'Device flow failed: the IdP returned an unexpected ' @@ -1092,7 +1125,8 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: interval, retry_after, at_least_increment=True) continue if error == 'expired_token': - self._renderer.on_failure( + self._render_safe( + self._renderer.on_failure, 'Code expired — run the cell again to retry.') raise OidcTimeoutError( 'The device code expired before authorization completed. ' @@ -1100,7 +1134,8 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: error=error) # access_denied or any other terminal error. description = body.get('error_description') or error or 'unknown error' - self._renderer.on_failure(f'Sign-in failed: {description}') + self._render_safe( + self._renderer.on_failure, f'Sign-in failed: {description}') raise OidcDeviceFlowError( f'Device flow failed: {description}', status=status, diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index e3387617..b3ba850e 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -71,7 +71,9 @@ class OidcConfig: device_authorization_endpoint: str """IdP device-authorization endpoint (RFC 8628 §3.1).""" scope: str = 'openid' - """Space-separated scopes; ``openid`` is added automatically in groups mode.""" + """Space-separated scopes. Stored verbatim; ``openid`` is added automatically + by :class:`~questdb.auth.OidcDeviceAuth` (not by this dataclass) in groups + mode, since presenting the ``id_token`` requires it.""" groups_in_token: bool = False """When true, present the ``id_token`` (groups encoded in it) rather than the ``access_token`` — mirroring QuestDB's own selection.""" @@ -307,11 +309,14 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: # connection. So ``https://attacker.evil\@idp.good/token`` validates as host # ``idp.good`` (passing the issuer-origin pin) while urllib connects to the whole # ``attacker.evil\@idp.good``. A real credential-endpoint authority never carries -# userinfo, a non-ASCII character, a backslash, whitespace or a control char, so -# reject them — fail closed — mirroring the host hygiene already enforced in -# ``_adapters._ILLEGAL_HOST_CHARS`` and ``_render._SAFE_HOST_RE``. (Non-ASCII is -# checked with ``str.isascii`` in the function, not this regex.) -_UNSAFE_AUTHORITY_RE = re.compile(r'[\\\s\x00-\x1f\x7f]') +# userinfo, a non-ASCII character, a backslash, whitespace, a control char, or a +# ``%`` (percent-encoding, or an IPv6 zone-id such as ``fe80::1%eth0`` — an +# on-host link-local artifact, never a way to reach a remote IdP), so reject them +# — fail closed — mirroring the host hygiene already enforced in +# ``_adapters._ILLEGAL_HOST_CHARS`` and ``_render._SAFE_HOST_RE`` (both of which +# also reject ``%``). (Non-ASCII is checked with ``str.isascii`` in the function, +# not this regex.) +_UNSAFE_AUTHORITY_RE = re.compile(r'[\\\s\x00-\x1f\x7f%]') # Tab / newline / CR are SILENTLY REMOVED by urllib.parse.urlparse() from # anywhere in the URL before it produces ``.netloc`` (CPython's @@ -344,8 +349,9 @@ def _reject_confusable_authority(url: str, *, label: str) -> None: or _UNSAFE_AUTHORITY_RE.search(netloc)): raise OidcConfigError( f'The OIDC {label} URL {url!r} has an unsafe authority (userinfo ' - "'@', a non-ASCII character, a backslash, whitespace, or a control " - 'character). A real endpoint host is plain ASCII (a DNS name, an ' + "'@', a non-ASCII character, a backslash, whitespace, a control " + "character, or '%'). A real endpoint host is plain ASCII (a DNS " + 'name, an ' 'xn-- punycode label, or an IP literal) and never contains these, so ' 'this indicates a malformed or tampered configuration; refusing to ' 'send credentials to a host the HTTP transport may resolve ' diff --git a/test/test_auth.py b/test/test_auth.py index 3515d2c7..71902685 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -771,6 +771,30 @@ def on_success(self, identity, expires_in): auth = self.make_auth(renderer=_Boom()) self.assertEqual(auth.token(), ID_TOKEN) + def test_raising_failure_renderer_does_not_mask_typed_error(self): + # The on_prompt/on_waiting/on_failure callbacks are best-effort too: a + # custom renderer whose on_failure raises must NOT replace the + # authoritative OidcDeviceFlowError describing the real sign-in outcome + # with its own exception (which would break the typed-error contract — + # every failure path raises an OidcError subclass). The caller still sees + # the access_denied error, not the renderer's RuntimeError. + class _Boom(Renderer): + def on_failure(self, message): + raise RuntimeError('renderer blew up') + + self.state.token_script = [ + (400, {'error': 'access_denied', + 'error_description': 'user said no'}), + ] + auth = self.make_auth(renderer=_Boom()) + with self.assertRaises(OidcDeviceFlowError) as cm: + auth.token() + self.assertEqual(cm.exception.error, 'access_denied') + # And the non-reentrant acquisition lock is released (no state corruption). + self.assertIsNone(auth._lock_owner) + self.assertTrue(auth._lock.acquire(blocking=False)) + auth._lock.release() + def test_poll_honors_retry_after(self): # Issue 8: a slow_down poll response carrying Retry-After backs off by # that many seconds (clamped), not the fixed +5s. @@ -3186,6 +3210,32 @@ def test_tab_newline_cr_authority_rejected(self): # A clean ASCII authority is NOT flagged. self._validate('https://idp.good/token', 'https://idp.good/device') + def test_percent_authority_rejected(self): + # A '%' in a credential-endpoint authority is rejected fail-closed on + # every construction path: a real endpoint host never carries one. It is + # either an IPv6 zone-id (e.g. 'fe80::1%eth0', an on-host link-local + # artifact, never a way to reach a remote IdP) or percent-encoding (which + # urlparse keeps in .hostname but a resolver/transport may decode, + # diverging the validated host from the connected one). This mirrors the + # host hygiene in _adapters._ILLEGAL_HOST_CHARS and _render._SAFE_HOST_RE, + # which both reject '%' too. + from questdb.auth._discovery import _reject_confusable_authority + for url in ('https://[fe80::1%25eth0]/token', # IPv6 zone-id (%25 == %) + 'https://idp%2egood.evil/token'): # percent-encoded host + with self.assertRaises(OidcConfigError): + _reject_confusable_authority(url, label='token endpoint') + with self.assertRaises(OidcConfigError): # public co-location entry + self._validate(url, url) + with self.assertRaises(OidcConfigError): # and the full constructor + OidcDeviceAuth( + client_id='questdb', + device_authorization_endpoint=url, + token_endpoint=url, + renderer=Renderer()) + # A plain (zone-id-free) IPv6 literal authority is NOT flagged. + self._validate('https://[fe80::1]:443/token', + 'https://[fe80::1]:443/device') + def test_confusable_issuer_rejected(self): # M1 (defense-in-depth): a confusable issuer authority would make the # issuer-origin pin compare against the wrong host. With explicit From 57e522da57fde1250dd873002facffc241e508b2 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 29 Jun 2026 23:54:42 +0100 Subject: [PATCH 087/104] opt-in token persistence --- CHANGELOG.rst | 14 +- docs/api.rst | 20 + docs/auth.rst | 69 ++- examples/oidc_device_auth.py | 17 + src/questdb/auth/__init__.py | 10 + src/questdb/auth/_cache.py | 16 + src/questdb/auth/_device.py | 385 +++++++++++- src/questdb/auth/_discovery.py | 18 +- src/questdb/auth/_http.py | 27 +- src/questdb/auth/_render.py | 26 +- src/questdb/auth/_store.py | 724 +++++++++++++++++++++++ test/test.py | 2 + test/test_auth.py | 1006 +++++++++++++++++++++++++++++++- 13 files changed, 2281 insertions(+), 53 deletions(-) create mode 100644 src/questdb/auth/_store.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 08205e6f..23d76eb4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -42,8 +42,18 @@ Highlights: Google: ``accounts.google.com`` issuer, ``oauth2.googleapis.com`` endpoints), while still pinning endpoints advertised over an untrusted ``/settings`` channel to the issuer. -* In-process token cache with silent refresh (tokens are never written to - disk). +* In-process token cache with silent refresh; in-memory only by default (no + token ever written to disk). +* Opt-in **token persistence** (:class:`~questdb.auth.FileTokenStore`, passed as + ``token_store=``) so a restarted process resumes from a saved refresh token + instead of prompting again. The default file store keeps one owner-only + (``0600``) plaintext file per identity under ``~/.questdb/oidc-tokens/``, + written atomically and coordinated across processes with a lock file; supply a + custom :class:`~questdb.auth.TokenStore` to back it with an OS keychain. The + on-disk format is a language-neutral contract shared with the Java client. The + per-request ``timeout`` is capped at 120s (matching the Java client) so a slow + refresh held under the file store's cross-process lock can't outlast its + staleness window; a larger value raises ``OidcConfigError`` at construction. * Convenience adapters (:func:`~questdb.auth.sqlalchemy_engine`, :func:`~questdb.auth.psycopg_connect`) that wire the auto-refreshed token into PG-wire as the ``_sso`` password. diff --git a/docs/api.rst b/docs/api.rst index 3aec0901..e50bc2bd 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -92,6 +92,26 @@ See the :ref:`oidc_auth` guide for an overview. :undoc-members: :show-inheritance: +.. autoclass:: questdb.auth.FileTokenStore + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: questdb.auth.TokenStore + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: questdb.auth.TokenStoreKey + :members: + :undoc-members: + :show-inheritance: + +.. autoclass:: questdb.auth.PersistedToken + :members: + :undoc-members: + :show-inheritance: + .. autoexception:: questdb.auth.OidcError :show-inheritance: diff --git a/docs/auth.rst b/docs/auth.rst index 878d46cb..8c21e6fb 100644 --- a/docs/auth.rst +++ b/docs/auth.rst @@ -160,9 +160,66 @@ is raised instead, so you can retry without being needlessly re-prompted. A lock serializes refresh so parallel cells/threads don't double-prompt. The token is held in a process-global, in-memory cache, so re-running a cell -reuses it instead of re-prompting; a kernel restart re-prompts once. Tokens are -deliberately never written to disk: an interactive sign-in is cheap relative to -the risk of a refresh token sitting in a plaintext file at rest. +reuses it instead of re-prompting; a kernel restart re-prompts once. By default +nothing is written to disk — opt into persistence (below) to survive a restart. + +Persisting the token across restarts +------------------------------------- + +By default the token lives in memory only, so a restarted process (a fresh +kernel, a re-run script, a new container) has to run the device flow again. Pass +a ``token_store`` to persist it; the restarted process then resumes from the +saved refresh token — a silent call to the token endpoint — instead of prompting +again: + +.. code-block:: python + + from questdb.auth import OidcDeviceAuth, FileTokenStore + + auth = OidcDeviceAuth.from_questdb( + "https://questdb.example.com:9000", + token_store=FileTokenStore.at_default_location()) + auth.token() # prompts the first time; after a restart it refreshes silently + +:meth:`~questdb.auth.FileTokenStore.at_default_location` writes one file per +identity under ``~/.questdb/oidc-tokens/`` (override the directory with the +``QUESTDB_CLIENT_OIDC_TOKEN_STORE_DIR`` environment variable). The file name is a +hash of the endpoints, client id, scope, audience and groups-in-token mode, so +tokens for different servers or identities never collide. After a restart, +:meth:`~questdb.auth.OidcDeviceAuth.token` also works as the *first* call — no +explicit sign-in needed — which suits a long-lived adapter such as +:func:`~questdb.auth.sqlalchemy_engine`. +:meth:`~questdb.auth.OidcDeviceAuth.clear` removes the persisted entry and forces +a fresh sign-in next time. + +The token is stored as **plaintext JSON protected by file permissions** — +``0600`` file, ``0700`` directory on POSIX systems (Linux, macOS), the same +approach ``gcloud``, ``aws`` and ``gh`` take. On Windows these POSIX permissions +cannot be enforced, so the file relies on the user-profile directory's default +ACL and the client prints a one-line warning to ``stderr`` the first time it +cannot enforce them. Enabling persistence therefore writes a long-lived refresh +token to disk: anyone who can read the file holds a credential until it expires +or is revoked. To encrypt it at rest, implement your own +:class:`~questdb.auth.TokenStore` (backed by an OS keychain or a secrets manager) +and pass that instead of :class:`~questdb.auth.FileTokenStore`. A persisted file +is treated as **untrusted input** on load — a tampered, corrupt, oversized, or +identity-mismatched entry is ignored (the client falls back to a refresh or an +interactive sign-in), and the bearer token it serves (put verbatim into an +``Authorization`` header or the PG-wire ``_sso`` password) is rejected if it +carries control or non-ASCII characters, so a tampered credential is never placed +on the wire. + +:class:`~questdb.auth.FileTokenStore` is safe to share between processes that +sign in as the same identity: each update is written atomically (so a concurrent +reader never sees a half-written credential), and when the identity provider +rotates the refresh token on each refresh, the read-refresh-write is serialized +across processes with a lock file so they don't race each other into an +unnecessary re-prompt. That coordination is **best-effort**: under heavy +contention, or if the lock can't be taken, a process falls back to refreshing +without it — still integrity-safe (the atomic write stands), just uncoordinated +for that one refresh, which on a rotating-refresh IdP may cost an extra sign-in. +The on-disk format is a language-neutral contract, so the Java QuestDB client and +this one can share the same file. Non-interactive contexts ------------------------- @@ -247,6 +304,12 @@ Security notes authorization endpoint (so it must be discovered from the IdP), ``issuer=`` is **required** for exactly this reason — the helper refuses to guess the discovery origin from the server-supplied token endpoint. +* **Token persistence is opt-in and off by default.** Passing a ``token_store`` + writes a long-lived refresh token to disk; :class:`~questdb.auth.FileTokenStore` + protects it with owner-only file permissions (``0600``/``0700``) rather than + encryption. A persisted file is validated as untrusted input on load and the + served bearer token is rejected if it carries control / non-ASCII characters. + See `Persisting the token across restarts`_. * Adapters avoid logging the token / PG DSN. Avoid logging them yourself. * Standard proxy / CA settings (``HTTPS_PROXY``, ``REQUESTS_CA_BUNDLE``, ``SSL_CERT_FILE``) are honoured for the IdP / discovery transport; you can diff --git a/examples/oidc_device_auth.py b/examples/oidc_device_auth.py index eb5df9e2..354ac1d5 100644 --- a/examples/oidc_device_auth.py +++ b/examples/oidc_device_auth.py @@ -13,6 +13,7 @@ import sys from questdb.auth import ( + FileTokenStore, OidcDeviceAuth, OidcError, psycopg_connect, @@ -55,6 +56,22 @@ def pg_wire(url: str = QUESTDB_URL): print(cur.fetchone()) +def persist_across_restarts(url: str = QUESTDB_URL) -> OidcDeviceAuth: + """Survive a process restart without prompting again. + + By default the token is in-memory only, so a restarted kernel / script + re-runs the device flow. Pass a ``token_store`` to persist it: the restarted + process resumes from the saved refresh token (a silent token-endpoint call) + instead of re-prompting. ``FileTokenStore`` writes one file per identity + under ``~/.questdb/oidc-tokens/`` (``0600``, owner-only); supply your own + ``TokenStore`` to back it with an OS keychain for at-rest encryption. + """ + auth = OidcDeviceAuth.from_questdb( + url, token_store=FileTokenStore.at_default_location()) + auth.token() # prompts the first time; silent on later runs / after restart + return auth + + def bring_your_own_client(url: str = QUESTDB_URL): """You just want the token (REST / ingestion / anything).""" auth = OidcDeviceAuth.from_questdb(url) diff --git a/src/questdb/auth/__init__.py b/src/questdb/auth/__init__.py index 1e8b190b..5c1d389d 100644 --- a/src/questdb/auth/__init__.py +++ b/src/questdb/auth/__init__.py @@ -61,9 +61,16 @@ OidcDeviceFlowError, OidcTimeoutError, ) +from ._store import ( + FileTokenStore, + PersistedToken, + TokenStore, + TokenStoreKey, +) from ._adapters import sqlalchemy_engine, psycopg_connect __all__ = [ + 'FileTokenStore', 'OidcConfig', 'OidcConfigError', 'OidcDeviceAuth', @@ -72,7 +79,10 @@ 'OidcInteractionRequired', 'OidcNetworkError', 'OidcTimeoutError', + 'PersistedToken', 'TokenSet', + 'TokenStore', + 'TokenStoreKey', 'psycopg_connect', 'sqlalchemy_engine', ] diff --git a/src/questdb/auth/_cache.py b/src/questdb/auth/_cache.py index 71522b61..43e82fe8 100644 --- a/src/questdb/auth/_cache.py +++ b/src/questdb/auth/_cache.py @@ -192,3 +192,19 @@ def store_if_current( return False _MEMORY_STORE[key] = replace(tokens) return True + + def is_current(self, key: str, generation: int) -> bool: + """ + True if no :meth:`clear` bumped ``key``'s generation since ``generation``. + + The read-only companion to :meth:`store_if_current`, for a caller that has + already committed the in-memory write but must perform a *second*, + non-atomic side effect (persisting the token to disk): it re-checks this + immediately before that side effect — under the store's own lock — so a + ``clear()`` that landed in between (and deleted the persisted file) is + still honored and the side effect is skipped rather than resurrecting the + file. Same generation semantics as :meth:`store_if_current`: an in-flight + acquisition keeps a bumped generation alive, so this observes it. + """ + with _MEMORY_LOCK: + return _MEMORY_GENERATION.get(key, 0) == generation diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index f05793d0..dc1212aa 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -30,6 +30,7 @@ import binascii import json import math +import sys import threading import time import webbrowser @@ -47,6 +48,12 @@ OidcTimeoutError, ) from ._http import build_ssl_context, post_form, safe_urlparse +from ._store import ( + PersistedToken, + TokenStore, + TokenStoreKey, + _canonical_endpoint, +) from ._render import ( Renderer, _safe_target, @@ -68,6 +75,13 @@ _DEFAULT_EXPIRES_IN = 300 # token TTL fallback (absent/invalid/<=0) _MAX_EXPIRES_IN = 3600 # cap on the token TTL +# Upper bound on the configurable per-request HTTP timeout (seconds). A +# token-endpoint round-trip never needs longer, and bounding it keeps a refresh +# held under the FileTokenStore cross-process lock safely shorter than that +# store's lock-staleness window, so a slow refresh's live lock is not stolen by a +# peer. Matches the Java client's 120s cap. +_MAX_TIMEOUT = 120 + # Clamp the device-authorization timing fields (RFC 8628): a hostile/buggy # response must not time the flow out before its first poll, pin the polling # thread (which holds the acquisition lock) in one huge sleep, or keep the loop @@ -228,6 +242,35 @@ def _validate_positive_number(value: Any, name: str) -> None: f'got {value!r}') +def _validate_timeout(value: Any) -> None: + """Require ``timeout`` to be positive, finite, and within :data:`_MAX_TIMEOUT`. + + The cap mirrors the Java client: a token-endpoint round-trip never needs + longer, and a larger value could let a slow refresh outlast the + :class:`~questdb.auth.FileTokenStore` cross-process lock's staleness window, + so a peer could steal a live holder's lock mid-refresh. + """ + _validate_positive_number(value, 'timeout') + if value > _MAX_TIMEOUT: + raise OidcConfigError( + f'timeout must not exceed {_MAX_TIMEOUT} seconds; a token-endpoint ' + 'round-trip never needs longer, and a larger value could let a slow ' + "refresh outlast the token store's cross-process lock staleness " + 'window.') + + +def _has_only_token_chars(token: str) -> bool: + """True if every character of ``token`` is printable ASCII (0x20–0x7e). + + A real OAuth token is printable ASCII; a control or non-ASCII character would + be smuggled verbatim into an ``Authorization: Bearer`` header or a PG-wire + ``_sso`` password (a decoded CR/LF is a header-injection vector), so a token + loaded from the attacker-writable persistence file is rejected unless it + passes this check. + """ + return all(0x20 <= ord(c) <= 0x7e for c in token) + + class OidcDeviceAuth: """ Acquire and refresh an OIDC token via the device authorization grant. @@ -242,6 +285,14 @@ class OidcDeviceAuth: Acquisition is serialized so concurrent callers don't double-prompt, while a valid cached token is returned without blocking on another's sign-in. + Token state is in-memory only by default and does not survive a process + restart. Pass a ``token_store`` (e.g. + :meth:`FileTokenStore.at_default_location() + `) to persist it, so a + restarted process resumes from the saved refresh token — one silent + token-endpoint round-trip — instead of running the device flow again. See + :ref:`oidc_auth`. + **Concurrency note.** The lock is held for a whole interactive sign-in (up to the device-code lifetime, ~30 min): a caller with a *valid* cached token never blocks, but one whose token is missing/expired waits behind the @@ -291,6 +342,7 @@ def __init__( renderer: Optional[Renderer] = None, default_interval: int = 5, timeout: float = 30, + token_store: Optional[TokenStore] = None, _clock=None): # injectable time source for testing # Validate types up front so a bad-typed arg raises the module's typed # error, not a bare AttributeError/TypeError surfacing later from @@ -323,7 +375,7 @@ def __init__( # socket call; a non-numeric value would otherwise escape as a bare # TypeError rather than the typed error this block exists to raise. _validate_positive_number(default_interval, 'default_interval') - _validate_positive_number(timeout, 'timeout') + _validate_timeout(timeout) # Sending the id_token requires the ``openid`` scope. if groups_in_token and 'openid' not in scope.split(): @@ -382,6 +434,31 @@ def __init__( # instance's token()/clear() — into a clear error instead of a deadlock. self._lock_owner: Optional[int] = None self._tokens: Optional[TokenSet] = None + # Opt-in token persistence (default None == in-memory only, the previous + # behaviour). Key any persisted entry by the identity it belongs to: + # canonicalise the endpoints (lower-case scheme/host, explicit port) and + # use the already-normalised audience, so the hash matches across + # processes and language clients sharing this identity. + self._token_store = token_store + self._store_key: Optional[TokenStoreKey] = None if token_store is None \ + else TokenStoreKey( + client_id=self.config.client_id, + token_endpoint=_canonical_endpoint(self.config.token_endpoint), + device_authorization_endpoint=_canonical_endpoint( + self.config.device_authorization_endpoint), + scope=self.config.scope, + audience=self.config.audience, + groups_in_token=self.config.groups_in_token) + # Load the persisted entry at most once per instance (even if it yields + # nothing), so a missing or bad file is not re-read on every call. + self._store_load_attempted = False + # The refresh token last written to the store, so _persist_if_rotated() + # can skip the hot refresh path when the IdP does not rotate it. + self._last_persisted_refresh_token: Optional[str] = None + # True while the store's cross-process lock is held for a coordinated + # refresh (the in_lock action), so the disk save performed under it does + # not re-acquire our own lock. Read only inside that action. + self._store_lock_held = False clock = _clock or _SYSTEM_CLOCK self._sleep = clock.sleep self._monotonic = clock.monotonic @@ -409,6 +486,7 @@ def from_questdb( renderer: Optional[Renderer] = None, default_interval: int = 5, timeout: float = 30, + token_store: Optional[TokenStore] = None, _clock=None) -> 'OidcDeviceAuth': # injectable time source """ Build an :class:`OidcDeviceAuth` by discovering config from QuestDB. @@ -424,13 +502,19 @@ def from_questdb( from a server-supplied endpoint, so a tampered ``/settings`` cannot redirect the device-code / refresh-token POSTs. See :ref:`oidc_auth`. Raises :class:`OidcConfigError` if the configuration can't be resolved. + + Pass ``token_store=`` (e.g. + :meth:`FileTokenStore.at_default_location() + `) to persist the token + so a restarted process resumes from the saved refresh token instead of + prompting again; the default is in-memory only. """ # Validate before resolve_config consumes `timeout` on its /settings and # discovery HTTP calls (which run before cls() would validate it), so a # bad timeout fails fast with the typed error rather than a bare # TypeError from urllib. _validate_positive_number(default_interval, 'default_interval') - _validate_positive_number(timeout, 'timeout') + _validate_timeout(timeout) ctx = build_ssl_context(ca_bundle) cfg = resolve_config( questdb_url=url, @@ -460,6 +544,7 @@ def from_questdb( renderer=renderer, default_interval=default_interval, timeout=timeout, + token_store=token_store, _clock=_clock) # -- public API --------------------------------------------------------- @@ -539,6 +624,23 @@ def clear(self) -> None: try: self._tokens = None self._cache.clear(self.cache_key) + self._last_persisted_refresh_token = None + if self._token_store is not None: + try: + # Delete the file under the store's per-identity lock so a + # concurrent in-flight _store on another instance (which + # sees the generation just bumped by _cache.clear above) + # can't resurrect it: that save re-checks the generation + # under the same lock and skips (see _save_if_current). + self._token_store.in_lock( + self._store_key, + lambda: self._token_store.clear(self._store_key)) + except Exception as e: + # Best-effort: a store failure must not break clear(). + self._warn_persistence('clear', e) + # Don't reload the entry we just removed on the next + # token() / sign-in. + self._store_load_attempted = True finally: self._lock_owner = None @@ -641,6 +743,12 @@ def _obtain_tokens(self, *, allow_interactive: bool = True) -> TokenSet: # MemoryCache.release). generation = self._cache_generation() try: + # First call with a token store: seed self._tokens from the + # persisted entry (once), so a restart resumes from a saved + # refresh token instead of re-prompting. Adopts into this + # instance only; the shared cache and disk are written by + # _store after an acquisition. + self._maybe_load_from_store() # Promote a cached token under the lock, consulting the # shared store even when self._tokens is already set: another # instance sharing the process-global cache may have acquired @@ -703,24 +811,15 @@ def _acquire( # interactive device-flow fallback below is gated by it. tokens = self._tokens if tokens is not None and tokens.refresh_token: - try: - refreshed = self._refresh(tokens) - except OidcNetworkError: - # Transient: the refresh token is still valid, so the interactive - # flow (same network) wouldn't help and would needlessly - # re-prompt. Surface it; the cached token is kept for a retry. - raise - except OidcError: - # Refresh token rejected (expired/revoked) or unusable response: - # fall through to a fresh interactive sign-in. - pass - else: - # Accept only a refresh that yields the kind we need: some IdPs - # don't re-issue the id_token on refresh, so fall through rather - # than cache an unusable response and loop on every call. - if self._has_required_token(refreshed): - self._store(refreshed, generation) - return refreshed + # A token store serialises the read-refresh-write across processes + # (and adopts a peer's just-rotated refresh token) through its + # per-identity lock; without one this is a plain silent refresh. + # OidcNetworkError propagates (the refresh token is still valid, so + # the interactive flow — same network — wouldn't help and would + # needlessly re-prompt; the cached token is kept for a retry). + refreshed = self._try_refresh_coordinated(tokens, generation) + if refreshed is not None: + return refreshed # The refresh path is exhausted: the refresh_token is proven useless # (rejected, or the IdP won't re-issue the required kind), so the # device flow below is the only way forward. Drop the stale token — @@ -756,12 +855,256 @@ def _store(self, tokens: TokenSet, generation: int) -> None: # that bumped the generation drops the write, so clear() isn't silently # undone. self._tokens = tokens - self._cache.store_if_current(self.cache_key, tokens, generation) + stored = self._cache.store_if_current(self.cache_key, tokens, generation) + # Persist on the same condition as the shared-cache write, and re-check + # that condition again under the store lock right before the disk write + # (see _save_if_current): a concurrent clear() that bumped the generation + # and deleted the file between the CAS above and the save must not be + # undone by re-creating the file. _persist_if_rotated then skips a + # non-rotated refresh so the hot path doesn't rewrite the file every few + # minutes. + if stored: + self._persist_if_rotated(generation) def _cache_generation(self) -> int: # Per-key clear()-generation for the cross-instance CAS in _store. return self._cache.generation(self.cache_key) + # -- persistence (opt-in TokenStore) ------------------------------------ + + def _try_refresh_coordinated( + self, tokens: TokenSet, generation: int) -> Optional[TokenSet]: + # Returns the stored refreshed TokenSet, or None to fall through to the + # device flow; raises OidcNetworkError on a transient failure (the caller + # keeps the still-valid refresh token and retries later). With a token + # store, serialise the read-refresh-write across processes — and adopt a + # peer's just-rotated refresh token — through the store's per-identity + # lock; without one, just run the refresh. + if self._token_store is None: + return self._try_refresh_locally(tokens, generation) + # Mark the store lock as held for the duration of the in_lock action, so + # the disk save it performs (via _store -> _persist_if_rotated -> + # _save_if_current) writes directly rather than re-acquiring our own lock + # (which would deadlock/degrade on it). The flag is only read inside the + # action, i.e. while the lock is genuinely held. + self._store_lock_held = True + try: + return self._token_store.in_lock( + self._store_key, lambda: self._refresh_under_lock(generation)) + except OidcNetworkError: + # The refresh itself hit a transient error (raised by the action, not + # the store): propagate so the caller keeps the still-valid refresh + # token and retries later, never a needless re-prompt. + raise + except Exception as e: + # A custom store's lock backend failed (the bundled FileTokenStore + # degrades internally and never raises here). Persistence is + # best-effort, so warn and fall through to a lock-free refresh below + # rather than abort an otherwise-valid sign-in. + self._warn_persistence('lock', e) + finally: + self._store_lock_held = False + # Reached only when in_lock raised a non-network store failure above: the + # flag is now cleared, so this lock-free refresh's own persist takes its + # normal lock-acquiring path. + return self._try_refresh_locally(tokens, generation) + + def _try_refresh_locally( + self, tokens: TokenSet, generation: int) -> Optional[TokenSet]: + try: + refreshed = self._refresh(tokens) + except OidcNetworkError: + # Transient: the refresh token is still valid, so propagate for a + # retry rather than fall through to a needless interactive re-prompt. + raise + except OidcError: + # Refresh token rejected (expired/revoked) or unusable response: fall + # through to a fresh interactive sign-in. + return None + # Accept only a refresh that yields the kind we need: some IdPs don't + # re-issue the id_token on refresh, so fall through rather than cache an + # unusable response and loop on every call. + if self._has_required_token(refreshed): + self._store(refreshed, generation) + return refreshed + return None + + def _refresh_under_lock(self, generation: int) -> Optional[TokenSet]: + # Runs inside the store's cross-process lock. Re-read the store first: a + # peer sharing this identity may have refreshed (and rotated the refresh + # token) since our last load. Adopt a fresher entry and skip the network + # when it already yields a valid token; otherwise refresh with the + # freshest known refresh token (the one just adopted, so a rotated token + # is not replayed). + # + # Only re-read when the in-memory refresh token still matches what we + # last persisted. If they differ, a previous save failed (persistence is + # best-effort), so the in-memory token is newer than the on-disk one; + # re-adopting would regress it to the stale — and, on a rotating IdP, + # already-revoked — on-disk token and force a needless re-prompt. In that + # case keep the in-memory token and refresh with it. + tokens = self._tokens + if tokens is None: + return None + if tokens.refresh_token == self._last_persisted_refresh_token: + try: + fresh = self._token_store.load(self._store_key) + except Exception as e: + self._warn_persistence('load', e) + fresh = None + if self._adopt(fresh): + adopted = self._tokens + if (adopted is not None and adopted.is_valid(self._now()) + and self._has_required_token(adopted)): + # A peer already refreshed; skip the network and seed the + # shared cache from the adopted token. + self._store(adopted, generation) + return adopted + # Adopted but stale: refresh with its (possibly rotated) token. + tokens = self._tokens + return self._try_refresh_locally(tokens, generation) + + def _maybe_load_from_store(self) -> None: + if self._token_store is None or self._store_load_attempted: + return + # Attempt the disk read once per instance, even if it yields nothing, so + # a missing or bad file is not re-read on every call. + self._store_load_attempted = True + try: + persisted = self._token_store.load(self._store_key) + except Exception as e: + # Best-effort: a store read failure must not break sign-in. + self._warn_persistence('load', e) + return + self._adopt(persisted) + + def _adopt(self, persisted: Optional[PersistedToken]) -> bool: + # Build a usable TokenSet from a persisted entry and make it this + # instance's view, returning whether it was adopted. Used by the lazy + # load and by the re-read inside the cross-process lock. + if persisted is None: + return False + tokens = self._tokenset_from_persisted(persisted) + if tokens is None: + return False + self._tokens = tokens + # It is already on disk, so a later non-rotating refresh must not rewrite + # the file. + self._last_persisted_refresh_token = tokens.refresh_token + return True + + def _tokenset_from_persisted( + self, persisted: PersistedToken) -> Optional[TokenSet]: + # The file is attacker-writable, so treat the served token (the one + # token() puts verbatim into an Authorization header or a PG-wire + # password) as untrusted: reject a control/non-ASCII char — and the whole + # entry — rather than route a tampered credential onto the wire. A null + # served token is unusable. + access_token = _str_or_none(persisted.access_token) + id_token = _str_or_none(persisted.id_token) + refresh_token = _str_or_none(persisted.refresh_token) + served = id_token if self.config.groups_in_token else access_token + if not served or not _has_only_token_chars(served): + return None + # The file is attacker-writable (and may have been written under a skewed + # clock), so bound how long the loaded token is trusted exactly as a + # token from the wire: never past _MAX_EXPIRES_IN from now. Capping (not + # flooring) the expiry preserves an already-expired entry, so a stale + # access token still falls through to a refresh rather than being served + # forever. + now = self._now() + max_life = float(_MAX_EXPIRES_IN) + ttl = max(0.0, min(persisted.token_ttl, max_life)) + expires_at = min(persisted.expires_at, now + max_life) + # issued_at lets TokenSet.is_valid() cap the skew at half a short + # lifetime, exactly as a wire token does. + issued_at = expires_at - ttl + claims = (_decode_jwt_claims(id_token) + or _decode_jwt_claims(access_token)) + return TokenSet( + access_token=access_token, + id_token=id_token, + refresh_token=refresh_token, + expires_at=expires_at, + issued_at=issued_at, + token_type='Bearer', + scope=self.config.scope, + sub=_str_or_none(claims.get('sub'))) + + def _snapshot(self) -> PersistedToken: + # A PersistedToken mirroring the current in-memory token. token_ttl is the + # lifetime the expiry was derived from (expires_at - issued_at), mirroring + # how a wire response sets them; falls back to 0 when issued_at is unknown. + t = self._tokens + ttl = max(0.0, t.expires_at - t.issued_at) if t.issued_at else 0.0 + return PersistedToken( + access_token=t.access_token, + id_token=t.id_token, + refresh_token=t.refresh_token, + expires_at=t.expires_at, + token_ttl=ttl) + + def _persist_if_rotated(self, generation: int) -> None: + if self._token_store is None: + return + # Persist on a new or rotated refresh token (the interactive sign-in, or + # a provider that rotates the refresh token on every refresh); skip when + # it is unchanged, so the hot refresh path does not rewrite the file every + # few minutes. The on-disk access token then goes stale, which costs only + # one silent refresh on the next restart. With no refresh token there is + # nothing worth persisting (a restart could not resume from it anyway). + refresh_token = self._tokens.refresh_token if self._tokens else None + if refresh_token == self._last_persisted_refresh_token: + return + # Serialise the disk write against a concurrent clear()'s file delete + # through the store's per-identity lock, so the two side effects can't + # interleave into a resurrected file (clear() deletes under the same + # lock). The coordinated-refresh path already holds that lock, so persist + # inline there rather than deadlock/degrade re-acquiring it; the + # interactive sign-in path (no lock held) acquires it here. + if self._store_lock_held: + self._save_if_current(generation, refresh_token) + else: + # in_lock itself is best-effort: a custom store whose lock backend + # raises must not fail an otherwise-valid sign-in (the in-memory token + # is good regardless, and _save_if_current already swallows a save + # failure). The bundled FileTokenStore degrades internally and never + # raises here; this guards a custom TokenStore. + try: + self._token_store.in_lock( + self._store_key, + lambda: self._save_if_current(generation, refresh_token)) + except Exception as e: + self._warn_persistence('save', e) + + def _save_if_current( + self, generation: int, refresh_token: Optional[str]) -> None: + # Runs under the store's per-identity lock. Re-check the clear()-generation + # captured for this acquisition: a clear() (here or on another instance + # sharing the process-global cache) that bumped it AND deleted the file + # since our store_if_current must win, so skip the save rather than + # resurrect the file the user just cleared. clear() deletes the file under + # this same lock, so the re-check and the save are atomic against it. + if not self._cache.is_current(self.cache_key, generation): + return + try: + self._token_store.save(self._store_key, self._snapshot()) + self._last_persisted_refresh_token = refresh_token + except Exception as e: + # Best-effort: a save failure never fails an otherwise-valid sign-in; + # the token is valid in memory regardless. + self._warn_persistence('save', e) + + def _warn_persistence(self, operation: str, cause: Exception) -> None: + # Best-effort persistence: report to stderr and carry on with the + # in-memory token. The store never puts token bytes in its messages, so + # this cannot leak the secret. + detail = str(cause) + sys.stderr.write( + f'questdb client: OIDC token store {operation} failed; continuing ' + f'without persistence' + + (f' [{detail}]' if detail else '') + '\n') + def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: expires_in = _int_or_default( body.get('expires_in'), _DEFAULT_EXPIRES_IN) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index b3ba850e..ca022156 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -544,6 +544,18 @@ def resolve_config( 'explicitly (token_endpoint=..., device_authorization_endpoint=...), ' 'or connect to QuestDB over https so /settings is authenticated.') + # Vet the issuer authority up front — BEFORE it is used to build the IdP + # discovery URL below. A confusable issuer (urllib's parse-vs-connect + # divergence, e.g. r"https://attacker.evil\@idp.good") would otherwise drive + # a .well-known GET to the unvetted host before the issuer-pin block far + # below rejects it. That GET carries no credential and the pin block would + # still abort, so this is hardening, not a fix — but failing before any + # network call is cheaper, and it guarantees the pin block's + # _normalized_origin(issuer) compares against the host the transport + # actually connects to. + if issuer: + _reject_confusable_authority(issuer, label='issuer') + # Fall back to IdP discovery when QuestDB doesn't advertise the device # (and/or token) endpoint. This contacts the IdP, so it is held to # https/loopback (insecure=False) regardless of the QuestDB flag. @@ -614,9 +626,9 @@ def resolve_config( # outside the issuer path), so pinning them would reject a legitimate IdP. # The co-location check in OidcDeviceAuth.__init__ still applies on top. if issuer: - # The pin compares each /settings endpoint's origin against the issuer's; - # a confusable issuer authority would pin to the wrong host, so vet it too. - _reject_confusable_authority(issuer, label='issuer') + # The issuer authority was vetted up front (above, before discovery), so + # _normalized_origin here compares each /settings endpoint's origin + # against the same host the transport would connect to. issuer_origin = _normalized_origin(issuer) for label, url, from_settings, confirmed_by_idp in ( ('token endpoint', token_endpoint, token_from_settings, diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 69073eca..6df0eec5 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -390,19 +390,26 @@ def get_json( 'GET', url, headers=headers, timeout=timeout, ctx=ctx, insecure=insecure) if not resp.ok: - # Attach the HTTP status (mirroring post_form) so a future retry caller - # can tell a terminal 4xx from a transient 5xx/429 the same way the poll - # loop / silent refresh do. Today's callers (fetch_settings, IdP - # discovery) are one-shot and ignore it; this keeps the contract uniform. - raise OidcError( - f'HTTP {resp.status} from {url}: {resp.text()[:200]}', - status=resp.status) + # Map to the OidcError SUBCLASS that matches the cause, so a caller can + # `except OidcConfigError` / `except OidcNetworkError` around + # from_questdb() the same way _refresh / _poll_for_token classify + # post_form's errors: a 5xx/429 is a transient server / rate-limit issue, + # anything else (a 4xx/3xx from /settings or IdP discovery — a wrong URL, + # OIDC not advertised, an auth gate) is a configuration one. The HTTP + # status is attached either way (mirroring post_form) so a future retry + # caller can still classify terminal-vs-transient uniformly. + msg = f'HTTP {resp.status} from {url}: {resp.text()[:200]}' + if resp.status >= 500 or resp.status == 429: + raise OidcNetworkError(msg, status=resp.status) + raise OidcConfigError(msg, status=resp.status) try: return resp.json() except (ValueError, UnicodeDecodeError, RecursionError) as e: - # RecursionError (deeply-nested JSON) isn't a ValueError, so catch it - # explicitly to keep the typed contract. - raise OidcError( + # A non-JSON body where OIDC JSON was expected (an HTML login/error page + # from a proxy, or the wrong URL) is a configuration problem, not a + # transport one. RecursionError (deeply-nested JSON) isn't a ValueError, + # so catch it explicitly to keep the typed contract. + raise OidcConfigError( f'Invalid JSON from {url}: {e}', status=resp.status) from e diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index b9976f12..dfa2a420 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -310,8 +310,11 @@ def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: # that silently misses additions: control (Cc), format (Cf: bidi / zero-width / # soft hyphen / tag chars / the deprecated U+206x), unassigned (Cn), # private-use (Co), surrogates (Cs) and line/paragraph separators (Zl/Zp). -# Spaces (Zs) and combining marks (Mn, e.g. accents) are kept so a legitimate -# URL/identity still renders. +# The ordinary ASCII space (U+0020, itself category Zs) and combining marks +# (Mn, e.g. accents) are kept so a legitimate identity still renders; every +# OTHER space separator (NBSP U+00A0, ideographic space U+3000, ...) is folded +# to a plain space below, since an invisible-as-space char is a known phishing +# primitive (it can hide trailing text in a user_code / identity / error). _STRIP_CATEGORIES = frozenset({'Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Zl', 'Zp'}) # Invisible characters Unicode classifies as letters (category Lo), so the rule # above won't catch them, but they render as nothing and are used to hide/spoof @@ -338,10 +341,21 @@ def _strip_control(text: Optional[str]) -> str: return '' if not isinstance(text, str): text = str(text) - return ''.join( - ch for ch in text - if ch not in _STRIP_EXTRA - and unicodedata.category(ch) not in _STRIP_CATEGORIES) + out = [] + for ch in text: + if ch in _STRIP_EXTRA: + continue + category = unicodedata.category(ch) + if category in _STRIP_CATEGORIES: + continue + # Fold an exotic space separator (NBSP, ideographic space, ...) to a + # plain ASCII space: it renders invisible-as-space and can hide trailing + # text, but the ordinary U+0020 of a legitimate identity must survive. + if category == 'Zs' and ch != ' ': + out.append(' ') + else: + out.append(ch) + return ''.join(out) def format_prompt(resp: Dict[str, Any]) -> str: diff --git a/src/questdb/auth/_store.py b/src/questdb/auth/_store.py new file mode 100644 index 00000000..34c7a7c8 --- /dev/null +++ b/src/questdb/auth/_store.py @@ -0,0 +1,724 @@ +################################################################################ +## ___ _ ____ ____ +## / _ \ _ _ ___ ___| |_| _ \| __ ) +## | | | | | | |/ _ \/ __| __| | | | _ \ +## | |_| | |_| | __/\__ \ |_| |_| | |_) | +## \__\_\\__,_|\___||___/\__|____/|____/ +## +## Copyright (c) 2014-2019 Appsicle +## Copyright (c) 2019-2024 QuestDB +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## +################################################################################ + +""" +Opt-in token persistence for :mod:`questdb.auth`. + +By default :class:`~questdb.auth.OidcDeviceAuth` keeps its tokens in memory only, +so a restarted process must run the device flow again. A :class:`TokenStore` +persists the token state so the restarted process resumes from a saved refresh +token — one silent token-endpoint round-trip — instead of re-prompting. + +:class:`FileTokenStore` is the default implementation: one plaintext JSON file +per identity, protected at rest by file permissions (``0600`` file, ``0700`` +directory) rather than encryption — the same posture ``gcloud``, ``aws`` and +``gh`` take. Supply your own :class:`TokenStore` (backed by an OS keychain, a +KMS, or a vault) to encrypt the refresh token at rest. + +The on-disk format (directory, file name, JSON schema, atomic-write and +lock-file protocols) is a deliberately **language-neutral contract** so the Java +QuestDB client and this one can share the same file. The Java client is the +reference implementation; this module mirrors it. +""" + +from __future__ import annotations + +import abc +import contextlib +import hashlib +import json +import math +import os +import socket +import sys +import tempfile +import threading +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +from ._errors import OidcConfigError, OidcError +from ._http import safe_urlparse + +# Frozen cross-language contract. The schema version tags both the on-disk ``v`` +# field and the canonical-string hash prefix (which also doubles as a domain +# tag), so a future format bump produces a different hash — hence a different +# file — rather than silently colliding with v1 entries. Derive the prefix from +# the version so the two can never drift apart. +_SCHEMA_VERSION = 1 +_CANONICAL_PREFIX = f'questdb-oidc-token-v{_SCHEMA_VERSION}' + +# Environment variable overriding the default token-store directory (the +# language-neutral analogue of the Java client's +# ``questdb.client.oidc.token.store.dir`` system property). +TOKEN_STORE_DIR_ENV = 'QUESTDB_CLIENT_OIDC_TOKEN_STORE_DIR' + +# Reject a token file larger than this; a real entry is a few KB even with a +# group-laden id token, so anything past this is corrupt or hostile and is not +# read into memory. +_MAX_FILE_BYTES = 1 << 20 + +# Wait this long (seconds) for the per-identity lock file before giving up and +# running without it (atomic replacement still guards integrity). Kept short +# because token() can take this lock on the latency-sensitive flush path: a real +# refresh round-trip is sub-second, so a peer not done within this budget is +# treated as too slow and we degrade to a lock-free refresh rather than stall. +_DEFAULT_LOCK_ACQUIRE_BUDGET = 3.0 +# Treat a lock older than this (seconds) as abandoned by a crashed holder and +# steal it. Must stay comfortably above the longest a live holder can hold it +# (one refresh under the lock); OidcDeviceAuth caps its HTTP timeout at 120s and +# the worst-case hold is a few times that, so this 10-minute window stays safely +# above it. +_DEFAULT_LOCK_STALE = 600.0 +_LOCK_POLL_SLICE = 0.05 +# Floor on a configured ``lock_stale``. A lock may legitimately be held for the +# whole of one refresh under it: OidcDeviceAuth caps its HTTP timeout at 120s, +# applied per network leg, so the worst-case live hold is ~2x that. Reject a +# ``lock_stale`` at or below this floor — a shorter window would let ``_is_stale`` +# declare a LIVE holder's lock abandoned and a peer steal it mid-refresh, which +# the atomic steal cannot rescue (it guards two acquirers racing to break one +# *stale* lock, not a window so short a *live* lock reads as stale). The default +# (_DEFAULT_LOCK_STALE) sits comfortably above this. +_MIN_LOCK_STALE = 240.0 + +# Set once if the platform cannot enforce owner-only POSIX permissions on the +# token files (e.g. Windows), so the at-rest protection falls back to the +# directory's inherited ACL; warns the user once. +_warned_no_posix_perms = False +_warn_lock = threading.Lock() + + +def _warn_no_posix_perms_once() -> None: + # Best-effort, once per process: the token store could not enforce 0600/0700, + # so the persisted refresh token is protected only by the directory's + # inherited ACL. ASCII-only, and never includes a path or token byte. + global _warned_no_posix_perms + with _warn_lock: + if _warned_no_posix_perms: + return + _warned_no_posix_perms = True + sys.stderr.write( + 'questdb client: the OIDC token store could not enforce owner-only ' + '(0600/0700) permissions on this filesystem; the persisted refresh ' + "token is protected only by the directory's default ACL. Back the " + 'store with an OS keychain for at-rest encryption.\n') + + +def _nonempty_str(value: Any) -> Optional[str]: + """A persisted field as a non-empty ``str``, else ``None``. + + The file is attacker-writable, so a non-string (a JSON number/list from a + hand-edited or hostile file) reads as absent rather than landing in a token + field as a raw object, and an empty string — never a usable token — is + likewise dropped. + """ + return value if isinstance(value, str) and value else None + + +def _millis_to_seconds(value: Any) -> float: + """An on-disk ``*_millis`` field as epoch/duration **seconds**, else ``0.0``. + + A non-numeric or non-finite value (a hostile file) reads as ``0.0``, which + marks the entry expired so it falls through to a refresh rather than being + served. ``bool`` is an ``int`` subclass but never a meaningful timestamp. + """ + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0.0 + try: + result = float(value) / 1000.0 + except (OverflowError, ValueError): + return 0.0 + # NaN / ±Inf are real floats (json.loads accepts bare NaN / Infinity), so the + # divide above does not raise; map them to 0.0 (expired) per the contract + # rather than let a non-finite timestamp reach the expiry math. + if not math.isfinite(result): + return 0.0 + return result + + +def _seconds_to_millis(value: Any) -> int: + """Epoch/duration **seconds** as an on-disk ``*_millis`` int; the inverse of + :func:`_millis_to_seconds`, mapping a non-finite value to ``0`` (expired). + + ``PersistedToken`` is public, so a direct caller of :meth:`FileTokenStore.save` + could pass ``inf``/``nan`` — ``int(round(inf * 1000))`` raises ``OverflowError`` + and ``round(nan)`` raises ``ValueError``, escaping the store's ``OidcError`` + contract. Mapping them to ``0`` keeps ``save`` typed and makes a non-finite + expiry read back as expired rather than valid-forever, exactly as the load + side already does. ``bool`` is an ``int`` subclass but never a timestamp. + """ + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0 + try: + seconds = float(value) + except (OverflowError, ValueError): + return 0 # e.g. an int too large to convert to float + if not math.isfinite(seconds): + return 0 + return int(round(seconds * 1000)) + + +def _canonical_endpoint(url: str) -> str: + """Canonicalise an endpoint URL for the cross-language store-key hash. + + ``scheme://host:port/path`` with the scheme and host lower-cased, the port + always explicit (the device-flow default 443/80 when absent), and the parsed + path (defaulting to ``/``). A stable rendering that hashes to the same + :class:`TokenStoreKey` across processes and language clients sharing this + identity. Mirrors the Java client's ``canonicalEndpoint``. + """ + parts, explicit_port = safe_urlparse(url) + scheme = (parts.scheme or '').lower() + host = (parts.hostname or '').lower() + # urllib strips the brackets off an IPv6 literal, which would make the + # host:port boundary ambiguous ("::1:443") and, worse, diverge from the + # bracketed authority form the Java client renders (URI.getHost() keeps the + # brackets) — the two clients would then hash an IPv6 endpoint differently and + # never share the file. Re-add them. + if ':' in host: + host = f'[{host}]' + default_port = {'https': 443, 'http': 80}.get(scheme) + port = explicit_port if explicit_port is not None else default_port + path = parts.path or '/' + return f'{scheme}://{host}:{port}{path}' + + +@dataclass(frozen=True) +class PersistedToken: + """An immutable snapshot of the token state an + :class:`~questdb.auth.OidcDeviceAuth` holds, passed to and from a + :class:`TokenStore` so the device flow need not re-run after a process + restart. + + ``expires_at`` is an absolute epoch-seconds value (not a monotonic reading), + so it stays meaningful across a restart. ``token_ttl`` is the (clamped) + lifetime that expiry was derived from. The token strings are kept out of + ``repr`` so a credential can't leak into a log line or traceback. + """ + + access_token: Optional[str] = field(default=None, repr=False) + id_token: Optional[str] = field(default=None, repr=False) + refresh_token: Optional[str] = field(default=None, repr=False) + expires_at: float = 0.0 # absolute epoch seconds; survives restart + token_ttl: float = 0.0 # seconds; the lifetime expires_at was derived from + + +@dataclass(frozen=True) +class TokenStoreKey: + """The non-secret identity a persisted token belongs to. + + The client id, the canonicalised token and device-authorization endpoints, + the scope, the optional audience, and whether the server expects groups + encoded in the token. A :class:`TokenStore` keys its entries by this so a + token minted for one server / identity provider / scope / audience is never + served to a process configured for another. + + :meth:`hash` is a stable lowercase-hex SHA-256 over a canonical, + NUL-separated rendering of the fields — a file name (or opaque key) that is + identical across client implementations (the Java client mirrors this), so + several processes (and languages) sharing one identity address the same + persisted entry. The fields are exposed (they are not secret) so a store can + record and re-check them on load as a defence against a hash collision or a + copied file. + """ + + client_id: str + token_endpoint: str + device_authorization_endpoint: str + scope: str + audience: Optional[str] + groups_in_token: bool + + def hash(self) -> str: + """A stable lowercase-hex SHA-256 of the canonical identity string.""" + # NUL-separate the fields so no field value can be confused with a + # separator (an OAuth client id, url, scope or audience never contains a + # NUL). The prefix tags the domain and schema version. + canonical = '\x00'.join(( + _CANONICAL_PREFIX, + self.client_id or '', + self.token_endpoint or '', + self.device_authorization_endpoint or '', + self.scope or '', + self.audience or '', + '1' if self.groups_in_token else '0', + )) + return hashlib.sha256(canonical.encode('utf-8')).hexdigest() + + +class TokenStore(abc.ABC): + """Persists the token state of an :class:`~questdb.auth.OidcDeviceAuth`. + + A restarted process resumes from a saved refresh token instead of running + the interactive device flow again. Persistence is opt-in: an + ``OidcDeviceAuth`` with no store keeps its tokens in memory only (the + previous behaviour). + + The default implementation is :class:`FileTokenStore`. Supply your own to + back persistence with an OS keychain, a secrets manager, or a vault — for + example to encrypt the refresh token at rest, which the file store does not + do. + + Calls are made while ``OidcDeviceAuth`` holds its own instance lock, so an + implementation need not be thread-safe against concurrent calls from one + ``OidcDeviceAuth`` instance; it does, however, share its backing storage + with other processes (and other language clients), so it must keep a + concurrent reader from observing a half-written entry. A store reports a + failure by **raising**; ``OidcDeviceAuth`` treats persistence as best-effort + and a raised failure as non-fatal — it warns to ``stderr`` and continues + with the in-memory token, which is valid regardless of whether it could be + saved. + """ + + @abc.abstractmethod + def load(self, key: TokenStoreKey) -> Optional[PersistedToken]: + """Load the persisted token for this identity, or ``None`` if there is + none usable (no entry, an entry that does not match ``key``, or one that + cannot be read as a valid token). A ``None`` return makes + ``OidcDeviceAuth`` fall back to a refresh or an interactive sign-in, so + an unreadable or stale entry is recoverable rather than fatal. + + **Security — the implementation MUST re-verify identity.** + ``OidcDeviceAuth`` does not re-check the returned token against ``key``; + it trusts ``load`` to only ever return an entry stored under the *same* + identity. A store addressed solely by :meth:`TokenStoreKey.hash` must + therefore also record the identity fields in the persisted payload and + re-compare them on load (as :class:`FileTokenStore` does), so a hash + collision, a copied secret, or a swapped backing entry cannot serve one + identity's token to a session configured for another — returning a + wrong-identity token here routes that credential onto the wire.""" + + @abc.abstractmethod + def save(self, key: TokenStoreKey, token: PersistedToken) -> None: + """Persist (atomically replace) the token for this identity.""" + + @abc.abstractmethod + def clear(self, key: TokenStoreKey) -> None: + """Remove any persisted entry for this identity. A no-op when nothing is + stored. Called from :meth:`~questdb.auth.OidcDeviceAuth.clear`.""" + + def in_lock(self, key: TokenStoreKey, action: Callable[[], Any]) -> Any: + """Run ``action`` while holding a cross-process lock scoped to ``key``, + so a refresh by another process sharing this identity is observed rather + than raced, and return its result. + + The default runs ``action`` with no locking, which is correct for a + single process or a non-rotating refresh token; :class:`FileTokenStore` + overrides it with a lock-file protocol. **Most stores should NOT override + this** — the no-op default is correct unless the backend is shared across + processes *and* the IdP rotates the refresh token on each refresh. Note + ``action`` re-enters this same store (it calls :meth:`load` and + :meth:`save` on it). An implementation that cannot acquire the lock should + run ``action`` anyway (degrade) rather than fail a sign-in. + """ + return action() + + +class FileTokenStore(TokenStore): + """The default :class:`TokenStore`: one plaintext JSON file per identity. + + The refresh token is protected at rest by file permissions (``0600`` file, + ``0700`` directory) rather than encryption — matching ``gcloud``, ``aws`` and + ``gh``; for encryption at rest supply a :class:`TokenStore` backed by an OS + keychain or a secrets manager instead. + + The default location is ``${user.home}/.questdb/oidc-tokens/``, overridable + with the ``QUESTDB_CLIENT_OIDC_TOKEN_STORE_DIR`` environment variable. The + file name is ``.json``, so several identities coexist + and the name leaks neither the endpoint nor the client id. The on-disk format + is a language-neutral contract so other QuestDB clients can share the file. + + **Integrity (always).** :meth:`save` writes a sibling temp file then + atomically renames it over the target, so a crash or an overlapping reader — + in any process or language — sees the whole old or whole new file, never a + torn credential. + + **Rotating refresh tokens.** :meth:`in_lock` serialises the + read-refresh-write of a token refresh across processes with an + ``O_CREAT|O_EXCL`` lock file (``.lock``) — not an OS advisory lock, + which a Java ``FileLock`` and a Python ``flock`` cannot reliably share. It + steals a stale lock left by a crashed holder, and degrades to running without + the lock (integrity is still protected) rather than stall a sign-in if it + cannot acquire one. + + The store never writes a token value into a log or an exception message; only + file paths and OS error kinds may surface. + """ + + def __init__( + self, + directory: Any, + *, + lock_acquire_budget: float = _DEFAULT_LOCK_ACQUIRE_BUDGET, + lock_stale: float = _DEFAULT_LOCK_STALE): + """ + :param directory: the directory to hold the token files; created on first + write with owner-only permissions. + :param lock_acquire_budget: how long (seconds) :meth:`in_lock` waits to + acquire a peer's lock before degrading to a lock-free refresh rather + than stalling a sign-in. + :param lock_stale: a lock older than this (seconds) is treated as + abandoned by a crashed holder and stolen. It MUST exceed the longest + a live holder can hold the lock (one refresh under the lock), so a + value at or below the worst-case hold (~240s, twice + ``OidcDeviceAuth``'s 120s HTTP-timeout cap) is rejected to keep a + peer from stealing a live holder's lock mid-refresh; the default + (600s) stays safely above that. + """ + if not directory: + raise OidcConfigError('the token store directory is required') + if not (lock_acquire_budget > 0): + raise OidcConfigError( + 'the token store lock_acquire_budget must be positive') + # A staleness window at or below the worst-case live hold makes a + # freshly acquired lock look abandoned, so acquirers would steal each + # other's LIVE locks. Require it to exceed _MIN_LOCK_STALE (and so, + # transitively, to be positive). + if not (lock_stale > _MIN_LOCK_STALE): + raise OidcConfigError( + 'the token store lock_stale must exceed ' + f'{_MIN_LOCK_STALE:g} seconds (the worst-case time a live ' + 'holder can hold the lock during a refresh); a shorter window ' + "would let a peer steal a live holder's lock mid-refresh.") + self._directory = os.fspath(directory) + self._lock_acquire_budget = lock_acquire_budget + self._lock_stale = lock_stale + + @classmethod + def at(cls, directory: Any) -> 'FileTokenStore': + """A store rooted at the given directory.""" + return cls(directory) + + @classmethod + def at_default_location(cls) -> 'FileTokenStore': + """A store at ``$QUESTDB_CLIENT_OIDC_TOKEN_STORE_DIR`` if that environment + variable is set, otherwise at ``${user.home}/.questdb/oidc-tokens/``.""" + override = os.environ.get(TOKEN_STORE_DIR_ENV) + if override: + return cls(override) + home = os.path.expanduser('~') + # expanduser returns the literal '~' when the home directory can't be + # resolved (no HOME/USERPROFILE and no passwd entry, e.g. a distroless + # container). Joining onto it would yield a RELATIVE path and silently + # create a surprise directory named '~' under the cwd, so fail clearly + # and point at the override instead. + if not os.path.isabs(home): + raise OidcConfigError( + 'could not resolve the home directory for the default OIDC ' + f'token-store location; set the {TOKEN_STORE_DIR_ENV} ' + 'environment variable to an absolute path, or construct ' + 'FileTokenStore(directory) explicitly.') + return cls(os.path.join(home, '.questdb', 'oidc-tokens')) + + def load(self, key: TokenStoreKey) -> Optional[PersistedToken]: + path = self._token_file(key) + try: + size = os.stat(path).st_size + except FileNotFoundError: + return None + except OSError as e: + raise OidcError( + f'could not read the OIDC token store file: {e}') from e + # An empty or implausibly large file is not a usable entry; ignore it + # rather than read it into memory. + if size <= 0 or size > _MAX_FILE_BYTES: + return None + try: + with open(path, 'rb') as f: + data = f.read(_MAX_FILE_BYTES + 1) + except FileNotFoundError: + return None + except OSError as e: + raise OidcError( + f'could not read the OIDC token store file: {e}') from e + if len(data) > _MAX_FILE_BYTES: + return None + return self._parse_and_verify(key, data) + + def save(self, key: TokenStoreKey, token: PersistedToken) -> None: + content = self._serialize(key, token) + try: + self._ensure_directory() + target = self._token_file(key) + # mkstemp creates the temp file with 0600 (O_CREAT|O_EXCL, mode 0600) + # on POSIX, so there is no world-readable window before the rename. + fd, tmp = tempfile.mkstemp( + prefix=key.hash(), suffix='.tmp', dir=self._directory) + moved = False + try: + # Force the payload to disk before the rename, so a crash between + # the write and the atomic rename cannot leave the target + # pointing at unflushed (zero/partial) bytes. + with os.fdopen(fd, 'wb') as f: + f.write(content) + f.flush() + os.fsync(f.fileno()) + # Atomic on POSIX (rename(2)) and on Windows. + os.replace(tmp, target) + moved = True + # Persist the rename itself: without fsync-ing the directory a + # host crash right after the rename can lose the new entry on + # some filesystems. Best-effort (a lost entry only costs one + # silent re-prompt), and a no-op where a directory fd can't be + # fsynced (Windows). + self._fsync_directory() + finally: + if not moved: + with contextlib.suppress(OSError): + os.remove(tmp) + except OSError as e: + raise OidcError( + f'could not persist the OIDC token to the token store: ' + f'{e}') from e + + def clear(self, key: TokenStoreKey) -> None: + try: + os.remove(self._token_file(key)) + except FileNotFoundError: + pass + except OSError as e: + raise OidcError( + f'could not remove the OIDC token store file: {e}') from e + + def in_lock(self, key: TokenStoreKey, action: Callable[[], Any]) -> Any: + lock = None + held = False + try: + self._ensure_directory() + lock = self._lock_file(key) + held = self._acquire_lock(lock) + except OSError: + # Could not prepare the lock directory or file; run without the lock. + # Atomic replacement still keeps every reader consistent — only a + # rotating-refresh-token race across processes is left unguarded for + # this one refresh. + held = False + try: + return action() + finally: + if held: + with contextlib.suppress(OSError): + # Best-effort release; a leftover lock goes stale and the + # next acquirer steals it. + os.remove(lock) + + # -- internals ---------------------------------------------------------- + + def _token_file(self, key: TokenStoreKey) -> str: + return os.path.join(self._directory, key.hash() + '.json') + + def _lock_file(self, key: TokenStoreKey) -> str: + return os.path.join(self._directory, key.hash() + '.lock') + + def _ensure_directory(self) -> None: + if os.path.isdir(self._directory): + # Re-assert owner-only permissions on a pre-existing directory: one + # left world/group-accessible by another tool, a permissive umask, or + # a hostile local pre-create would otherwise expose the token files. + self._restrict_to_owner() + return + os.makedirs(self._directory, mode=0o700, exist_ok=True) + self._restrict_to_owner() + + def _restrict_to_owner(self) -> None: + # Best-effort: the at-rest protection of the plaintext token files is + # exactly these owner-only directory permissions. On a non-POSIX + # filesystem (Windows) POSIX modes do not apply, so fall back to the + # directory's inherited ACL and warn once. + if os.name != 'posix': + _warn_no_posix_perms_once() + return + try: + os.chmod(self._directory, 0o700) + except OSError: + # The directory is not ours to chmod: keep the existing permissions. + pass + + def _fsync_directory(self) -> None: + # Flush the directory entry so an atomic rename into it survives a host + # crash. POSIX only — a directory fd can't be opened/fsynced on Windows; + # best-effort everywhere (a lost entry only costs a re-prompt). + if os.name != 'posix': + return + try: + dir_fd = os.open(self._directory, os.O_RDONLY) + try: + os.fsync(dir_fd) + finally: + os.close(dir_fd) + except OSError: + pass + + def _serialize(self, key: TokenStoreKey, token: PersistedToken) -> bytes: + # A null value (an absent audience, or a token kind the grant did not + # return) is omitted entirely rather than written as JSON null — the only + # encoding under which a present value (a token equal to "null" included) + # round-trips verbatim and an absent one reads back as null. Field order + # follows the frozen schema for readability; key order is not semantic. + obj = { + 'v': _SCHEMA_VERSION, + 'client_id': key.client_id, + 'token_endpoint': key.token_endpoint, + 'device_authorization_endpoint': key.device_authorization_endpoint, + 'scope': key.scope, + } + if key.audience is not None: + obj['audience'] = key.audience + obj['groups_in_token'] = key.groups_in_token + if token.access_token is not None: + obj['access_token'] = token.access_token + if token.id_token is not None: + obj['id_token'] = token.id_token + if token.refresh_token is not None: + obj['refresh_token'] = token.refresh_token + obj['expires_at_millis'] = _seconds_to_millis(token.expires_at) + obj['token_ttl_millis'] = _seconds_to_millis(token.token_ttl) + return json.dumps(obj, separators=(',', ':')).encode('utf-8') + + def _parse_and_verify( + self, key: TokenStoreKey, data: bytes) -> Optional[PersistedToken]: + try: + obj = json.loads(data) + except (ValueError, UnicodeDecodeError): + # Corrupt or truncated file: treat as no usable entry, fall back to + # refresh / interactive. + return None + if not isinstance(obj, dict): + return None + # Schema and fingerprint must match the live identity; a mismatch is a + # hash collision or a file copied from a different identity, so ignore it + # rather than serve the wrong identity's token. + if obj.get('v') != _SCHEMA_VERSION: + return None + if (obj.get('client_id') != key.client_id + or obj.get('token_endpoint') != key.token_endpoint + or obj.get('device_authorization_endpoint') + != key.device_authorization_endpoint + or obj.get('scope') != key.scope + or not _audience_matches(key.audience, obj.get('audience')) + or bool(obj.get('groups_in_token')) != key.groups_in_token): + return None + return PersistedToken( + access_token=_nonempty_str(obj.get('access_token')), + id_token=_nonempty_str(obj.get('id_token')), + refresh_token=_nonempty_str(obj.get('refresh_token')), + expires_at=_millis_to_seconds(obj.get('expires_at_millis')), + token_ttl=_millis_to_seconds(obj.get('token_ttl_millis'))) + + def _acquire_lock(self, lock: str) -> bool: + deadline = time.monotonic() + self._lock_acquire_budget + while True: + try: + self._create_lock_file(lock) + return True + except FileExistsError: + if self._is_stale(lock): + self._steal_stale_lock(lock) + # Fall through to the bounded wait below rather than retry + # immediately: a steal contest between several acquirers (or + # a misconfigured tiny lock_stale) must not hot-spin. + if time.monotonic() >= deadline: + # Give up and run without the lock rather than stall. + return False + time.sleep(_LOCK_POLL_SLICE) + except OSError: + return False # unexpected IO; degrade to no lock + + def _create_lock_file(self, lock: str) -> None: + # The O_EXCL create IS the acquisition. Write the holder metadata through + # this same fd — never via a second, non-exclusive open(): a concurrent + # steal could replace the file between the create and a second open, so a + # plain open('w') would truncate a peer's fresh lock (or resurrect one + # just removed). Holder bytes are debugging-only; staleness is judged by + # mtime, never by parsing them, so a write failure must not fail an + # acquisition we already won. + fd = os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + try: + with contextlib.suppress(OSError): + os.write(fd, self._holder_bytes()) + finally: + os.close(fd) + + def _steal_stale_lock(self, lock: str) -> None: + # Break a stale lock ATOMICALLY: rename it aside to a private path. Only + # one racer can rename a given file away — the losers get an OSError (it + # is already gone) and simply retry the O_EXCL create, which itself + # admits a single winner. This replaces an unconditional os.remove(lock), + # under which several acquirers could each delete a different generation + # of the lock and ALL believe they won (two live holders at once). Then + # re-judge staleness on the moved-aside file: if a peer recreated a fresh + # lock between our staleness check and the rename, we moved that live lock + # by mistake — restore it rather than strand the peer; only a genuinely + # stale file is removed. This makes a single break the common outcome and + # never blindly deletes a live lock; under pathological N-way concurrent + # stealing a rare transient double-acquire can still slip through (no + # file-only protocol prevents it without OS support), which stays + # integrity-safe — the atomic write holds — and costs at most one extra + # re-prompt, the same best-effort degradation as running lock-free. + private = f'{lock}.stale.{os.getpid()}.{threading.get_ident()}' + try: + os.replace(lock, private) + except OSError: + return # lost the steal race; the lock is already gone — retry create + if self._is_stale(private): + with contextlib.suppress(OSError): + os.remove(private) + else: + # Moved a still-live lock by mistake; put it back. If the slot was + # retaken meanwhile, our private copy is redundant — drop it. + try: + os.replace(private, lock) + except OSError: + with contextlib.suppress(OSError): + os.remove(private) + + def _holder_bytes(self) -> bytes: + # pid@host plus a timestamp, to help debug a stuck lock; never parsed. + try: + return (f'{os.getpid()}@{socket.gethostname()} ' + f'{time.time()}').encode('utf-8') + except Exception: + return b'' # metadata only — never fail acquisition over it + + def _is_stale(self, lock: str) -> bool: + try: + mtime = os.stat(lock).st_mtime + except OSError: + # Can't determine the age, so don't steal. (A pre-planted dangling + # symlink at the lock path lands here forever, so that identity + # degrades to lock-free coordination — integrity-safe, since the + # atomic write still holds, and an attacker who can plant it already + # has write access to the token directory.) + return False + return (time.time() - mtime) > self._lock_stale + + +def _audience_matches(key_audience: Optional[str], file_audience: Any) -> bool: + # The file omits a null audience entirely, so an absent (or hand-edited JSON + # null) field reads as None and matches a None key audience; a present + # audience must be an exact string match. A non-string file value never + # matches, so a hostile entry is rejected. + if key_audience is None: + return file_audience is None + return isinstance(file_audience, str) and file_audience == key_audience diff --git a/test/test.py b/test/test.py index 9d1eda6f..8aa74cd3 100755 --- a/test/test.py +++ b/test/test.py @@ -46,6 +46,8 @@ TestConfigHelpers, TestEndpointValidation, TestCacheKey, + TestFileTokenStore, + TestPersistence, TestTransportSecurity, TestRendererSecurity, ) diff --git a/test/test_auth.py b/test/test_auth.py index 71902685..499ee007 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -38,10 +38,14 @@ import base64 import contextlib import importlib.util +import io import json import os +import shutil import sys +import tempfile import threading +import time import types import unittest import http.server @@ -53,6 +57,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) from questdb.auth import ( # noqa: E402 + FileTokenStore, OidcDeviceAuth, OidcError, OidcConfigError, @@ -60,7 +65,10 @@ OidcTimeoutError, OidcInteractionRequired, OidcNetworkError, + PersistedToken, TokenSet, + TokenStore, + TokenStoreKey, sqlalchemy_engine, psycopg_connect, ) @@ -68,6 +76,9 @@ MemoryCache, _MEMORY_GENERATION, _MEMORY_INFLIGHT, _MEMORY_STORE) from questdb.auth._render import Renderer # noqa: E402 from questdb.auth._adapters import _require_host # noqa: E402 +from questdb.auth._store import ( # noqa: E402 + TOKEN_STORE_DIR_ENV, _CANONICAL_PREFIX, _MIN_LOCK_STALE, _SCHEMA_VERSION, + _canonical_endpoint, _millis_to_seconds, _seconds_to_millis) _HAS_PG_DRIVER = ( importlib.util.find_spec('psycopg') is not None @@ -1236,6 +1247,39 @@ def test_non_object_jwt_payload_does_not_crash(self): self.assertEqual(auth._tokens.id_token, array_token) self.assertIsNone(auth._tokens.sub) + def test_corrupt_jwt_middle_segment_does_not_crash(self): + # A 3-segment token whose middle segment is invalid base64, or decodes to + # non-UTF-8 / non-JSON bytes (a malformed or hostile id_token), must + # degrade to no-claims rather than crash the best-effort identity decode + # (the binascii.Error / UnicodeDecodeError / ValueError arms). See + # _decode_jwt_claims. + from questdb.auth._device import _decode_jwt_claims + + def seg(raw): + return base64.urlsafe_b64encode(raw).rstrip(b'=').decode() + + for bad in ( + 'aaa.A.sig', # bad base64 length + f'aaa.{seg(bytes([0x80, 0x81, 0x82]))}.sig', # non-UTF-8 bytes + f'aaa.{seg(b"not json")}.sig'): # valid text, not JSON + self.assertEqual(_decode_jwt_claims(bad), {}) + + def test_select_raises_config_error_when_required_kind_absent(self): + # _select is the final gate before a token is handed to QuestDB. Every + # caller already checks _has_required_token, so its own OidcConfigError + # branch is defense-in-depth -- but assert it directly: in groups mode a + # TokenSet without an id_token, and otherwise one without an + # access_token, must raise a clear config error (not return None/empty). + groups_auth = self.make_auth(groups_in_token=True) + with self.assertRaises(OidcConfigError) as cm: + groups_auth._select( + TokenSet(access_token=ACCESS_TOKEN, id_token=None)) + self.assertIn('id_token', str(cm.exception)) + access_auth = self.make_auth(groups_in_token=False) + with self.assertRaises(OidcConfigError) as cm: + access_auth._select(TokenSet(access_token=None)) + self.assertIn('access_token', str(cm.exception)) + def test_non_string_token_fields_do_not_crash(self): # A buggy/hostile IdP returning a non-string access_token / id_token (a # JSON number/bool/object) must not crash token() with a raw @@ -2379,6 +2423,26 @@ def test_store_if_current_drops_write_after_concurrent_clear(self): cache.store_if_current(key, TokenSet(access_token='T2'), gen2)) self.assertIsNotNone(cache.load(key)) + def test_evict_does_not_bump_generation_unlike_clear(self): + # evict() drops the cached token WITHOUT bumping the clear()-generation, + # so the SAME acquisition that evicted its own unusable token can still + # land its replacement; clear() bumps it, so a store captured before the + # clear is dropped. This is the distinction the refresh-then-resign path + # relies on (_acquire evicts the doomed token, then _store the fresh one). + cache = MemoryCache() + key = 'k' + gen = cache.generation(key) # acquisition begins + cache.evict(key) # drop our own bad token + self.assertTrue( # replacement still lands + cache.store_if_current(key, TokenSet(access_token='new'), gen)) + self.assertEqual(cache.load(key).access_token, 'new') + # Contrast: a clear() with the same in-flight generation DOES drop a + # store captured before it. + cache.clear(key) + self.assertFalse( + cache.store_if_current(key, TokenSet(access_token='x'), gen)) + cache.release(key) + def test_generation_pruned_when_no_acquisition_in_flight(self): # M8: the per-key clear()-generation must not accumulate forever. After a # clear() and a completed re-acquisition (no acquisition left in flight), @@ -3017,6 +3081,18 @@ def test_settings_url_drops_query_and_fragment(self): self.assertEqual(_settings_url('https://h:9000/base/#frag'), 'https://h:9000/base/settings') + def test_settings_url_requires_explicit_scheme(self): + # A scheme-less QuestDB URL ("questdb.example.com:9000") mis-parses -- + # urllib reads the host as the scheme -- so _settings_url rejects it with + # a clear typed error up front, rather than letting it surface much later + # as a confusing "insecure URL (scheme 'questdb.example.com')" from + # _require_secure. A non-http(s) scheme is rejected for the same reason. + from questdb.auth._discovery import _settings_url + for bad in ('questdb.example.com:9000', 'h:9000', + 'ftp://h:9000', '//h:9000'): + with self.assertRaises(OidcConfigError): + _settings_url(bad) + def test_settings_config_ignores_user_writable_preferences(self): # QuestDB /settings nests server-authoritative values under "config" # alongside a user-writable "preferences" sibling (the web console @@ -3867,22 +3943,33 @@ def test_post_form_non_dict_json_raises_oidc_error(self): self.assertEqual(cm.exception.status, 400) def test_get_json_non_2xx_raises_oidc_error(self): - # A non-2xx /settings or discovery response must surface as OidcError. - # See M4. + # A non-2xx /settings or discovery response must surface as a typed + # OidcError SUBCLASS matching the cause (mirroring how _refresh / + # _poll_for_token classify post_form): a 5xx/429 is transient -> + # OidcNetworkError, a 4xx/3xx is a config problem -> OidcConfigError. The + # HTTP status is attached either way so a retry caller can classify + # terminal-vs-transient the same way. See M4. from questdb.auth import _http with _raw_response_server(500, 'text/plain', b'boom') as b: - with self.assertRaises(OidcError) as cm: + with self.assertRaises(OidcNetworkError) as cm: _http.get_json(b + '/settings', timeout=5) - # The HTTP status is attached (mirroring post_form) so a future retry - # caller can classify terminal-vs-transient the same way. self.assertEqual(cm.exception.status, 500) + with _raw_response_server(429, 'text/plain', b'slow') as b: + with self.assertRaises(OidcNetworkError) as cm: + _http.get_json(b + '/settings', timeout=5) + self.assertEqual(cm.exception.status, 429) + with _raw_response_server(404, 'text/plain', b'nope') as b: + with self.assertRaises(OidcConfigError) as cm: + _http.get_json(b + '/settings', timeout=5) + self.assertEqual(cm.exception.status, 404) def test_get_json_non_json_2xx_raises_oidc_error(self): - # A 2xx /settings or discovery body that isn't JSON must surface as - # OidcError, not a raw JSONDecodeError. See M4. + # A 2xx /settings or discovery body that isn't JSON (an HTML login/error + # page, the wrong URL) is a configuration problem -> OidcConfigError, not + # a raw JSONDecodeError. See M4. from questdb.auth import _http with _raw_response_server(200, 'text/html', b'x') as b: - with self.assertRaises(OidcError) as cm: + with self.assertRaises(OidcConfigError) as cm: _http.get_json( b + '/.well-known/openid-configuration', timeout=5) self.assertEqual(cm.exception.status, 200) # status attached @@ -4334,6 +4421,42 @@ def test_strip_control_removes_bidi_and_zero_width(self): self.assertNotIn(chr(0x202e), text) self.assertIn('idp.example.com', text) + def test_strip_control_folds_exotic_whitespace_to_ascii_space(self): + # An invisible-as-space separator (NBSP, ideographic space, ...) is a + # phishing primitive: it can pad a user_code / identity / error to hide + # trailing text that looks like a normal gap. _strip_control folds every + # non-ASCII Zs to a plain U+0020, while the ordinary ASCII space of a + # legitimate identity survives untouched. + from questdb.auth._render import _strip_control + for cp in (0x00a0, 0x2000, 0x2007, 0x202f, 0x205f, 0x3000): + self.assertEqual(_strip_control('A' + chr(cp) + 'B'), 'A B', + f'U+{cp:04X} not folded to a plain space') + # A hidden-text payload no longer reads as a clean four-char code. + self.assertEqual( + _strip_control('WXYZ' + chr(0x3000) * 4 + 'DELETE-ME'), + 'WXYZ DELETE-ME') + # Ordinary ASCII spaces (and accented names) are preserved. + self.assertEqual(_strip_control('Alice Smith'), 'Alice Smith') + self.assertEqual(_strip_control('café'), 'café') + + def test_format_prompt_renders_complete_uri_line(self): + # The plain-text prompt shows verification_uri_complete on its own + # "(or open directly: ...)" line when present, and omits the line when + # absent. The complete URL is IDNA-normalized like the main link. + from questdb.auth._render import format_prompt + with_complete = format_prompt({ + 'user_code': 'WXYZ', + 'verification_uri': 'https://idp.example.com/device', + 'verification_uri_complete': + 'https://idp.example.com/device?user_code=WXYZ'}) + self.assertIn('or open directly', with_complete) + self.assertIn( + 'https://idp.example.com/device?user_code=WXYZ', with_complete) + without = format_prompt({ + 'user_code': 'WXYZ', + 'verification_uri': 'https://idp.example.com/device'}) + self.assertNotIn('or open directly', without) + def test_oidc_error_sanitizes_message_and_fields(self): # OidcError strips control/bidi chars from its message centrally, so no # raise site can leak an ANSI/bidi sequence into an uncaught traceback (a @@ -4538,5 +4661,872 @@ def _display(self, html_str): self.assertNotIn(' _MAX_FILE_BYTES + self.assertIsNone(self.store.load(self.key)) + + def test_empty_file_ignored(self): + open(self._file(), 'wb').close() + self.assertIsNone(self.store.load(self.key)) + + def test_garbage_file_ignored(self): + with open(self._file(), 'w') as fh: + fh.write('not json {{{') + self.assertIsNone(self.store.load(self.key)) + + def test_non_object_json_ignored(self): + with open(self._file(), 'w') as fh: + fh.write('[1, 2, 3]') + self.assertIsNone(self.store.load(self.key)) + + def test_wrong_schema_version_ignored(self): + with open(self._file(), 'w') as fh: + json.dump({'v': 2, 'client_id': 'questdb', + 'token_endpoint': 'https://idp:443/token', + 'device_authorization_endpoint': + 'https://idp:443/device', + 'scope': 'openid', 'groups_in_token': False, + 'refresh_token': 'RT'}, fh) + self.assertIsNone(self.store.load(self.key)) + + def test_fingerprint_mismatch_ignored(self): + # A file copied/renamed to a different identity's name still carries the + # original fingerprint; the in-file re-check rejects it (defence in depth + # against a hash collision), independent of the file-name hash. + other = TokenStoreKey( + 'other', 'https://idp:443/token', 'https://idp:443/device', + 'openid', None, False) + self.store.save(self.key, self._pt()) + shutil.copy(self._file(self.key), self._file(other)) + self.assertIsNone(self.store.load(other)) + + def test_audience_null_omitted_and_literal_null_roundtrips(self): + # A None audience is omitted (not written as JSON null), and a token that + # is literally the string "null" round-trips verbatim. + self.store.save(self.key, self._pt(refresh_token='null')) + with open(self._file()) as fh: + raw = fh.read() + self.assertNotIn('"audience"', raw) + self.assertEqual(self.store.load(self.key).refresh_token, 'null') + + def test_audience_in_fingerprint_roundtrips(self): + key = TokenStoreKey( + 'questdb', 'https://idp:443/token', 'https://idp:443/device', + 'openid', 'api://billing', False) + self.store.save(key, self._pt()) + with open(self._file(key)) as fh: + self.assertIn('"audience"', fh.read()) + self.assertIsNotNone(self.store.load(key)) + # A different audience is a different identity (different file). + other = TokenStoreKey( + 'questdb', 'https://idp:443/token', 'https://idp:443/device', + 'openid', 'api://other', False) + self.assertIsNone(self.store.load(other)) + + def test_absent_token_fields_read_back_as_none(self): + self.store.save(self.key, self._pt(access_token=None, id_token=None)) + got = self.store.load(self.key) + self.assertIsNone(got.access_token) + self.assertIsNone(got.id_token) + self.assertEqual(got.refresh_token, 'RT') + + def test_clear_removes_file_and_is_idempotent(self): + self.store.save(self.key, self._pt()) + self.store.clear(self.key) + self.assertIsNone(self.store.load(self.key)) + self.store.clear(self.key) # no-op, must not raise + + def test_constructor_validates_args(self): + with self.assertRaises(OidcConfigError): + FileTokenStore('') + with self.assertRaises(OidcConfigError): + FileTokenStore(self.dir, lock_acquire_budget=0) + with self.assertRaises(OidcConfigError): + FileTokenStore(self.dir, lock_stale=-1) + # lock_stale must EXCEED the worst-case live hold, not merely be + # positive: a tiny window would let a peer steal a LIVE holder's lock + # mid-refresh. A positive-but-too-small value, and the boundary itself, + # are rejected; a value above the floor is accepted. + with self.assertRaises(OidcConfigError): + FileTokenStore(self.dir, lock_stale=1) + with self.assertRaises(OidcConfigError): + FileTokenStore(self.dir, lock_stale=_MIN_LOCK_STALE) + FileTokenStore(self.dir, lock_stale=_MIN_LOCK_STALE + 1) # ok + + def test_at_default_location_honours_env(self): + with mock.patch.dict(os.environ, {TOKEN_STORE_DIR_ENV: self.dir}): + store = FileTokenStore.at_default_location() + store.save(self.key, self._pt()) + self.assertTrue(os.path.exists(self._file())) + + def test_at_default_location_unresolvable_home_raises(self): + # With no env override and no resolvable home (HOME/USERPROFILE unset and + # no passwd entry, e.g. a distroless container), expanduser('~') returns + # the literal '~'. Joining onto it would create a surprise RELATIVE '~' + # directory under cwd, so fail clearly and point at the env override. + env = {k: v for k, v in os.environ.items() if k != TOKEN_STORE_DIR_ENV} + with mock.patch.dict(os.environ, env, clear=True), \ + mock.patch('os.path.expanduser', return_value='~'): + with self.assertRaises(OidcConfigError) as cm: + FileTokenStore.at_default_location() + self.assertIn(TOKEN_STORE_DIR_ENV, str(cm.exception)) + + def test_repr_hides_secrets(self): + text = repr(self._pt(access_token='SECRET-AT', refresh_token='SECRET-RT')) + self.assertNotIn('SECRET-AT', text) + self.assertNotIn('SECRET-RT', text) + + def test_in_lock_runs_action_and_releases(self): + lock = os.path.join(self.dir, self.key.hash() + '.lock') + seen = {} + + def action(): + seen['held'] = os.path.exists(lock) + return 'result' + + self.assertEqual(self.store.in_lock(self.key, action), 'result') + self.assertTrue(seen['held']) # held for the whole action + self.assertFalse(os.path.exists(lock)) # released afterwards + + def test_in_lock_releases_on_exception(self): + lock = os.path.join(self.dir, self.key.hash() + '.lock') + + def boom(): + raise OidcError('boom') + + with self.assertRaises(OidcError): + self.store.in_lock(self.key, boom) + self.assertFalse(os.path.exists(lock)) # released despite the exception + + def test_in_lock_degrades_when_lock_is_held(self): + store = FileTokenStore( + self.dir, lock_acquire_budget=0.1, lock_stale=600) + os.makedirs(self.dir, exist_ok=True) + lock = os.path.join(self.dir, self.key.hash() + '.lock') + open(lock, 'w').close() # a live peer holds it + ran = [] + store.in_lock(self.key, lambda: ran.append(1)) + self.assertEqual(ran, [1]) # degraded: ran without the lock + self.assertTrue(os.path.exists(lock)) # peer's lock left untouched + + def test_in_lock_steals_a_stale_lock(self): + # lock_stale must clear _MIN_LOCK_STALE; back-date the lock well past it + # (os.utime, instant) rather than use a tiny window, so the lock reads as + # abandoned without a real wait. + store = FileTokenStore( + self.dir, lock_acquire_budget=1.0, lock_stale=300) + os.makedirs(self.dir, exist_ok=True) + lock = os.path.join(self.dir, self.key.hash() + '.lock') + open(lock, 'w').close() + os.utime(lock, (time.time() - 100_000, time.time() - 100_000)) # stale + seen = {} + store.in_lock(self.key, lambda: seen.setdefault( + 'held', os.path.exists(lock))) + self.assertTrue(seen['held']) # stole it and re-created + self.assertFalse(os.path.exists(lock)) # released afterwards + + def test_concurrent_steal_stays_exclusive_two_threads(self): + # Two threads racing to break ONE stale lock must not both run at once: + # the atomic rename-aside steal re-checks staleness on the moved file and + # restores a peer's FRESH lock instead of deleting it (a blind os.remove + # would let the slower thread delete the winner's just-created lock and + # both believe they won). With two threads there is no third to exploit + # the brief restore window, so exclusion here is reliable. (Perfect + # exclusion under pathological N-way concurrent stealing is best-effort by + # design — a file lock can't guarantee it without OS support; the + # no-stale-lock serialization property is covered by + # test_in_lock_serializes_concurrent_acquirers, and no-hang/no-leak under + # heavier steal contention by the test below.) + store = FileTokenStore( + self.dir, lock_acquire_budget=2.0, lock_stale=300) + os.makedirs(self.dir, exist_ok=True) + lock = os.path.join(self.dir, self.key.hash() + '.lock') + open(lock, 'w').close() + os.utime(lock, (time.time() - 100_000, time.time() - 100_000)) # stale + counter_lock = threading.Lock() + active = [0] + max_seen = [0] + ran = [0] + + def action(): + with counter_lock: + active[0] += 1 + max_seen[0] = max(max_seen[0], active[0]) + time.sleep(0.02) + with counter_lock: + active[0] -= 1 + ran[0] += 1 + + threads = [threading.Thread( + target=lambda: store.in_lock(self.key, action)) for _ in range(2)] + for t in threads: + t.start() + for t in threads: + t.join(timeout=30) + self.assertEqual(ran[0], 2) # both actions ran + self.assertEqual(max_seen[0], 1) # never two at once + self.assertFalse(os.path.exists(lock)) # released; no leftover + + def test_concurrent_steal_does_not_hang_or_leak(self): + # Many threads racing to break ONE stale lock must not deadlock, must all + # eventually run, and must leave no lock or `.stale.` temp file behind + # (the steal renames aside and either removes a confirmed-stale file or + # restores a fresh one — never orphaning a temp). This stresses the steal + # path under contention; exclusion under that contention is best-effort + # (see the two-thread test above), so this asserts the guarantees that + # always hold. + store = FileTokenStore( + self.dir, lock_acquire_budget=10.0, lock_stale=300) + os.makedirs(self.dir, exist_ok=True) + lock = os.path.join(self.dir, self.key.hash() + '.lock') + open(lock, 'w').close() + os.utime(lock, (time.time() - 100_000, time.time() - 100_000)) # stale + ran = [0] + counter_lock = threading.Lock() + + def action(): + with counter_lock: + ran[0] += 1 + time.sleep(0.01) + + threads = [threading.Thread( + target=lambda: store.in_lock(self.key, action)) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join(timeout=30) + self.assertFalse(any(t.is_alive() for t in threads)) # no deadlock + self.assertEqual(ran[0], 8) # all ran + self.assertFalse(os.path.exists(lock)) # lock cleaned up + self.assertEqual( # no temp leak + [n for n in os.listdir(self.dir) if '.stale.' in n], []) + + def test_steal_recheck_does_not_delete_a_lock_that_became_fresh(self): + # The core of the atomic steal, tested deterministically: if a lock is + # judged stale by the acquire-loop check but turns out FRESH by the time + # it is moved aside (a peer recreated it — exactly the TOCTOU the original + # blind os.remove(lock) mishandled), it must be restored, never deleted, + # and we must NOT acquire over the live peer. Drive that race with a + # stubbed _is_stale: "stale" to the first (acquire-loop) check, "fresh" to + # the post-rename re-check. + store = FileTokenStore( + self.dir, lock_acquire_budget=0.2, lock_stale=300) + os.makedirs(self.dir, exist_ok=True) + lock = os.path.join(self.dir, self.key.hash() + '.lock') + with open(lock, 'w') as f: + f.write('PEER') # a peer's fresh (live) lock, with a marker + calls = [0] + + def fake_is_stale(_): + calls[0] += 1 + return calls[0] == 1 # stale to the acquire check, fresh on re-check + + with mock.patch.object(store, '_is_stale', fake_is_stale): + held = store._acquire_lock(lock) + self.assertFalse(held) # deferred to the peer, did not steal + self.assertTrue(os.path.exists(lock)) # peer's live lock not destroyed + with open(lock) as f: + self.assertEqual(f.read(), 'PEER') # restored intact, not overwritten + + def test_in_lock_serializes_concurrent_acquirers(self): + # The O_CREAT|O_EXCL lock file must actually serialize two real threads + # sharing one store + key: with an acquire budget generous relative to + # the tiny holds (so neither degrades to the lock-free path), no two + # actions ever overlap. This exercises the serialization PROPERTY the + # lock exists for, which the other in_lock tests (single-threaded, with a + # hand-placed lock file) do not. + store = FileTokenStore(self.dir, lock_acquire_budget=10.0, lock_stale=600) + os.makedirs(self.dir, exist_ok=True) + counter_lock = threading.Lock() + active = [0] + max_seen = [0] + ran = [0] + + def action(): + with counter_lock: + active[0] += 1 + max_seen[0] = max(max_seen[0], active[0]) + time.sleep(0.02) + with counter_lock: + active[0] -= 1 + ran[0] += 1 + + threads = [threading.Thread(target=lambda: store.in_lock(self.key, action)) + for _ in range(6)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertEqual(ran[0], 6) # all actions ran + self.assertEqual(max_seen[0], 1) # never two at once + + def test_non_finite_and_negative_millis(self): + # json.loads accepts bare NaN / Infinity; a hand-edited or hostile file + # must not smuggle a non-finite timestamp into the expiry math. Non-finite + # reads as 0.0 (expired); a finite negative value passes through (it is + # rejected later by TokenSet.is_valid, which treats expires_at <= 0 as + # expired), and non-numeric / bool read as 0.0. + self.assertEqual(_millis_to_seconds(float('nan')), 0.0) + self.assertEqual(_millis_to_seconds(float('inf')), 0.0) + self.assertEqual(_millis_to_seconds(float('-inf')), 0.0) + self.assertEqual(_millis_to_seconds(-5000), -5.0) + self.assertEqual(_millis_to_seconds('x'), 0.0) + self.assertEqual(_millis_to_seconds(True), 0.0) + # End to end: a file with an Infinity expiry loads as expired (0.0), not + # as a token valid forever. + with open(self._file(), 'w') as fh: + fh.write('{"v":1,"client_id":"questdb",' + '"token_endpoint":"https://idp:443/token",' + '"device_authorization_endpoint":"https://idp:443/device",' + '"scope":"openid","groups_in_token":false,' + '"refresh_token":"RT","access_token":"AT","id_token":"IT",' + '"expires_at_millis":Infinity,"token_ttl_millis":Infinity}') + got = self.store.load(self.key) + self.assertEqual(got.expires_at, 0.0) + self.assertEqual(got.token_ttl, 0.0) + + def test_seconds_to_millis_maps_non_finite_to_zero(self): + # Inverse of _millis_to_seconds: a finite value scales to millis; a + # non-finite / non-numeric / bool maps to 0 (expired), so the serializer + # can't raise a raw OverflowError/ValueError on it. + self.assertEqual(_seconds_to_millis(3.6), 3600) + self.assertEqual(_seconds_to_millis(0.0), 0) + self.assertEqual(_seconds_to_millis(-5.0), -5000) + self.assertEqual(_seconds_to_millis(float('inf')), 0) + self.assertEqual(_seconds_to_millis(float('-inf')), 0) + self.assertEqual(_seconds_to_millis(float('nan')), 0) + self.assertEqual(_seconds_to_millis('x'), 0) + self.assertEqual(_seconds_to_millis(True), 0) + + def test_save_non_finite_expiry_does_not_raise(self): + # PersistedToken is public, so a direct caller can pass a non-finite + # expiry. save() must keep the OidcError contract (not raise a raw + # OverflowError from int(round(inf)) / ValueError from round(nan)) and + # store it as expired (0), symmetric with the load side. + self.store.save(self.key, self._pt( + expires_at=float('inf'), token_ttl=float('nan'))) + got = self.store.load(self.key) + self.assertEqual(got.expires_at, 0.0) + self.assertEqual(got.token_ttl, 0.0) + self.assertEqual(got.refresh_token, 'RT') # the rest still round-trips + + def test_schema_version_and_canonical_prefix_are_linked(self): + # The on-disk 'v' field and the hash-prefix version are derived from one + # constant, so they can't drift apart on a future format bump. + self.assertEqual(_CANONICAL_PREFIX, f'questdb-oidc-token-v{_SCHEMA_VERSION}') + + def test_round_trip_ipv6_endpoint(self): + # An IPv6-endpoint identity round-trips: the bracketed canonical form is + # stored and re-matched on load. + key = TokenStoreKey( + 'questdb', 'https://[::1]:443/token', 'https://[::1]:443/device', + 'openid', None, False) + self.store.save(key, self._pt()) + self.assertEqual(self.store.load(key).refresh_token, 'RT') + + +class _FakeStore(TokenStore): + """An in-memory TokenStore for the auth-level persistence tests. + + Persists a single PersistedToken across simulated restarts and counts the + SPI calls, so a test can assert that a restart resumed without a device flow. + """ + + def __init__(self): + self.saved = None + self.saves = 0 + self.loads = 0 + self.clears = 0 + self.in_locks = 0 + self.fail_save = False + self.fail_clear = False + self.fail_in_lock = False + # Optional override: load_fn(loads_count) -> PersistedToken | None, so a + # test can model a peer that refreshes between two loads. + self.load_fn = None + + def load(self, key): + self.loads += 1 + if self.load_fn is not None: + return self.load_fn(self.loads) + return self.saved + + def save(self, key, token): + self.saves += 1 + if self.fail_save: + raise OidcError('simulated disk failure') + self.saved = token + + def clear(self, key): + self.clears += 1 + if self.fail_clear: + raise OidcError('simulated clear failure') + self.saved = None + + def in_lock(self, key, action): + self.in_locks += 1 + if self.fail_in_lock: + # Model a custom store whose lock backend fails. The contract says a + # raised store failure is non-fatal; OidcDeviceAuth must degrade, not + # abort token(). + raise OidcError('simulated lock failure') + return action() + + +class _CountingFileStore(FileTokenStore): + """The REAL FileTokenStore, counting in_lock entries. + + Lets an auth-level test assert that a coordinated refresh persists the + rotated token INLINE (one in_lock for the whole read-refresh-write) rather + than re-acquiring the lock it already holds for the nested save — the + _store_lock_held guard. _FakeStore can't catch that regression: it no-ops the + real lock. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.in_lock_calls = 0 + + def in_lock(self, key, action): + self.in_lock_calls += 1 + return super().in_lock(key, action) + + +class TestPersistence(AuthTestBase): + """OidcDeviceAuth wired to a TokenStore (opt-in persistence).""" + + def _restart(self): + # Simulate a process restart: the on-disk/fake store survives, but the + # process-global in-memory cache does not — so the next instance must + # resume from the store, not the shared MemoryCache. + _MEMORY_STORE.clear() + _MEMORY_GENERATION.clear() + _MEMORY_INFLIGHT.clear() + + def _reset_server_counters(self): + self.state.device_requests = 0 + self.state.token_requests = [] + self.state.refresh_requests = 0 + self.state.refresh_forms = [] + + def test_sign_in_persists_token(self): + store = _FakeStore() + auth = self.make_auth(token_store=store) + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(store.saves, 1) + self.assertEqual(store.saved.refresh_token, 'REFRESH-1') + self.assertEqual(store.saved.id_token, ID_TOKEN) + + def test_no_store_means_no_persistence(self): + # Default behaviour is unchanged: no store, nothing persisted. + auth = self.make_auth() + self.assertIsNone(auth._token_store) + self.assertIsNone(auth._store_key) + self.assertEqual(auth.token(), ID_TOKEN) + + def test_restart_with_valid_token_is_zero_network(self): + store = _FakeStore() + self.make_auth(token_store=store).token() # sign in, persist + self._restart() + self._reset_server_counters() + # Fresh instance + fresh clock: the persisted token is still valid. + auth = self.make_auth(token_store=store, clock=FakeClock()) + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(self.state.device_requests, 0) # no re-prompt + self.assertEqual(self.state.token_requests, []) # no poll + self.assertEqual(self.state.refresh_requests, 0) # no refresh either + + def test_restart_with_expired_token_refreshes_silently(self): + store = _FakeStore() + self.make_auth(token_store=store).token() # sign in, persist + self._restart() + self._reset_server_counters() + # The persisted access/id token has expired, but the refresh token is + # valid: resume with one silent refresh, never the device flow. + late = FakeClock() + late.wall += 10_000 + auth = self.make_auth(token_store=store, clock=late) + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(self.state.device_requests, 0) # no re-prompt + self.assertEqual(self.state.refresh_requests, 1) # silent refresh + self.assertGreaterEqual(store.in_locks, 1) # coordinated refresh + self.assertEqual( + self.state.refresh_forms[0]['refresh_token'], 'REFRESH-1') + + def test_token_works_as_first_call_after_restore(self): + # No explicit sign-in: token() is the entry point and resumes from disk. + store = _FakeStore() + self.make_auth(token_store=store).token() + self._restart() + self._reset_server_counters() + auth = self.make_auth(token_store=store, clock=FakeClock()) + self.assertEqual(auth.token(), ID_TOKEN) # straight from the store + self.assertEqual(self.state.device_requests, 0) + + def test_non_rotating_refresh_writes_once(self): + store = _FakeStore() + clock = FakeClock() + auth = self.make_auth(token_store=store, clock=clock) + auth.token() + self.assertEqual(store.saves, 1) + # Expire the cached token; the default refresh returns the SAME refresh + # token, so the file is not rewritten (the on-disk one is still valid). + clock.wall += 10_000 + auth.token() + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual(store.saves, 1) # no rewrite on a non-rotating refresh + + def test_rotating_refresh_rewrites_file(self): + store = _FakeStore() + clock = FakeClock() + auth = self.make_auth(token_store=store, clock=clock) + auth.token() + self.assertEqual(store.saves, 1) + self.state.refresh_response = (200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'refresh_token': 'REFRESH-2', # rotated + 'token_type': 'Bearer', 'expires_in': 3600}) + clock.wall += 10_000 + auth.token() + self.assertEqual(store.saves, 2) # rewritten with the rotated token + self.assertEqual(store.saved.refresh_token, 'REFRESH-2') + + def test_transient_refresh_error_propagates_through_lock(self): + # A transient 5xx during a coordinated (lock-held) refresh surfaces as + # OidcNetworkError — the refresh token is kept for a retry, not discarded + # into a needless re-prompt — and never falls through to the device flow. + store = _FakeStore() + clock = FakeClock() + auth = self.make_auth(token_store=store, clock=clock) + auth.token() # sign in, persist + clock.wall += 10_000 # expire the access/id token + self.state.refresh_response = (503, {'error': 'server_error'}) + with self.assertRaises(OidcNetworkError): + auth.token() + self.assertEqual(self.state.device_requests, 1) # only the sign-in + self.assertGreaterEqual(store.in_locks, 1) + + def test_clear_removes_persisted_entry_and_reprompts(self): + store = _FakeStore() + auth = self.make_auth(token_store=store) + auth.token() + self.assertEqual(store.saves, 1) + auth.clear() + self.assertEqual(store.clears, 1) + self.assertIsNone(store.saved) + # A fresh instance after the clear finds nothing and re-prompts. + self._restart() + self._reset_server_counters() + self.make_auth(token_store=store).token() + self.assertEqual(self.state.device_requests, 1) + self.assertEqual(store.saves, 2) # the new sign-in persists again + + def test_save_failure_is_non_fatal(self): + store = _FakeStore() + store.fail_save = True + auth = self.make_auth(token_store=store) + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertEqual(auth.token(), ID_TOKEN) # valid despite save failing + self.assertEqual(store.saves, 1) # attempted + self.assertIn('token store save failed', err.getvalue()) + + def test_load_failure_is_non_fatal(self): + store = _FakeStore() + + def boom(_): + raise OidcError('simulated read failure') + + store.load_fn = boom + auth = self.make_auth(token_store=store) + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertEqual(auth.token(), ID_TOKEN) # falls back to device flow + self.assertEqual(self.state.device_requests, 1) + self.assertIn('token store load failed', err.getvalue()) + + def test_persisted_token_with_control_char_is_rejected(self): + # The file is attacker-writable: a served token carrying a control char + # (a CR/LF injection vector) is rejected and the entry ignored, falling + # back to a fresh sign-in rather than routing it onto the wire. + store = _FakeStore() + store.saved = PersistedToken( + access_token=ACCESS_TOKEN, id_token='bad\x01id-token', + refresh_token='REFRESH-1', + expires_at=FakeClock().now() + 3600, token_ttl=3600.0) + self._restart() + self._reset_server_counters() + auth = self.make_auth(token_store=store, clock=FakeClock()) # groups mode + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(self.state.device_requests, 1) # rejected -> device flow + + def test_coordinated_refresh_adopts_peer_rotation(self): + # Under the cross-process lock, re-reading the store sees a peer's freshly + # rotated token and adopts it instead of POSTing a (now revoked) refresh. + store = _FakeStore() + now = FakeClock().now() + expired = PersistedToken( + access_token=ACCESS_TOKEN, id_token=ID_TOKEN, + refresh_token='REFRESH-1', expires_at=now - 10, token_ttl=3600.0) + fresh = PersistedToken( + access_token=ACCESS_TOKEN, id_token=ID_TOKEN, + refresh_token='REFRESH-2', expires_at=now + 3600, token_ttl=3600.0) + # 1st load (lazy) sees the expired entry; 2nd load (re-read under the + # lock) sees the peer's fresh, rotated entry. + store.load_fn = lambda n: expired if n == 1 else fresh + auth = self.make_auth(token_store=store, clock=FakeClock()) + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(self.state.refresh_requests, 0) # adopted, no network + self.assertEqual(store.in_locks, 1) + self.assertEqual(auth._tokens.refresh_token, 'REFRESH-2') + + def test_timeout_cap_rejected(self): + # The HTTP timeout is capped so a slow refresh can't outlast the file + # store's lock-staleness window. + with self.assertRaises(OidcConfigError): + self.make_auth(timeout=300) + + def test_store_key_built_from_canonical_config(self): + store = _FakeStore() + auth = self.make_auth(token_store=store, groups_in_token=True) + key = auth._store_key + self.assertEqual(key.client_id, 'questdb') + self.assertTrue(key.token_endpoint.endswith('/token')) + # Explicit numeric port in the authority (a bare ':' would also match the + # scheme, so assert a port follows the host). + self.assertRegex(key.token_endpoint, r'://[^/]+:\d+/') + self.assertTrue(key.groups_in_token) + # 'openid' is auto-added in groups mode and is part of the identity. + self.assertIn('openid', key.scope) + + def test_control_char_in_served_token_rejected_access_mode(self): + # The served-token character screen must also fire when the server does + # NOT expect groups in the token: token() then serves the access_token, so + # a control char there (a CR/LF / header-injection vector) must reject the + # whole entry and fall back to a fresh sign-in. Mirrors the groups-mode + # test, covering the other branch of the served-kind selection. + store = _FakeStore() + store.saved = PersistedToken( + access_token='bad\x01access-token', id_token=ID_TOKEN, + refresh_token='REFRESH-1', + expires_at=FakeClock().now() + 3600, token_ttl=3600.0) + self._restart() + self._reset_server_counters() + auth = self.make_auth( + token_store=store, groups_in_token=False, clock=FakeClock()) + self.assertEqual(auth.token(), ACCESS_TOKEN) + self.assertEqual(self.state.device_requests, 1) # rejected -> device flow + + def test_clear_failure_is_non_fatal(self): + # A store that raises on clear() must not break OidcDeviceAuth.clear(): + # it warns to stderr and carries on (the in-memory/process cache is still + # cleared). Mirrors save/load being best-effort. + store = _FakeStore() + store.fail_clear = True + auth = self.make_auth(token_store=store) + auth.token() + err = io.StringIO() + with contextlib.redirect_stderr(err): + auth.clear() # must not raise despite the store failing + self.assertEqual(store.clears, 1) + self.assertIn('token store clear failed', err.getvalue()) + + def test_disk_persist_skipped_when_generation_bumped(self): + # The disk save honors the same clear()-generation as the shared-cache + # write: if a concurrent clear() bumped the generation between the cache + # CAS and the save, _save_if_current skips it, so the file the clear() + # deleted is not resurrected. + store = _FakeStore() + auth = self.make_auth(token_store=store) + auth.token() # sign in, persist once + self.assertEqual(store.saves, 1) + key = auth.cache_key + # A concurrent clear() bumps the generation while an acquisition is in + # flight (generation() marks it in-flight, so clear() bumps, not prunes). + stale = auth._cache.generation(key) + auth._cache.clear(key) + try: + auth._save_if_current(stale, 'SOME-RT') + finally: + auth._cache.release(key) + self.assertEqual(store.saves, 1) # save skipped: clear() won + # Sanity: with the current generation it WOULD save. + current = auth._cache.generation(key) + try: + auth._save_if_current(current, 'SOME-RT2') + finally: + auth._cache.release(key) + self.assertEqual(store.saves, 2) + + # -- end-to-end with the REAL FileTokenStore -------------------------------- + # The tests above wire a _FakeStore whose in_lock no-ops, so the real + # on-disk file, the cross-process lock, and the under-lock save are never + # exercised together through OidcDeviceAuth. These drive the real store. + + def _file_store_dir(self): + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, ignore_errors=True) + return d + + def test_real_file_store_round_trips_across_restart(self): + d = self._file_store_dir() + auth = self.make_auth(token_store=FileTokenStore.at(d)) + self.assertEqual(auth.token(), ID_TOKEN) # sign in, persist to disk + json_files = [n for n in os.listdir(d) if n.endswith('.json')] + self.assertEqual(len(json_files), 1) # one file on disk + self._restart() # drop the in-memory cache + self._reset_server_counters() + auth2 = self.make_auth( + token_store=FileTokenStore.at(d), clock=FakeClock()) + self.assertEqual(auth2.token(), ID_TOKEN) # resumed from disk + self.assertEqual(self.state.device_requests, 0) # no re-prompt + self.assertEqual(self.state.refresh_requests, 0) # token still valid + + def test_real_file_store_rotating_refresh_saves_inline_under_lock(self): + # A rotating refresh persists the rotated token through the REAL lock via + # the coordinated path, saving INLINE rather than re-acquiring the lock it + # already holds. Assert the rotated token reached disk AND that the + # refresh took exactly ONE further in_lock (the coordination); a + # _store_lock_held regression would re-enter in_lock for the nested save. + d = self._file_store_dir() + store = _CountingFileStore.at(d) + clock = FakeClock() + auth = self.make_auth(token_store=store, clock=clock) + auth.token() # sign in: 1 in_lock (save) + self.assertEqual(store.in_lock_calls, 1) + self.state.refresh_response = (200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'refresh_token': 'REFRESH-2', 'token_type': 'Bearer', + 'expires_in': 3600}) + clock.wall += 10_000 # expire -> coordinated + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual(store.in_lock_calls, 2) # +1 coordination, save inline + self.assertEqual( + store.load(auth._store_key).refresh_token, 'REFRESH-2') # on disk + + # -- custom-store in_lock failures are best-effort (non-fatal) -------------- + + def test_in_lock_failure_is_non_fatal_on_sign_in(self): + # A custom store whose in_lock raises (its lock backend failed) must not + # break a completed sign-in: persistence is best-effort, so token() warns + # and returns the valid in-memory token. (The bundled FileTokenStore + # degrades internally and never raises here.) + store = _FakeStore() + store.fail_in_lock = True + auth = self.make_auth(token_store=store) + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertEqual(auth.token(), ID_TOKEN) + self.assertIn('token store', err.getvalue()) + + def test_in_lock_failure_degrades_refresh_to_lock_free(self): + # If in_lock raises on the coordinated-refresh path, degrade to a + # lock-free refresh rather than abort: the silent refresh still happens + # and token() is served, never a needless device-flow re-prompt. Also + # exercises the second guarded site (the save's in_lock then fails too and + # is swallowed). + store = _FakeStore() + clock = FakeClock() + auth = self.make_auth(token_store=store, clock=clock) + auth.token() # sign in (in_lock still works) + store.fail_in_lock = True # now the lock backend fails + clock.wall += 10_000 # expire -> needs a refresh + err = io.StringIO() + with contextlib.redirect_stderr(err): + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(self.state.refresh_requests, 1) # refreshed lock-free + self.assertEqual(self.state.device_requests, 1) # only the sign-in + self.assertIn('token store', err.getvalue()) + + if __name__ == '__main__': unittest.main() From 0b637195319f6a4cf45a2058fee80b1a58935454 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 30 Jun 2026 01:07:33 +0100 Subject: [PATCH 088/104] fix(auth): bound the response head read by the deadline The response HEAD read (status line + headers) inside urllib's open() had no wall-clock bound: urllib's timeout is per-socket-read and http.client's begin() loops readline() over reads, so a peer dribbling the status/header bytes one per timeout window kept open() blocked for up to ~_MAXLINE * timeout per line. This pinned the thread holding the acquisition lock and defeated the existing body-read watchdog, which is armed only after open() returns. Capture the connection socket as soon as it connects (a do_open-only mixin that swaps in a socket-capturing connection class, leaving the stdlib's version-specific TLS handling untouched) and arm a watchdog that shuts it down at the deadline, released once the head read finishes so it can't disturb the body read. A stalled head now surfaces as a typed OidcNetworkError instead of hanging. Add a real-socket regression test that dribbles the status line. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_http.py | 128 ++++++++++++++++++++++++++++++++++++-- test/test_auth.py | 68 ++++++++++++++++++++ 2 files changed, 192 insertions(+), 4 deletions(-) diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 6df0eec5..11fc2bac 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -175,11 +175,109 @@ def redirect_request(self, *args, **kwargs): return None -def _opener(ctx: Optional[ssl.SSLContext]) -> urllib.request.OpenerDirector: +class _DeadlineSocket: + """Holds the live connection socket so an overall-deadline watchdog can break + a *head* read (status line + headers) that dribbles past the deadline. + + ``urllib``'s timeout is per-socket-read, and ``http.client``'s + ``begin()``/``_read_status`` loops ``readline()`` over socket reads — so a + peer feeding the status/header bytes one per timeout window keeps a single + ``open()`` blocked for up to ``_MAXLINE`` (~64k) * timeout *per line*, pinning + the calling thread (which holds the acquisition lock). :func:`_read_body` + already guards the *body* this way; this guards the phase inside ``open()`` + (everything after the socket connects), which ``_read_body``'s watchdog — + armed only once ``open()`` returns — cannot reach. + + The socket is shut down at most once, and never after :meth:`release` (called + when ``open()`` returns or raises), so a timer that fires in the instant the + head read finishes can't tear down the healthy socket the body read is about + to use. + """ + + __slots__ = ('_sock', '_done', '_lock') + + def __init__(self) -> None: + self._sock: Any = None + self._done = False + self._lock = threading.Lock() + + def attach(self, sock: Any) -> None: + # Called from the opener's connection once it has connected. Ignored once + # the head phase is over, so a late connect can't re-arm a released guard. + with self._lock: + if not self._done: + self._sock = sock + + def shutdown(self) -> None: + # The watchdog action. Shut the socket down (inside the lock, so it is + # mutually exclusive with release) to break a head read blocked past the + # deadline; a no-op once released. + with self._lock: + if self._done or self._sock is None: + return + _shutdown_socket(self._sock) + + def release(self) -> None: + # The head read is over; stop guarding so the body read's own watchdog + # owns the socket without interference. + with self._lock: + self._done = True + + +def _capturing_connection(base: type, watch: _DeadlineSocket) -> type: + # A connection class that hands its socket to `watch` the moment it connects, + # so the head-read watchdog (armed in `request`) can break a stalled status/ + # header read. Built per request because `watch` is per request. + class _CapturingConnection(base): # type: ignore[valid-type,misc] + def connect(self): + super().connect() + watch.attach(self.sock) + + return _CapturingConnection + + +class _CaptureMixin: + """Mixin for HTTP(S) handlers that swaps in a socket-capturing connection. + + It overrides only ``do_open`` to wrap whatever connection class the stdlib + handler hands it, so the version-specific ``http_open``/``https_open`` logic + (TLS context / ``check_hostname`` handling, which differs across CPython + releases) runs unchanged — this layer only records the connection's socket + for the head-read watchdog, never touching TLS verification. + """ + + def __init__(self, watch: '_DeadlineSocket', *args, **kwargs): + self._watch = watch + super().__init__(*args, **kwargs) + + def do_open(self, http_class, req, **kwargs): + return super().do_open( + _capturing_connection(http_class, self._watch), req, **kwargs) + + +class _CapturingHTTPHandler(_CaptureMixin, urllib.request.HTTPHandler): + """``HTTPHandler`` whose connection captures its socket for the watchdog.""" + + +class _CapturingHTTPSHandler(_CaptureMixin, urllib.request.HTTPSHandler): + """``HTTPSHandler`` whose connection captures its socket for the watchdog.""" + + +def _opener( + ctx: Optional[ssl.SSLContext], + watch: Optional['_DeadlineSocket'] = None, +) -> urllib.request.OpenerDirector: # build_opener keeps the default ProxyHandler (reads *_PROXY env vars) while - # letting us pin our own TLS context and forbid redirects. + # letting us pin our own TLS context and forbid redirects. When `watch` is + # given, swap in socket-capturing HTTP(S) handlers so the head-read watchdog + # can reach the connection socket; the capturing HTTPS handler forwards the + # same `ctx` (None => stdlib default context), so TLS verification is + # unchanged on either path. handlers: list = [_NoRedirect()] - if ctx is not None: + if watch is not None: + handlers.append(_CapturingHTTPHandler(watch)) + handlers.append(_CapturingHTTPSHandler(watch, context=ctx)) + elif ctx is not None: handlers.append(urllib.request.HTTPSHandler(context=ctx)) return urllib.request.build_opener(*handlers) @@ -334,7 +432,29 @@ def request( req_headers.update(headers) req = urllib.request.Request( url, data=body, headers=req_headers, method=method.upper()) - with _opener(ctx).open(req, timeout=timeout) as resp: + # Bound the response HEAD read (status line + headers) by the same + # wall-clock as the body. urllib's timeout is per-socket-read and + # http.client's begin() loops readline() over reads, so without this a + # peer dribbling the status/header bytes one per timeout window keeps + # open() blocked for ~_MAXLINE * timeout per line — pinning this thread, + # which holds the acquisition lock. The capturing opener hands the + # connection socket to `watch` as soon as it connects; the watchdog shuts + # it down at the deadline to break such a stall (mapped to a typed + # OidcNetworkError by the handlers below). _read_body installs its own + # watchdog for the body that follows. + watch = _DeadlineSocket() + head_timer = threading.Timer(timeout, watch.shutdown) + head_timer.daemon = True + head_timer.start() + try: + resp = _opener(ctx, watch).open(req, timeout=timeout) + finally: + # Head read finished (returned or raised): stop guarding so the body + # read's own watchdog owns the socket, and a late timer fire can't + # tear down the healthy socket. + watch.release() + head_timer.cancel() + with resp: return HttpResponse( getattr(resp, 'status', resp.getcode()), _read_body(resp, max_bytes=_MAX_RESPONSE_BYTES, diff --git a/test/test_auth.py b/test/test_auth.py index 499ee007..05bc2ec0 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -3883,6 +3883,74 @@ def call(): 'watchdog never fired (C1 regression).') self.assertIsInstance(result.get('error'), OidcNetworkError) + def test_request_aborts_real_socket_head_dribble(self): + # Regression for M1 (the HEAD read), against the REAL socket stack. The + # body watchdog above is armed inside _read_body, which runs only after + # open() returns; open() itself reads the status line + headers via + # http.client begin(), whose _read_status()/readline() loops over socket + # reads until it sees a newline. A server that dribbles the STATUS LINE + # one byte at a time, never terminating it, keeps open() blocked for up to + # _MAXLINE (~hours) — the per-socket timeout resets on each byte and the + # body watchdog is not armed yet — hanging the calling thread (which holds + # the acquisition lock). request()'s head watchdog must shut the socket + # down at the deadline and surface a typed OidcNetworkError. + import socket + from questdb.auth import _http + + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(('127.0.0.1', 0)) + srv.listen(1) + port = srv.getsockname()[1] + stop = threading.Event() + + def serve(): + try: + conn, _ = srv.accept() + except OSError: + return + try: + conn.recv(65536) # consume the request line/headers + # Dribble the STATUS LINE one byte at a time, never sending its + # terminating CRLF, each well inside the per-socket timeout window + # so urllib's per-read timeout never fires and begin()'s + # _read_status() stays blocked in readline(). + while not stop.is_set(): + try: + conn.sendall(b'a') + except OSError: + break + stop.wait(0.1) + finally: + conn.close() + + server_thread = threading.Thread(target=serve, daemon=True) + server_thread.start() + + result = {} + + def call(): + try: + _http.request( + 'GET', f'http://127.0.0.1:{port}/x', timeout=1.0) + result['returned'] = True + except Exception as e: # noqa: BLE001 - record for the assert below + result['error'] = e + + t = threading.Thread(target=call, daemon=True) + t.start() + t.join(8.0) + hung = t.is_alive() # capture before cleanup unblocks it + stop.set() + srv.close() + server_thread.join(2.0) + + self.assertFalse( + hung, + 'request() hung on a status-line dribble: the head-read watchdog ' + 'never fired (M1 regression — the response head read is unbounded).') + self.assertIsInstance(result.get('error'), OidcNetworkError) + def test_bad_ca_bundle_raises_config_error(self): # A missing or invalid CA bundle path (explicit or via env) must surface # as OidcConfigError, not a raw FileNotFoundError / ssl.SSLError. See M1. From 7790c4a01fc4089f8763bedf9c72bd7a3187d6d4 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 30 Jun 2026 01:26:16 +0100 Subject: [PATCH 089/104] fix(auth): unify token identity keys; screen network tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M2 — the in-memory cache_key and the on-disk TokenStoreKey derived identity by different rules, so they could disagree: the token-endpoint query string was kept in memory but dropped on disk (two distinct tenants could collide onto one store file and be served each other's token), while scope order and a trailing slash were normalised in memory but not on disk (splitting one identity across two files). Give both a single normalisation: _canonical_endpoint now keeps the query and strips a trailing slash, and the store key's scope is order-normalised via the shared _normalize_scope, exactly as cache_key already does. For the common case (no query, no trailing slash) the on-disk hash is unchanged, so the cross-language file-sharing contract is preserved; edge cases now agree and fail safe. M3 — a token taken straight from the (untrusted) IdP token endpoint was not screened for control/non-ASCII characters, unlike a token loaded from the persistence file. A decoded CR/LF in the served token is an Authorization-header / PG-wire _sso-password injection vector. Route the wire-bound access/id tokens through the same _has_only_token_chars gate the file path uses (new _safe_token_or_none); a screened-out token reads as absent, so the grant fails terminally instead of routing a tampered credential onto the wire. M4 — add the two missing persistence tests: a coordinated refresh using the newer in-memory token when a prior save left disk stale, and clear() staying non-fatal when the store's lock backend (not clear() itself) raises. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 63 +++++++++++++++--- src/questdb/auth/_store.py | 46 ++++++++++---- test/test_auth.py | 123 ++++++++++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+), 21 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index dc1212aa..e8f022b1 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -116,6 +116,21 @@ def _str_or_none(value: Any) -> Optional[str]: return value if isinstance(value, str) else None +def _normalize_scope(scope: Optional[str]) -> str: + """A scope string as its order-insensitive canonical form: the space-joined + sorted token set (``''`` when empty/None). + + Two configs differing only in scope ORDER (``'openid groups'`` vs + ``'groups openid'``) are the SAME identity, so they must share one in-memory + cache entry AND one on-disk token-store file. Used by BOTH + :attr:`OidcDeviceAuth.cache_key` and the :class:`~questdb.auth.TokenStoreKey` + built in ``__init__``, so the two can't disagree on what "the same scope" + means (a disagreement would split one identity across two store files, or — + worse, in the other direction — serve one identity's token to another). + """ + return ' '.join(sorted(scope.split())) if scope else '' + + def _int_or_default(value: Any, default: int) -> int: """ ``int(value)`` for an untrusted numeric field (``expires_in`` / ``interval``), @@ -271,6 +286,26 @@ def _has_only_token_chars(token: str) -> bool: return all(0x20 <= ord(c) <= 0x7e for c in token) +def _safe_token_or_none(value: Any) -> Optional[str]: + """A wire-bound credential token (``access_token``/``id_token``) from an + untrusted IdP response as a printable-ASCII ``str``, else ``None``. + + Like :func:`_str_or_none`, but ALSO drops a token carrying a control or + non-ASCII character. That token is put verbatim into an + ``Authorization: Bearer`` header or a PG-wire ``_sso`` password, where a + decoded CR/LF is a header-injection vector. The IdP is untrusted (hostile or + MITM'd), exactly like the persistence file, so the network path applies the + same gate the file path already does in :meth:`_tokenset_from_persisted` + (via :func:`_has_only_token_chars`). A dropped token reads as absent, so a + missing required kind then raises the clear terminal error (see + :meth:`_select`) rather than routing a tampered credential onto the wire. + """ + token = value if isinstance(value, str) else None + if token is not None and not _has_only_token_chars(token): + return None + return token + + class OidcDeviceAuth: """ Acquire and refresh an OIDC token via the device authorization grant. @@ -446,7 +481,11 @@ def __init__( token_endpoint=_canonical_endpoint(self.config.token_endpoint), device_authorization_endpoint=_canonical_endpoint( self.config.device_authorization_endpoint), - scope=self.config.scope, + # Order-normalise the scope exactly as cache_key does, so the + # in-memory and on-disk identities can't disagree (see + # _normalize_scope); the endpoints are canonicalised the same way + # cache_key's _normalize_url renders them. + scope=_normalize_scope(self.config.scope), audience=self.config.audience, groups_in_token=self.config.groups_in_token) # Load the persisted entry at most once per instance (even if it yields @@ -592,7 +631,7 @@ def cache_key(self) -> str: correcting, but at the cost of avoidable refreshes / re-prompts). """ c = self.config - scope = ' '.join(sorted(c.scope.split())) if c.scope else '' + scope = _normalize_scope(c.scope) # Normalize the issuer and token endpoint alike (lower-case scheme/host, # drop a default port) and strip a trailing slash, so a discovered # "https://idp/token/" and an explicit "https://idp/token" — or a stray @@ -1115,13 +1154,19 @@ def _tokenset_from_response(self, body: Dict[str, Any]) -> TokenSet: # Cap a long (or hostile) IdP-stated lifetime so a cached token is # re-validated at least hourly (matches the Java client). expires_in = min(expires_in, _MAX_EXPIRES_IN) - # Coerce the credential fields to str-or-None up front: a non-string - # token from a buggy/hostile IdP must read as absent rather than be - # stored, re-sent on a refresh, or emitted as ``Bearer `` — and - # the best-effort JWT decode below must not see a non-string. A missing - # required kind then raises the clear terminal error (see _select). - access_token = _str_or_none(body.get('access_token')) - id_token = _str_or_none(body.get('id_token')) + # Coerce the credential fields up front: a non-string token from a + # buggy/hostile IdP must read as absent rather than be stored, re-sent on + # a refresh, or emitted as ``Bearer `` — and the best-effort JWT + # decode below must not see a non-string. The wire-bound access/id tokens + # additionally go through _safe_token_or_none, which also drops a token + # carrying a control/non-ASCII char (a header / _sso-password injection + # vector): the IdP is untrusted, so the network path applies the same + # screen the persistence path already does. A dropped required kind then + # raises the clear terminal error (see _select). The refresh token is only + # ever re-sent url-encoded to the IdP (never onto a header), so it keeps + # the plain str-or-None coercion, matching _tokenset_from_persisted. + access_token = _safe_token_or_none(body.get('access_token')) + id_token = _safe_token_or_none(body.get('id_token')) refresh_token = _str_or_none(body.get('refresh_token')) claims = (_decode_jwt_claims(id_token) or _decode_jwt_claims(access_token)) diff --git a/src/questdb/auth/_store.py b/src/questdb/auth/_store.py index 34c7a7c8..0e7e0f8a 100644 --- a/src/questdb/auth/_store.py +++ b/src/questdb/auth/_store.py @@ -182,11 +182,18 @@ def _seconds_to_millis(value: Any) -> int: def _canonical_endpoint(url: str) -> str: """Canonicalise an endpoint URL for the cross-language store-key hash. - ``scheme://host:port/path`` with the scheme and host lower-cased, the port - always explicit (the device-flow default 443/80 when absent), and the parsed - path (defaulting to ``/``). A stable rendering that hashes to the same - :class:`TokenStoreKey` across processes and language clients sharing this - identity. Mirrors the Java client's ``canonicalEndpoint``. + ``scheme://host:port/path?query`` with the scheme and host lower-cased, the + port always explicit (the device-flow default 443/80 when absent), a trailing + slash stripped from the path, and the query preserved. A stable rendering that + hashes to the same :class:`TokenStoreKey` across processes and language + clients sharing this identity, and — crucially — that makes the SAME identity + *distinctions* as the in-memory :attr:`OidcDeviceAuth.cache_key` + (``_normalize_url`` + ``_normalize_scope``), so a token is never keyed one way + in memory and a different way on disk. Mirrors the Java client's + ``canonicalEndpoint``; keep the two in step (the on-disk hash is a + cross-language contract). For the common case — no trailing slash, no query — + this rendering is byte-for-byte unchanged, so cross-language sharing is + unaffected there. """ parts, explicit_port = safe_urlparse(url) scheme = (parts.scheme or '').lower() @@ -200,8 +207,19 @@ def _canonical_endpoint(url: str) -> str: host = f'[{host}]' default_port = {'https': 443, 'http': 80}.get(scheme) port = explicit_port if explicit_port is not None else default_port - path = parts.path or '/' - return f'{scheme}://{host}:{port}{path}' + # Strip a trailing slash (keeping at least '/'), so '…/token' and '…/token/' + # are ONE identity — matching cache_key, which rstrip('/')s. Without this the + # disk key splits one identity across two files on a trailing-slash spelling + # difference, forcing a needless re-prompt after a restart. + path = (parts.path or '/').rstrip('/') or '/' + # Keep the query: a token endpoint that differs only by query string is a + # different credential-routing target, so it must hash to a DIFFERENT file — + # matching cache_key, which keeps the query. Dropping it (the old behaviour) + # collided two distinct identities onto one file; _parse_and_verify then + # compared only the query-stripped endpoint, so it couldn't tell them apart + # and could serve one identity's token to the other. + query = f'?{parts.query}' if parts.query else '' + return f'{scheme}://{host}:{port}{path}{query}' @dataclass(frozen=True) @@ -228,11 +246,15 @@ class PersistedToken: class TokenStoreKey: """The non-secret identity a persisted token belongs to. - The client id, the canonicalised token and device-authorization endpoints, - the scope, the optional audience, and whether the server expects groups - encoded in the token. A :class:`TokenStore` keys its entries by this so a - token minted for one server / identity provider / scope / audience is never - served to a process configured for another. + The client id, the canonicalised token and device-authorization endpoints + (see :func:`_canonical_endpoint`), the order-normalised scope (the + space-joined sorted token set), the optional audience, and whether the server + expects groups encoded in the token. A :class:`TokenStore` keys its entries by + this so a token minted for one server / identity provider / scope / audience + is never served to a process configured for another. The endpoint and scope + fields must be passed already normalised — exactly as + :class:`~questdb.auth.OidcDeviceAuth` builds them — so a directly-constructed + key matches the same identity the auth object computes. :meth:`hash` is a stable lowercase-hex SHA-256 over a canonical, NUL-separated rendering of the fields — a file name (or opaque key) that is diff --git a/test/test_auth.py b/test/test_auth.py index 05bc2ec0..bc4c9d2e 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -990,6 +990,35 @@ def test_200_without_access_token_is_not_success(self): with self.assertRaises(OidcDeviceFlowError): auth.token() + def test_groups_mode_rejects_control_char_in_network_id_token(self): + # M3: a token straight from the (untrusted) IdP token endpoint is screened + # for control / non-ASCII chars exactly like a file-loaded one — a decoded + # CR/LF in the served token is an Authorization-header / _sso-password + # injection vector. groups mode serves the id_token, so a control char + # there must fail the grant terminally and cache nothing, not route a + # tampered credential onto the wire. (The persistence path already screens + # this; the network path must too, since the IdP is equally untrusted.) + self.state.token_script = [(200, { + 'access_token': ACCESS_TOKEN, 'id_token': 'bad\r\nid-token', + 'refresh_token': 'REFRESH-1', 'token_type': 'Bearer', + 'expires_in': 3600})] + auth = self.make_auth(groups_in_token=True) + with self.assertRaises(OidcDeviceFlowError): + auth.token() + self.assertIsNone(auth._tokens) # nothing cached + + def test_access_mode_rejects_control_char_in_network_access_token(self): + # M3, the OTHER served kind: with groups not in the token, token() serves + # the access_token, so a control char there must reject the grant too. + self.state.token_script = [(200, { + 'access_token': 'bad\x00access', 'id_token': ID_TOKEN, + 'refresh_token': 'REFRESH-1', 'token_type': 'Bearer', + 'expires_in': 3600})] + auth = self.make_auth(groups_in_token=False) + with self.assertRaises(OidcDeviceFlowError): + auth.token() + self.assertIsNone(auth._tokens) + def test_access_token_headers(self): auth = self.make_auth(groups_in_token=False) self.assertEqual(auth.headers(), @@ -3491,6 +3520,40 @@ def test_groups_in_token_distinguishes_key(self): self._auth(groups_in_token=True).cache_key, self._auth(groups_in_token=False).cache_key) + def test_in_memory_and_on_disk_keys_agree_on_identity(self): + # Regression for M2: the in-memory cache_key and the on-disk + # TokenStoreKey.hash() must make the SAME identity distinctions, or a + # token cached under one key in memory could be served from a different + # one on disk (a wrong-identity serve), or a single identity could split + # across two store files (a needless re-prompt after restart). Check the + # three axes that used to diverge. + store = FileTokenStore.at(tempfile.mkdtemp()) + + def keys(scope='openid', token_ep='https://idp.example.com/token'): + a = OidcDeviceAuth( + client_id='c', token_endpoint=token_ep, + device_authorization_endpoint='https://idp.example.com/device', + scope=scope, token_store=store) + return a.cache_key, a._store_key.hash() + + # (b) scope ORDER — the same identity on BOTH sides. + (ck1, sk1) = keys(scope='openid groups') + (ck2, sk2) = keys(scope='groups openid') + self.assertEqual(ck1, ck2) + self.assertEqual(sk1, sk2) + # (c) trailing SLASH — the same identity on BOTH sides. + (ck3, sk3) = keys(token_ep='https://idp.example.com/token') + (ck4, sk4) = keys(token_ep='https://idp.example.com/token/') + self.assertEqual(ck3, ck4) + self.assertEqual(sk3, sk4) + # (a) token-endpoint QUERY — a DIFFERENT identity on BOTH sides, so two + # query-distinguished tenants never collide onto one store file (which + # would serve one tenant's token to the other). + (ck5, sk5) = keys(token_ep='https://idp.example.com/token?tenant=a') + (ck6, sk6) = keys(token_ep='https://idp.example.com/token?tenant=b') + self.assertNotEqual(ck5, ck6) + self.assertNotEqual(sk5, sk6) + class TestTransportSecurity(unittest.TestCase): def test_require_secure_policy(self): @@ -5595,6 +5658,66 @@ def test_in_lock_failure_degrades_refresh_to_lock_free(self): self.assertEqual(self.state.device_requests, 1) # only the sign-in self.assertIn('token store', err.getvalue()) + def test_coordinated_refresh_uses_in_memory_token_when_newer_than_disk(self): + # M4 / _refresh_under_lock: when a prior save FAILED, the in-memory + # refresh token is NEWER than the last-persisted one. A coordinated + # refresh must then refresh with the in-memory token and must NOT re-read + # the store and regress to the stale on-disk token (which, on a rotating + # IdP, may already be revoked). Exercises the + # `refresh_token != _last_persisted_refresh_token` branch. + store = _FakeStore() + clock = FakeClock() + auth = self.make_auth(token_store=store, clock=clock) + auth.token() # sign in: in-mem & disk RT-1 + self.assertEqual(store.saved.refresh_token, 'REFRESH-1') + + # First refresh rotates to REFRESH-2, but the save FAILS — so in-memory is + # REFRESH-2 while the store and _last_persisted_refresh_token stay -1. + self.state.refresh_response = (200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'refresh_token': 'REFRESH-2', 'token_type': 'Bearer', + 'expires_in': 3600, 'scope': 'openid groups'}) + store.fail_save = True + clock.wall += 10_000 # expire -> refresh + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(auth._tokens.refresh_token, 'REFRESH-2') # in-mem + self.assertEqual(auth._last_persisted_refresh_token, 'REFRESH-1') # save failed + self.assertEqual(store.saved.refresh_token, 'REFRESH-1') # disk stale + + # Second refresh: in-memory (-2) != last-persisted (-1), so the coordinated + # path skips the under-lock re-read and refreshes with the in-memory -2, + # NOT the stale disk -1. + store.fail_save = False + self.state.refresh_response = (200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'refresh_token': 'REFRESH-3', 'token_type': 'Bearer', + 'expires_in': 3600, 'scope': 'openid groups'}) + loads_before = store.loads + clock.wall += 10_000 # expire again -> refresh + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual( + self.state.refresh_forms[-1]['refresh_token'], 'REFRESH-2') + self.assertEqual(store.loads, loads_before) # re-read skipped + self.assertEqual(auth._tokens.refresh_token, 'REFRESH-3') + + def test_clear_in_lock_failure_is_non_fatal(self): + # M4 / clear(): the file-delete runs under the store's cross-process lock. + # If the LOCK BACKEND itself raises (a custom store), clear() must warn and + # carry on — the in-memory cache is still cleared — not propagate. The + # existing clear-failure test covers clear() raising; this covers in_lock + # raising (the lock backend), the other guarded site. + store = _FakeStore() + auth = self.make_auth(token_store=store) + auth.token() + store.fail_in_lock = True + err = io.StringIO() + with contextlib.redirect_stderr(err): + auth.clear() # in_lock raises; must not propagate + self.assertIn('token store clear failed', err.getvalue()) + self.assertIsNone(auth._tokens) # in-memory cache cleared regardless + self.assertEqual(store.clears, 0) # clear() never reached (lock failed) + if __name__ == '__main__': unittest.main() From 2a55c5034a26b73284335474e5e5089a1130a56d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 30 Jun 2026 01:41:20 +0100 Subject: [PATCH 090/104] fix(auth): harden numeric, render, and issuer edge cases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four minor hardening fixes: - _seconds_to_millis checked finiteness BEFORE the *1000 scale, so a finite-but-huge value (e.g. 1e306) passed, overflowed to inf when scaled, and int(round(inf)) raised a raw OverflowError — escaping the store's OidcError contract via the public PersistedToken/save. Check finiteness after scaling. - FileTokenStore accepted a non-finite lock_stale / lock_acquire_budget: inf passes the bare `> 0` / `> _MIN_LOCK_STALE` comparisons, and an infinite staleness window means a crashed holder's lock is never reclaimed. Reject non-finite (and a too-large int) up front. - _strip_control kept variation selectors (U+FE00–FE0F, U+E0100–E01EF, category Mn) and enclosing combining marks (category Me), both invisible/overlay characters that reached a user_code / identity / URL path. Strip them, and cap a run of non-spacing marks (Mn) so a "Zalgo" stack can't smear over adjacent prompt lines; legitimate accents and a short combining run still render, and interleaved zero-width chars can't reset the cap. - The direct OidcDeviceAuth(...) constructor validated only the two endpoints, not the issuer (resolve_config does on the from_questdb path), so a confusable or malformed issuer constructed fine and then raised lazily from cache_key on the first token() call. Validate the issuer authority in __init__ so it fails fast at construction. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 18 +++++++++++- src/questdb/auth/_render.py | 57 ++++++++++++++++++++++++++++++------- src/questdb/auth/_store.py | 47 ++++++++++++++++++++++++------ test/test_auth.py | 48 ++++++++++++++++++++++++++++++- 4 files changed, 149 insertions(+), 21 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index e8f022b1..963c6db9 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -38,7 +38,12 @@ from typing import Any, Dict, Optional from ._cache import MemoryCache, TokenSet -from ._discovery import OidcConfig, resolve_config, validate_endpoint_origins +from ._discovery import ( + OidcConfig, + _reject_confusable_authority, + resolve_config, + validate_endpoint_origins, +) from ._errors import ( OidcConfigError, OidcDeviceFlowError, @@ -435,6 +440,17 @@ def __init__( validate_endpoint_origins( self.config.token_endpoint, self.config.device_authorization_endpoint) + # Vet the issuer authority here too. resolve_config does this on the + # from_questdb path (before it drives discovery); doing it in __init__ as + # well means the DIRECT constructor also fails fast at construction — like + # the endpoints do — for a confusable or malformed issuer (e.g. a bad + # port), instead of raising lazily from `cache_key` (which parses the + # issuer) on the first token() call. On the direct path the issuer feeds + # only cache bucketing, never credential routing, so this is fail-fast + # hygiene rather than a routing fix; it is idempotent with the + # from_questdb check. + if self.config.issuer: + _reject_confusable_authority(self.config.issuer, label='issuer') # `insecure` permits plaintext http only to QuestDB (e.g. local dev). # _idp_post always holds the IdP to https (or loopback http), so the diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index dfa2a420..365782ef 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -309,17 +309,36 @@ def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: # format codepoint is covered automatically, rather than an enumerated regex # that silently misses additions: control (Cc), format (Cf: bidi / zero-width / # soft hyphen / tag chars / the deprecated U+206x), unassigned (Cn), -# private-use (Co), surrogates (Cs) and line/paragraph separators (Zl/Zp). -# The ordinary ASCII space (U+0020, itself category Zs) and combining marks -# (Mn, e.g. accents) are kept so a legitimate identity still renders; every -# OTHER space separator (NBSP U+00A0, ideographic space U+3000, ...) is folded -# to a plain space below, since an invisible-as-space char is a known phishing -# primitive (it can hide trailing text in a user_code / identity / error). -_STRIP_CATEGORIES = frozenset({'Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Zl', 'Zp'}) -# Invisible characters Unicode classifies as letters (category Lo), so the rule -# above won't catch them, but they render as nothing and are used to hide/spoof -# text: the Hangul fillers. Stripped explicitly. -_STRIP_EXTRA = frozenset('\u115f\u1160\u3164\uffa0') +# private-use (Co), surrogates (Cs), line/paragraph separators (Zl/Zp), and +# ENCLOSING combining marks (Me, e.g. U+20E0 / U+0489) — which overlay the +# preceding glyph (a circle/slash/keycap) and are never part of a legitimate +# identity / URL / user_code. +# The ordinary ASCII space (U+0020, itself category Zs), non-enclosing combining +# marks (Mn accents — capped below — and Mc spacing marks, e.g. Indic vowel +# signs) are kept so a legitimate identity still renders; every OTHER space +# separator (NBSP U+00A0, ideographic space U+3000, ...) is folded to a plain +# space below, since an invisible-as-space char is a known phishing primitive +# (it can hide trailing text in a user_code / identity / error). +_STRIP_CATEGORIES = frozenset({'Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Me', 'Zl', 'Zp'}) +# Invisible characters the category rule above does NOT catch, stripped +# explicitly: +# - the Hangul fillers (category Lo) — render as nothing, used to hide/spoof; +# - variation selectors VS1–VS16 (U+FE00–U+FE0F) and the supplement +# (U+E0100–U+E01EF) — category Mn (so the "keep accents" rule below would keep +# them), invisible, and able to carry hidden payload through a user_code / URL +# / identity or flip an adjacent glyph's text/emoji presentation. +_STRIP_EXTRA = frozenset( + '\u115f\u1160\u3164\uffa0' + + ''.join(chr(c) for c in range(0xFE00, 0xFE10)) + + ''.join(chr(c) for c in range(0xE0100, 0xE01F0))) + +# Cap consecutive non-spacing marks (category Mn) kept on one base character. +# Mn marks stack vertically on the preceding glyph; a long run is a "Zalgo" +# overrun that smears across adjacent prompt lines and can obscure the real +# sign-in URL / code. A legitimate accented identity never needs more than a +# couple (Hebrew nikud+cantillation, decomposed Vietnamese ≈ 2), so this is +# generous for real text while neutralising a runaway stack. +_MAX_COMBINING_RUN = 4 def _strip_control(text: Optional[str]) -> str: @@ -342,12 +361,28 @@ def _strip_control(text: Optional[str]) -> str: if not isinstance(text, str): text = str(text) out = [] + combining_run = 0 for ch in text: if ch in _STRIP_EXTRA: + # Stripped chars are transparent to the combining-run count below, so + # an attacker can't reset the cap by interleaving zero-width / + # variation-selector chars between stacked marks. continue category = unicodedata.category(ch) if category in _STRIP_CATEGORIES: continue + if category == 'Mn': + # Non-spacing marks stack on the preceding base; a long run is a + # "Zalgo" overrun that smears across adjacent prompt lines. Keep a + # short legitimate run (accents / diacritics), drop the overflow. + # (Enclosing marks Me are stripped above; variation selectors are in + # _STRIP_EXTRA — neither reaches here.) + combining_run += 1 + if combining_run > _MAX_COMBINING_RUN: + continue + out.append(ch) + continue + combining_run = 0 # Fold an exotic space separator (NBSP, ideographic space, ...) to a # plain ASCII space: it renders invisible-as-space and can hide trailing # text, but the ordinary U+0020 of a legitimate identity must survive. diff --git a/src/questdb/auth/_store.py b/src/questdb/auth/_store.py index 0e7e0f8a..b2d53b45 100644 --- a/src/questdb/auth/_store.py +++ b/src/questdb/auth/_store.py @@ -174,9 +174,32 @@ def _seconds_to_millis(value: Any) -> int: seconds = float(value) except (OverflowError, ValueError): return 0 # e.g. an int too large to convert to float - if not math.isfinite(seconds): + scaled = seconds * 1000.0 + # Check finiteness AFTER the *1000 scale, not before: NaN/±Inf (json.loads + # accepts bare NaN/Infinity) AND a finite-but-huge value that overflows to inf + # only once scaled (e.g. 1e306 * 1000) both land here. Checking `seconds` + # alone would let the latter through, and int(round(inf)) then raises + # OverflowError — escaping the store's OidcError contract (PersistedToken is + # public, so a caller can pass such a value straight to save()). + if not math.isfinite(scaled): return 0 - return int(round(seconds * 1000)) + return int(round(scaled)) + + +def _is_finite_number(value: Any) -> bool: + """True if ``value`` is a finite real number (``int``/``float``). + + A non-numeric type, ``NaN``, ``±Inf``, or an ``int`` too large to convert to + ``float`` (``math.isfinite`` raises ``OverflowError`` on it) all read as + ``False``, so a caller can reject a non-finite duration before it reaches the + lock-timing math. Mirrors ``_device._validate_positive_number``'s handling. + """ + if not isinstance(value, (int, float)): + return False + try: + return math.isfinite(value) + except (OverflowError, ValueError): + return False def _canonical_endpoint(url: str) -> str: @@ -410,16 +433,24 @@ def __init__( """ if not directory: raise OidcConfigError('the token store directory is required') - if not (lock_acquire_budget > 0): + # Require finite, positive timings. inf passes the bare `> 0` / + # `> _MIN_LOCK_STALE` comparisons — and an infinite staleness window means + # a crashed holder's lock is NEVER judged stale, so its identity degrades + # to lock-free coordination forever — so reject non-finite up front + # (nan already fails the comparisons, since `nan > x` is False). The + # finiteness check is first so it short-circuits before comparing a huge + # int (whose `math.isfinite` would itself raise). + if not (_is_finite_number(lock_acquire_budget) and lock_acquire_budget > 0): raise OidcConfigError( - 'the token store lock_acquire_budget must be positive') + 'the token store lock_acquire_budget must be a positive, finite ' + 'number of seconds') # A staleness window at or below the worst-case live hold makes a # freshly acquired lock look abandoned, so acquirers would steal each - # other's LIVE locks. Require it to exceed _MIN_LOCK_STALE (and so, - # transitively, to be positive). - if not (lock_stale > _MIN_LOCK_STALE): + # other's LIVE locks. Require it finite and above _MIN_LOCK_STALE (and so, + # transitively, positive). + if not (_is_finite_number(lock_stale) and lock_stale > _MIN_LOCK_STALE): raise OidcConfigError( - 'the token store lock_stale must exceed ' + 'the token store lock_stale must be a finite number above ' f'{_MIN_LOCK_STALE:g} seconds (the worst-case time a live ' 'holder can hold the lock during a refresh); a shorter window ' "would let a peer steal a live holder's lock mid-refresh.") diff --git a/test/test_auth.py b/test/test_auth.py index bc4c9d2e..e9fe8be9 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -3354,6 +3354,20 @@ def test_confusable_issuer_rejected(self): device_authorization_endpoint='https://idp.good/device', issuer='https://idp.good@attacker.evil') + def test_issuer_validated_in_direct_constructor(self): + # The DIRECT OidcDeviceAuth(...) constructor — not only from_questdb / + # resolve_config — must vet the issuer authority, so a confusable or + # malformed issuer fails fast at construction (like the endpoints) instead + # of lazily from cache_key on the first token() call. + for bad in ('https://idp.good@attacker.evil', # userinfo confusable + 'https://idp.example:99999999999'): # malformed port + with self.assertRaises(OidcConfigError): + OidcDeviceAuth( + client_id='c', + token_endpoint='https://idp.good/token', + device_authorization_endpoint='https://idp.good/device', + scope='openid', issuer=bad) + def test_endpoint_path_under_issuer(self): # M1: segment-aware path containment used to isolate path-based realms. from questdb.auth._discovery import _endpoint_path_under_issuer as under @@ -4541,7 +4555,11 @@ def test_strip_control_removes_bidi_and_zero_width(self): # format chars, the Tags block, unassigned code points, # Arabic format marks and the other invisible Hangul fillers: 0x206a, 0x206f, 0x2065, 0xe0001, 0xe007f, 0x0600, - 0x1160, 0x3164, 0xffa0): + 0x1160, 0x3164, 0xffa0, + # variation selectors (category Mn, invisible) and enclosing + # combining marks (category Me, which overlay the preceding + # glyph) — neither belongs in an identity / URL / user_code: + 0xfe0e, 0xfe0f, 0xe0100, 0x20e0, 0x0489): self.assertEqual(_strip_control('a' + chr(cp) + 'b'), 'ab', f'U+{cp:04X} not stripped') # Legitimate text (incl. accents / CJK / printable ASCII) is preserved. @@ -4552,6 +4570,24 @@ def test_strip_control_removes_bidi_and_zero_width(self): self.assertNotIn(chr(0x202e), text) self.assertIn('idp.example.com', text) + def test_strip_control_caps_combining_run(self): + # A "Zalgo" stack — many non-spacing marks (Mn) on one base — smears over + # adjacent prompt lines and can obscure the sign-in URL/code. _strip_control + # keeps a short legitimate run and drops the overflow, while a normally + # accented identity (a mark or two) is preserved untouched. + from questdb.auth._render import _strip_control, _MAX_COMBINING_RUN + acute = chr(0x0301) # COMBINING ACUTE ACCENT (category Mn) + self.assertEqual( + _strip_control('a' + acute * 50 + 'b'), + 'a' + acute * _MAX_COMBINING_RUN + 'b') + # Interleaving zero-width chars must NOT reset the cap (a stripped char is + # transparent to the run), so the overflow is still dropped. + out = _strip_control('a' + (acute + chr(0x200b)) * 50 + 'b') + self.assertEqual(out.count(acute), _MAX_COMBINING_RUN) + # A legitimately accented identity is untouched. + self.assertEqual(_strip_control('e' + acute), 'e' + acute) + self.assertEqual(_strip_control('café 北京'), 'café 北京') + def test_strip_control_folds_exotic_whitespace_to_ascii_space(self): # An invisible-as-space separator (NBSP, ideographic space, ...) is a # phishing primitive: it can pad a user_code / identity / error to hide @@ -4971,6 +5007,13 @@ def test_constructor_validates_args(self): with self.assertRaises(OidcConfigError): FileTokenStore(self.dir, lock_stale=_MIN_LOCK_STALE) FileTokenStore(self.dir, lock_stale=_MIN_LOCK_STALE + 1) # ok + # Non-finite timings slip the bare `> 0` / `> _MIN_LOCK_STALE` comparisons + # (inf > x is True) — and an infinite stale window means a crashed + # holder's lock is never reclaimed — so they must be rejected too. + with self.assertRaises(OidcConfigError): + FileTokenStore(self.dir, lock_acquire_budget=float('inf')) + with self.assertRaises(OidcConfigError): + FileTokenStore(self.dir, lock_stale=float('inf')) def test_at_default_location_honours_env(self): with mock.patch.dict(os.environ, {TOKEN_STORE_DIR_ENV: self.dir}): @@ -5214,6 +5257,9 @@ def test_seconds_to_millis_maps_non_finite_to_zero(self): self.assertEqual(_seconds_to_millis(float('inf')), 0) self.assertEqual(_seconds_to_millis(float('-inf')), 0) self.assertEqual(_seconds_to_millis(float('nan')), 0) + # Finite, but 1e306 * 1000 overflows to inf: the finiteness check must run + # AFTER the scale, else int(round(inf)) raises a raw OverflowError. + self.assertEqual(_seconds_to_millis(1e306), 0) self.assertEqual(_seconds_to_millis('x'), 0) self.assertEqual(_seconds_to_millis(True), 0) From b0d2ce81f6bddbd723eced0c850b2dd0c58df390 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 30 Jun 2026 10:45:38 +0100 Subject: [PATCH 091/104] fix(auth): align in-memory and on-disk token keys The in-memory cache_key rstrip('/')-ed the whole rendered URL, while the on-disk _canonical_endpoint strips the trailing slash on the path component. That whole-string strip disagreed with the store on two endpoint shapes: a slash before a query ('.../token/?x' stayed split from '.../token?x'), and a slash that is part of a query value ('...?redirect=a/' wrongly collided with '...?redirect=a'). Either case keyed one identity two ways in memory vs on disk. Move the trailing-slash stripping into _normalize_url, applied to the path component (mirroring _canonical_endpoint), and drop the now-redundant cache_key rstrip. The two keys now make identical identity distinctions. Also add the missing TLS-verification coverage: a unit assertion that build_ssl_context() enforces CERT_REQUIRED + check_hostname (so a regression swapping in an unverified context fails a test), plus a behavioural handshake test that the default context rejects an untrusted self-signed cert while a custom-CA context still verifies and connects. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 27 ++++++-- test/test_auth.py | 126 ++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 7 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 963c6db9..87596e75 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -649,16 +649,19 @@ def cache_key(self) -> str: c = self.config scope = _normalize_scope(c.scope) # Normalize the issuer and token endpoint alike (lower-case scheme/host, - # drop a default port) and strip a trailing slash, so a discovered + # drop a default port, strip a trailing path slash), so a discovered # "https://idp/token/" and an explicit "https://idp/token" — or a stray # :443 / case difference — don't yield different keys and force an - # avoidable re-prompt. The path is otherwise kept (multi-tenant realms - # differ by it); only a trailing slash, which never distinguishes an - # endpoint, is dropped. - issuer = _normalize_url(c.issuer).rstrip('/') if c.issuer else '' + # avoidable re-prompt. The trailing-slash stripping happens inside + # _normalize_url, on the PATH component, so it is correct even when a + # query follows ("…/token/?x") and never touches a slash inside a query + # value ("…?redirect=a/") — keeping these distinctions identical to the + # on-disk _canonical_endpoint. The path is otherwise kept (multi-tenant + # realms differ by it). + issuer = _normalize_url(c.issuer) if c.issuer else '' return '\x1f'.join([ issuer, - _normalize_url(c.token_endpoint).rstrip('/'), + _normalize_url(c.token_endpoint), c.client_id, scope, c.audience or '', @@ -1588,5 +1591,15 @@ def _normalize_url(url: str) -> str: netloc = f'{host}:{port}' else: netloc = host + # Strip a trailing slash from the PATH COMPONENT (not the rendered URL), so + # '…/token' and '…/token/' are one identity whether or not a query follows — + # exactly as the on-disk _canonical_endpoint does, keeping the in-memory + # cache_key and the on-disk store key in agreement on identity. rstrip('/')-ing + # the whole rendered string (what cache_key used to do) both missed a slash + # hidden before a query ('…/token/?x' stayed split from '…/token?x') and could + # strip a slash that is part of a query VALUE ('…?redirect=a/' wrongly collided + # with '…?redirect=a'); the store keeps the query verbatim, so either case made + # the two keys disagree. Doing it on the path component alone fixes both. + path = (parts.path or '').rstrip('/') query = f'?{parts.query}' if parts.query else '' - return f'{scheme}://{netloc}{parts.path}{query}' + return f'{scheme}://{netloc}{path}{query}' diff --git a/test/test_auth.py b/test/test_auth.py index e9fe8be9..f889ca99 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -3567,6 +3567,22 @@ def keys(scope='openid', token_ep='https://idp.example.com/token'): (ck6, sk6) = keys(token_ep='https://idp.example.com/token?tenant=b') self.assertNotEqual(ck5, ck6) self.assertNotEqual(sk5, sk6) + # (d) trailing SLASH *and* a query together (regression for M1): the old + # cache_key rstrip('/')-ed the whole rendered URL, so the slash hidden + # before the query survived in memory ('…/token/?t' stayed split from + # '…/token?t') while the store stripped it on the path and kept them one + # — the two keys disagreed. They must agree: same identity on BOTH sides. + (ck7, sk7) = keys(token_ep='https://idp.example.com/token?tenant=a') + (ck8, sk8) = keys(token_ep='https://idp.example.com/token/?tenant=a') + self.assertEqual(ck7, ck8) + self.assertEqual(sk7, sk8) + # (e) a slash that is part of a query VALUE must NOT be stripped (the old + # whole-string rstrip could chop it), so two distinct query values stay a + # different identity on BOTH sides. + (ck9, sk9) = keys(token_ep='https://idp.example.com/token?redirect=a/') + (ck10, sk10) = keys(token_ep='https://idp.example.com/token?redirect=a') + self.assertNotEqual(ck9, ck10) + self.assertNotEqual(sk9, sk10) class TestTransportSecurity(unittest.TestCase): @@ -4044,6 +4060,116 @@ def test_bad_ca_bundle_raises_config_error(self): finally: os.unlink(bad) + def test_default_context_verifies_certificates(self): + # The single most load-bearing security default: every IdP credential + # POST (device-code, each poll, refresh) rides build_ssl_context(). It + # MUST verify the server certificate and check the hostname. A regression + # swapping in ssl._create_unverified_context(), or setting + # check_hostname=False / verify_mode=CERT_NONE, would silently expose + # every device-code and long-lived refresh-token POST to a MITM while + # breaking no other test. Assert the posture directly so such a + # regression fails here. + import ssl + from questdb.auth._http import build_ssl_context + ctx = build_ssl_context() # no-arg: the production default path + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + + def test_untrusted_server_certificate_is_rejected(self): + # Behavioural companion to the unit assertion above: a real TLS handshake + # against a server presenting a cert the default trust store does not + # trust (a self-signed cert — what a MITM would present) MUST fail, and a + # custom CA context built to trust that cert MUST still verify + connect. + # Together these catch a verification regression that the configuration + # assertion alone could miss (e.g. a verify that is silently bypassed on + # the request path). The cert is generated fresh at runtime, so there is + # no embedded-cert expiry to rot the test. + try: + from datetime import datetime, timedelta, timezone + from cryptography import x509 + from cryptography.x509.oid import NameOID + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + except ImportError: + self.skipTest('cryptography not installed (handshake test skipped; ' + 'test_default_context_verifies_certificates still ' + 'guards the verification posture)') + import http.server + import ipaddress + import ssl + import threading + from questdb.auth._http import build_ssl_context, get_json + + tmp = tempfile.mkdtemp() + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name = x509.Name( + [x509.NameAttribute(NameOID.COMMON_NAME, 'localhost')]) + now = datetime.now(timezone.utc) + cert = (x509.CertificateBuilder() + .subject_name(name).issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(days=1)) + .not_valid_after(now + timedelta(days=3650)) + .add_extension( + x509.SubjectAlternativeName([ + x509.DNSName('localhost'), + x509.IPAddress(ipaddress.ip_address('127.0.0.1'))]), + critical=False) + .sign(key, hashes.SHA256())) + certfile = os.path.join(tmp, 'cert.pem') + with open(certfile, 'wb') as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + with open(os.path.join(tmp, 'key.pem'), 'wb') as f: + f.write(key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption())) + keyfile = os.path.join(tmp, 'key.pem') + + class _Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body = b'{"ok": true}' + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *a): + pass + + class _QuietTLSServer(http.server.HTTPServer): + # A rejected handshake (the negative case below) is expected; don't + # let its traceback spam the test output. + def handle_error(self, request, client_address): + pass + + sctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + sctx.load_cert_chain(certfile, keyfile) + httpd = _QuietTLSServer(('127.0.0.1', 0), _Handler) + httpd.socket = sctx.wrap_socket(httpd.socket, server_side=True) + port = httpd.server_address[1] + t = threading.Thread(target=httpd.serve_forever, daemon=True) + t.start() + try: + url = f'https://localhost:{port}/' + # Production default context trusts only system roots -> the + # self-signed cert fails verification -> typed network error. + with self.assertRaises(OidcNetworkError): + get_json(url, timeout=5) + # A context that DOES trust the cert must still verify + check + # hostname (never relaxed by adding a CA), and connect cleanly. + trusting = build_ssl_context(ca_bundle=certfile) + self.assertEqual(trusting.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(trusting.check_hostname) + self.assertEqual(get_json(url, ctx=trusting, timeout=5), {'ok': True}) + finally: + httpd.shutdown() + t.join(10) + self.assertFalse(t.is_alive(), 'TLS test server thread did not stop') + httpd.server_close() + def test_deeply_nested_json_raises_oidc_error(self): # A RecursionError from json.loads (a deeply-nested JSON body exhausts # the decoder's stack) must be mapped to OidcError, not escape the From e4744e47f49e9dcab2984f1fb827a44940c7532c Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 30 Jun 2026 15:44:05 +0100 Subject: [PATCH 092/104] fix(auth): isolate persisted tokens by issuer Address moderate findings from the PR #133 review: - Persist and re-check `issuer` in the token store so a token pinned to one issuer is never served from disk to a session pinned to another. Kept out of the file-name hash, so the cross-language file contract and existing token files stay byte-stable; isolation is enforced via the on-load identity re-check, matching the in-memory cache_key. - FileTokenStore.load() now returns None (not raises) for an unreadable entry: a directory/symlink-loop at the token path (stat.S_ISREG/errno guard) and a deeply-nested attacker file (RecursionError), restoring its documented contract and matching every other json.loads site. - _display_url escapes a host that still carries a URL-structural char after IDNA (e.g. the U+FF3C/U+FE68 reverse-solidus fold) as a visible \uXXXX, instead of a bare backslash a browser would read as a slash. - CHANGELOG: relabel the Python 3.8->3.10 bump as a breaking change with the whole-package blast radius, EOL rationale, and a migration note. Adds regression tests for each; full auth suite passes (308 tests). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.rst | 17 ++++--- src/questdb/auth/_device.py | 10 +++- src/questdb/auth/_render.py | 15 ++++++ src/questdb/auth/_store.py | 95 ++++++++++++++++++++++++++++++++----- test/test_auth.py | 88 ++++++++++++++++++++++++++++++++++ 5 files changed, 206 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 23d76eb4..3fe85d6e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -63,12 +63,17 @@ Highlights: See the :ref:`OIDC authentication guide ` for details. -Python Version Support -~~~~~~~~~~~~~~~~~~~~~~~~ - -* Corrected the minimum supported Python declared in ``setup.py`` to 3.10, - matching the floor already adopted in 4.1.0 (which dropped Python 3.9). - Installs on Python 3.8 / 3.9 are now correctly rejected. +Breaking Changes +~~~~~~~~~~~~~~~~~ + +* The minimum supported Python is now **3.10**, raised from 3.8 in ``setup.py``. + Python 3.8 (end-of-life 2024-10) and 3.9 (end-of-life 2025-10) are no longer + supported, and ``pip`` will refuse to install this release on them. This + affects the whole ``questdb`` package, not only the new :mod:`questdb.auth` + module. This is the first release to enforce the floor: although the 4.1.0 + changelog described dropping older versions, its ``setup.py`` still declared + ``>=3.8``, so 3.8 / 3.9 installs were not actually rejected until now. Users + still on 3.8 / 3.9 should pin to an earlier ``questdb`` release. 4.1.0 (2025-11-28) ------------------ diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 87596e75..ff6af890 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -503,7 +503,15 @@ def __init__( # cache_key's _normalize_url renders them. scope=_normalize_scope(self.config.scope), audience=self.config.audience, - groups_in_token=self.config.groups_in_token) + groups_in_token=self.config.groups_in_token, + # Normalise the issuer exactly as cache_key does (_normalize_url), + # so the in-memory and on-disk identities agree on the issuer + # axis; None when unpinned. It feeds the on-load identity re-check + # (TokenStoreKey carries it for _parse_and_verify) but NOT the + # file-name hash, so a token pinned to one issuer is never served + # from disk to a session pinned to another. + issuer=(_normalize_url(self.config.issuer) + if self.config.issuer else None)) # Load the persisted entry at most once per instance (even if it yields # nothing), so a missing or bad file is not re-read on every call. self._store_load_attempted = False diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 365782ef..a3ba5b47 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -269,6 +269,21 @@ def _display_url(url: Optional[str]) -> str: # IDNA can't encode it (an illegal label); make the bytes visible # rather than let an invisible homoglyph through unchanged. ascii_host = _ascii_visible(host) + # A host that still carries a URL-structural character after normalization — + # backslash, slash, '@', '?' or '#' — is ambiguous: RFC 3986 keeps it in the + # authority while a WHATWG/browser parser folds '\' to '/' and ends the host + # early, so the host shown here would not be the one a browser resolves (the + # very divergence this function exists to close). The IDNA fold can MINT one: + # a fullwidth reverse solidus U+FF3C (or small reverse solidus U+FE68) is + # category Po, so it survives _strip_control, passes urlparse (whose NFKC + # delimiter-reject covers '/ @ :' but not '\'), and nameprep folds it to a + # literal '\'. Don't render a clean-looking but ambiguous URL — fall back to + # the control-stripped text with every non-ASCII char escaped to a visible + # \uXXXX, so the confusable is shown as e.g. '＼', not as a bare '\'. + # (':' is excluded: an IPv6 literal legitimately carries it and is bracketed + # just below.) + if any(c in ascii_host for c in '\\/@?#'): + return _ascii_visible(text) host_part = f'[{ascii_host}]' if ':' in ascii_host else ascii_host # IPv6 # Read the port separately and defensively: parts.port raises ValueError for # a malformed (non-integer / out-of-range) port. That must NOT abort host diff --git a/src/questdb/auth/_store.py b/src/questdb/auth/_store.py index b2d53b45..39e66274 100644 --- a/src/questdb/auth/_store.py +++ b/src/questdb/auth/_store.py @@ -46,11 +46,13 @@ import abc import contextlib +import errno import hashlib import json import math import os import socket +import stat import sys import tempfile import threading @@ -271,13 +273,14 @@ class TokenStoreKey: The client id, the canonicalised token and device-authorization endpoints (see :func:`_canonical_endpoint`), the order-normalised scope (the - space-joined sorted token set), the optional audience, and whether the server - expects groups encoded in the token. A :class:`TokenStore` keys its entries by - this so a token minted for one server / identity provider / scope / audience - is never served to a process configured for another. The endpoint and scope - fields must be passed already normalised — exactly as - :class:`~questdb.auth.OidcDeviceAuth` builds them — so a directly-constructed - key matches the same identity the auth object computes. + space-joined sorted token set), the optional audience, whether the server + expects groups encoded in the token, and the optional out-of-band issuer pin. + A :class:`TokenStore` keys its entries by this so a token minted for one + server / identity provider / scope / audience is never served to a process + configured for another. The endpoint, scope and issuer fields must be passed + already normalised — exactly as :class:`~questdb.auth.OidcDeviceAuth` builds + them — so a directly-constructed key matches the same identity the auth + object computes. :meth:`hash` is a stable lowercase-hex SHA-256 over a canonical, NUL-separated rendering of the fields — a file name (or opaque key) that is @@ -286,6 +289,16 @@ class TokenStoreKey: persisted entry. The fields are exposed (they are not secret) so a store can record and re-check them on load as a defence against a hash collision or a copied file. + + ``issuer`` participates in that on-load identity re-check (see + :meth:`FileTokenStore.load` / ``_issuer_matches``) but **not** in + :meth:`hash`: it is excluded from the file name so the cross-language + addressing contract — and every existing token file — stays byte-identical, + while a session pinned to a different issuer still never adopts another's + token (two issuer-differing configs share a file but reject each other's + contents on load). This mirrors the in-memory + :attr:`~questdb.auth.OidcDeviceAuth.cache_key`, which also distinguishes the + issuer, so the in-memory and on-disk identities agree on that axis. """ client_id: str @@ -294,12 +307,22 @@ class TokenStoreKey: scope: str audience: Optional[str] groups_in_token: bool + # Optional out-of-band issuer pin. Default keeps existing positional + # construction (and the frozen cross-language file-name hash) unchanged. + issuer: Optional[str] = None def hash(self) -> str: """A stable lowercase-hex SHA-256 of the canonical identity string.""" # NUL-separate the fields so no field value can be confused with a # separator (an OAuth client id, url, scope or audience never contains a # NUL). The prefix tags the domain and schema version. + # + # `issuer` is deliberately NOT folded in here: the file name is a frozen + # cross-language contract (the Java client mirrors it), so adding a field + # would change every entry's hash and break addressing. Issuer isolation + # is enforced on load instead (the in-file fingerprint re-check), which + # keeps the file name stable while still never serving one issuer's token + # to a session pinned to another. canonical = '\x00'.join(( _CANONICAL_PREFIX, self.client_id or '', @@ -487,21 +510,39 @@ def at_default_location(cls) -> 'FileTokenStore': def load(self, key: TokenStoreKey) -> Optional[PersistedToken]: path = self._token_file(key) try: - size = os.stat(path).st_size + st = os.stat(path) except FileNotFoundError: return None except OSError as e: + # A path that is not a usable regular file — a symlink loop (ELOOP) + # or a non-directory path component (ENOTDIR) — is "no usable + # entry", not a fatal error: per load()'s contract, fall back to a + # refresh / interactive sign-in rather than raise. A genuine I/O or + # permission error (EACCES, EIO, ...) still surfaces as OidcError. + if e.errno in (errno.ELOOP, errno.ENOTDIR): + return None raise OidcError( f'could not read the OIDC token store file: {e}') from e + # A directory (or other non-regular file) planted at the token path — + # by another tool, or a hostile co-tenant with write access to the store + # dir — is not a usable entry. Ignore it rather than fall through to + # open(), which would raise IsADirectoryError and escape load()'s "an + # unreadable entry returns None, not a fatal error" contract. + if not stat.S_ISREG(st.st_mode): + return None # An empty or implausibly large file is not a usable entry; ignore it # rather than read it into memory. - if size <= 0 or size > _MAX_FILE_BYTES: + if st.st_size <= 0 or st.st_size > _MAX_FILE_BYTES: return None try: with open(path, 'rb') as f: data = f.read(_MAX_FILE_BYTES + 1) except FileNotFoundError: return None + except IsADirectoryError: + # The regular file became a directory between the stat above and + # this open (a TOCTOU); treat it as no usable entry, as above. + return None except OSError as e: raise OidcError( f'could not read the OIDC token store file: {e}') from e @@ -638,6 +679,14 @@ def _serialize(self, key: TokenStoreKey, token: PersistedToken) -> bytes: } if key.audience is not None: obj['audience'] = key.audience + # Persisted for the on-load identity re-check, not the file name (issuer + # is excluded from TokenStoreKey.hash to keep the cross-language file + # contract stable). Omitted when absent, like audience, so an entry + # written without an issuer pin reads back as None and matches a None + # key issuer — and an older/other client that doesn't write the field + # interoperates unchanged. + if key.issuer is not None: + obj['issuer'] = key.issuer obj['groups_in_token'] = key.groups_in_token if token.access_token is not None: obj['access_token'] = token.access_token @@ -653,9 +702,14 @@ def _parse_and_verify( self, key: TokenStoreKey, data: bytes) -> Optional[PersistedToken]: try: obj = json.loads(data) - except (ValueError, UnicodeDecodeError): - # Corrupt or truncated file: treat as no usable entry, fall back to - # refresh / interactive. + except (ValueError, UnicodeDecodeError, RecursionError): + # Corrupt, truncated, or deeply-nested file: treat as no usable + # entry, fall back to refresh / interactive. RecursionError (the + # attacker-writable file nests JSON deep enough to exhaust the + # decoder's stack) is not a ValueError, so list it explicitly — + # matching every other json.loads on untrusted input in this client + # (_decode_jwt_claims, _http.get_json / post_form) — so a hostile + # file makes load() return None rather than crash the caller. return None if not isinstance(obj, dict): return None @@ -670,6 +724,7 @@ def _parse_and_verify( != key.device_authorization_endpoint or obj.get('scope') != key.scope or not _audience_matches(key.audience, obj.get('audience')) + or not _issuer_matches(key.issuer, obj.get('issuer')) or bool(obj.get('groups_in_token')) != key.groups_in_token): return None return PersistedToken( @@ -775,3 +830,19 @@ def _audience_matches(key_audience: Optional[str], file_audience: Any) -> bool: if key_audience is None: return file_audience is None return isinstance(file_audience, str) and file_audience == key_audience + + +def _issuer_matches(key_issuer: Optional[str], file_issuer: Any) -> bool: + # Same contract as _audience_matches, for the out-of-band issuer pin. The + # file omits a null issuer entirely, so an absent (or hand-edited JSON null) + # field reads as None and matches a None key issuer; a present issuer must be + # an exact (already-normalised) string match, and a non-string file value + # never matches. This is the on-load half of issuer isolation: issuer is part + # of the identity re-check but NOT the file-name hash (see + # TokenStoreKey.hash), so two configs differing only by issuer pin share a + # file yet never adopt each other's token — a session pinned to one issuer + # rejects a token persisted under another (and an un-pinned session rejects + # an issuer-pinned token, and vice versa). + if key_issuer is None: + return file_issuer is None + return isinstance(file_issuer, str) and file_issuer == key_issuer diff --git a/test/test_auth.py b/test/test_auth.py index f889ca99..7f3750f6 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -3525,6 +3525,29 @@ def test_issuer_realm_path_distinguishes_key(self): self._auth( issuer='https://idp.example.com/realms/staging').cache_key) + def test_store_key_isolates_issuer_by_fingerprint_not_hash(self): + # M1: two sessions differing ONLY by issuer pin. The in-memory cache_key + # distinguishes them, and so does the on-disk identity -- but via the + # in-file issuer fingerprint, NOT the file-name hash (a frozen + # cross-language contract kept byte-stable). So they address the SAME + # file yet carry distinct issuer fingerprints and never adopt each + # other's token. The issuer is normalized identically on both sides + # (_normalize_url), so memory and disk agree on the issuer axis. + store = FileTokenStore.at(tempfile.mkdtemp()) + a = self._auth( + issuer='https://idp.example.com/realms/a', token_store=store) + b = self._auth( + issuer='https://idp.example.com/realms/b', token_store=store) + self.assertNotEqual(a.cache_key, b.cache_key) # memory: distinct + self.assertEqual(a._store_key.hash(), b._store_key.hash()) # same file name + self.assertNotEqual( # distinct identity + a._store_key.issuer, b._store_key.issuer) + # Issuer spelling that doesn't change the security context (trailing + # slash / case / default port) must NOT split the on-disk identity + # either, exactly as it doesn't split cache_key. + c = self._auth(issuer='https://IDP.example.com/realms/a/', token_store=store) + self.assertEqual(a._store_key.issuer, c._store_key.issuer) + def test_groups_in_token_distinguishes_key(self): # groups_in_token selects which token kind _select returns, so two # sessions differing ONLY in that mode must not collide on one cache @@ -4897,6 +4920,25 @@ def test_display_url_neutralizes_unparseable_confusable_host(self): self.assertIsNone(_safe_target(raw)) # never clickable / opened self.assertNotIn(' None" contract. + depth = 60_000 + with open(self._file(), 'w') as fh: + fh.write('[' * depth + ']' * depth) + self.assertLess(os.path.getsize(self._file()), 1 << 20) # under the cap + self.assertIsNone(self.store.load(self.key)) + def test_wrong_schema_version_ignored(self): with open(self._file(), 'w') as fh: json.dump({'v': 2, 'client_id': 'questdb', @@ -5104,6 +5165,33 @@ def test_audience_in_fingerprint_roundtrips(self): 'openid', 'api://other', False) self.assertIsNone(self.store.load(other)) + def test_issuer_in_fingerprint_not_hash(self): + # M1: issuer participates in the on-load identity re-check but NOT the + # file-name hash, so two issuer-differing configs address the SAME file + # yet never adopt each other's token. (Contrast audience above, which IS + # in the hash, so a different audience is a different FILE.) This is the + # on-disk half of the issuer isolation the in-memory cache_key enforces. + key_x = TokenStoreKey( + 'questdb', 'https://idp:443/token', 'https://idp:443/device', + 'openid', None, False, issuer='https://idp/realms/x') + key_y = TokenStoreKey( + 'questdb', 'https://idp:443/token', 'https://idp:443/device', + 'openid', None, False, issuer='https://idp/realms/y') + # Same file name (issuer excluded from the hash), incl. the no-issuer key. + self.assertEqual(key_x.hash(), key_y.hash()) + self.assertEqual(key_x.hash(), self.key.hash()) + self.store.save(key_x, self._pt()) + with open(self._file(key_x)) as fh: + self.assertIn('"issuer"', fh.read()) + self.assertIsNotNone(self.store.load(key_x)) # same issuer: served + self.assertIsNone(self.store.load(key_y)) # other issuer: rejected + self.assertIsNone(self.store.load(self.key)) # un-pinned: rejected + # An un-pinned token is likewise not served to a pinned session. + self.store.save(self.key, self._pt()) # overwrite, no issuer field + with open(self._file(self.key)) as fh: + self.assertNotIn('"issuer"', fh.read()) + self.assertIsNone(self.store.load(key_x)) + def test_absent_token_fields_read_back_as_none(self): self.store.save(self.key, self._pt(access_token=None, id_token=None)) got = self.store.load(self.key) From 69b92354a5e4bbde31d8161b76e90df8030671e1 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 1 Jul 2026 02:56:15 +0100 Subject: [PATCH 093/104] fix(auth): harden retry-after, blank tokens, IPv6 keys, and store locks Latest hardening round for the OIDC device-flow module, incorporating a code-review pass over the working tree. Transport (_http.py): - _parse_retry_after: bound the digit-string length before int(). On Python >= 3.10.7 int() raises ValueError past 4300 digits, and this runs inside post_form before its own try/except, so a hostile IdP / on-path proxy could leak a raw ValueError past the module's typed-error contract. Only a short run of ASCII digits is accepted; anything longer reads as absent and the caller's fixed back-off applies. Token lifecycle (_device.py): - Reject a blank (empty or whitespace-only) served token on both the network and persisted paths: a run of spaces passed the printable-ASCII gate and would be cached and sent as "Bearer " instead of failing once with the clear "missing required token" error. - _normalize_url: re-bracket IPv6 literals to match the on-disk _canonical_endpoint, so two distinct endpoints ("[::1]:9000" vs "[::1:9000]") can't collapse to the ambiguous "::1:9000" and share one in-memory cache entry while staying separate on disk. Token store (_store.py): - Refuse a symlinked store-dir leaf (lstat) so the plaintext token files can't be redirected outside the owner-only directory. - Treat a future-dated lock mtime as fresh (never steal a possibly-live lock when our clock reads behind it) and raise the lock-stale floor to 300s for fsync/scheduling headroom above the ~240s network envelope. Tests / example: - Regression coverage for all of the above, including proving the SQLAlchemy per-connection token fetch is non-interactive (raises OidcInteractionRequired from a pool thread rather than prompting). - Example: close the psycopg connection via contextlib.closing (psycopg2 leaves it open on __exit__). Co-Authored-By: Claude Opus 4.8 (1M context) --- examples/oidc_device_auth.py | 9 +- src/questdb/auth/_device.py | 112 ++++++++++++++++- src/questdb/auth/_http.py | 23 +++- src/questdb/auth/_store.py | 92 +++++++++++--- test/test_auth.py | 236 ++++++++++++++++++++++++++++++++++- 5 files changed, 440 insertions(+), 32 deletions(-) diff --git a/examples/oidc_device_auth.py b/examples/oidc_device_auth.py index 354ac1d5..ad8530bb 100644 --- a/examples/oidc_device_auth.py +++ b/examples/oidc_device_auth.py @@ -10,6 +10,7 @@ loop), so it is not part of the automated example suite. """ +import contextlib import sys from questdb.auth import ( @@ -49,8 +50,12 @@ def pg_wire(url: str = QUESTDB_URL): for row in conn.execute(text('SELECT * FROM trades LIMIT 10')): print(row) - # Or a raw psycopg / psycopg2 connection (token captured at connect time): - with psycopg_connect(auth, url) as conn: + # Or a raw psycopg / psycopg2 connection (token captured at connect time). + # contextlib.closing guarantees the connection is closed on BOTH drivers: + # psycopg (v3) closes when its `with` block exits, but psycopg2's connection + # context manager only commits / rolls back the transaction and leaves the + # connection itself open. + with contextlib.closing(psycopg_connect(auth, url)) as conn: with conn.cursor() as cur: cur.execute('SELECT count() FROM trades') print(cur.fetchone()) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index ff6af890..3d002348 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -304,9 +304,15 @@ def _safe_token_or_none(value: Any) -> Optional[str]: (via :func:`_has_only_token_chars`). A dropped token reads as absent, so a missing required kind then raises the clear terminal error (see :meth:`_select`) rather than routing a tampered credential onto the wire. + + A blank token (empty or whitespace-only) also reads as absent: a real OAuth + token is never blank, and a run of spaces would otherwise pass the printable- + ASCII gate, be cached, and go out as ``Bearer `` — defeating the + "fail once with a clear error, don't cache an unusable token" guarantee. """ token = value if isinstance(value, str) else None - if token is not None and not _has_only_token_chars(token): + if token is not None and ( + not token.strip() or not _has_only_token_chars(token)): return None return token @@ -384,6 +390,59 @@ def __init__( timeout: float = 30, token_store: Optional[TokenStore] = None, _clock=None): # injectable time source for testing + """ + Construct with explicit IdP configuration (no server discovery). Prefer + :meth:`from_questdb`, which discovers most of these from a QuestDB server. + + :param client_id: the OAuth client id registered with the identity + provider for the device-authorization grant. + :param device_authorization_endpoint: the IdP's RFC 8628 + device-authorization URL, where the flow starts. + :param token_endpoint: the IdP's OAuth token URL, used to exchange the + device code and to refresh the token. + :param scope: space-separated OAuth scopes to request (default + ``'openid'``); ``'openid'`` is added automatically when + ``groups_in_token`` is set. + :param groups_in_token: ``True`` when the QuestDB server expects the + user's groups encoded in the token + (``acl.oidc.groups.encoded.in.token=true``); selects the ``id_token`` + over the ``access_token`` and forces the ``openid`` scope. + :param audience: the OAuth ``audience`` to request, or ``None`` to omit + it (an empty string is treated as ``None``). + :param issuer: the expected token issuer to pin. On this explicit-config + path it feeds only cache / token-store identity (which sessions may + share a token), not credential routing; it is validated as a + well-formed authority. + :param insecure: allow plaintext ``http`` to the QuestDB server (local + dev only). The IdP is always held to ``https`` (or loopback + ``http``), so the device code and refresh token are never sent in + cleartext even when this is set. + :param ca_bundle: path to a PEM CA bundle used to verify TLS to the IdP + (e.g. a private/corporate CA); also forwarded to a caller wiring the + token into its own transport. ``None`` uses the system trust store. + :param open_browser: attempt to open the verification URL in a browser + (default ``True``); when ``False`` the URL is only printed for the + user to open manually. + :param interactive: force interactive (``True``) or non-interactive + (``False``) mode; ``None`` (default) auto-detects a usable terminal. + A non-interactive context raises rather than starting a device-flow + prompt no one can answer. + :param qr: also render the verification URL as a QR code, convenient for + scanning from a phone when signing in on a headless / remote kernel. + :param renderer: a custom :class:`~questdb.auth.Renderer` for the + device-code prompt (overrides ``qr``). Its callbacks run while the + acquisition lock is held, so they must not call back into this + instance's :meth:`token` / :meth:`clear`. + :param default_interval: fallback poll interval in seconds when the IdP's + device-authorization response does not specify one (default ``5``; + clamped to the RFC 8628 range). + :param timeout: per-request HTTP timeout in seconds for each IdP call + (device-code, each poll, refresh); must be positive and must not + exceed 120s (a larger value is rejected). + :param token_store: an optional :class:`~questdb.auth.TokenStore` to + persist the token across process restarts; ``None`` (default) keeps + tokens in memory only. + """ # Validate types up front so a bad-typed arg raises the module's typed # error, not a bare AttributeError/TypeError surfacing later from # scope.split(), safe_urlparse(), or the cache-key join. @@ -571,6 +630,42 @@ def from_questdb( `) to persist the token so a restarted process resumes from the saved refresh token instead of prompting again; the default is in-memory only. + + :param url: the QuestDB base URL; ``{url}/settings`` is read to discover + the OIDC configuration. + :param client_id: override the discovered OAuth client id. + :param scope: override the discovered scopes (space-separated). + :param audience: override the discovered OAuth ``audience``. + :param groups_in_token: override the discovered groups-in-token mode + (``True`` selects the ``id_token`` and forces the ``openid`` scope). + :param issuer: pin the token issuer. **Required** when the server does + not advertise the device-authorization endpoint (so it is discovered + from the IdP), so a tampered ``/settings`` cannot redirect the + credential POSTs — see above. + :param token_endpoint: override the discovered token endpoint. + :param device_authorization_endpoint: override the discovered + device-authorization endpoint. + :param insecure: allow plaintext ``http`` to the QuestDB server for + discovery (local dev only); the IdP is always held to ``https`` (or + loopback ``http``). + :param ca_bundle: path to a PEM CA bundle used to verify TLS to QuestDB + and the IdP; ``None`` uses the system trust store. + :param open_browser: attempt to open the verification URL in a browser + (default ``True``); when ``False`` it is only printed. + :param interactive: force interactive (``True``) or non-interactive + (``False``) mode; ``None`` (default) auto-detects a usable terminal. + :param qr: also render the verification URL as a QR code. + :param renderer: a custom :class:`~questdb.auth.Renderer` for the + device-code prompt (overrides ``qr``); its callbacks must not + re-enter this instance's :meth:`token` / :meth:`clear`. + :param default_interval: fallback poll interval in seconds when the IdP + does not specify one (default ``5``). + :param timeout: per-request HTTP timeout in seconds for discovery and + every IdP call; must be positive and must not exceed 120s (a larger + value is rejected). + :param token_store: an optional :class:`~questdb.auth.TokenStore` to + persist the token across process restarts; ``None`` (default) keeps + tokens in memory only. """ # Validate before resolve_config consumes `timeout` on its /settings and # discovery HTTP calls (which run before cls() would validate it), so a @@ -1070,7 +1165,11 @@ def _tokenset_from_persisted( id_token = _str_or_none(persisted.id_token) refresh_token = _str_or_none(persisted.refresh_token) served = id_token if self.config.groups_in_token else access_token - if not served or not _has_only_token_chars(served): + # A null, blank (whitespace-only), or control/non-ASCII served token is + # unusable: reject the whole entry rather than serve a tampered or empty + # credential. The blank check mirrors _safe_token_or_none on the wire + # path so a run of spaces can't slip through the printable-ASCII gate. + if not served or not served.strip() or not _has_only_token_chars(served): return None # The file is attacker-writable (and may have been written under a skewed # clock), so bound how long the loaded token is trusted exactly as a @@ -1594,6 +1693,15 @@ def _normalize_url(url: str) -> str: parts, port = safe_urlparse(url) scheme = (parts.scheme or '').lower() host = (parts.hostname or '').lower() + # Re-add the brackets urllib strips off an IPv6 literal, exactly as the + # on-disk _canonical_endpoint does. Without them "::1" + port 9000 renders as + # the ambiguous "::1:9000", which collides with the distinct host "::1:9000" + # (default port): two different IPv6 endpoints would then key to ONE in-memory + # cache entry while the bracketing disk store keeps them apart — the "keyed + # one way in memory, a different way on disk" divergence this normalization + # exists to prevent. + if ':' in host: + host = f'[{host}]' default_port = {'https': 443, 'http': 80}.get(scheme) if port and port != default_port: netloc = f'{host}:{port}' diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 11fc2bac..31abdfa2 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -549,11 +549,26 @@ def _parse_retry_after(headers: Optional[Mapping[str, str]]) -> Optional[int]: if key.lower() == 'retry-after': value = val break - try: - secs = int(str(value).strip()) - except (TypeError, ValueError): + if value is None: + return None + text = str(value).strip() + # Accept only a bare run of ASCII digits. int() is looser than the RFC's + # delta-seconds: it also parses a leading sign ('+0010'), PEP 515 underscore + # group separators ('1_0'), and non-ASCII Unicode decimal digits (e.g. + # Arabic-Indic '٠', which int() maps to 0). str.isdigit() still admits + # those Unicode digits, so gate on isascii() as well; the result is then a + # guaranteed non-negative int (no sign, no separators, no overflow-to-None). + # Bound the length before int(): on Python >= 3.10.7 int() raises ValueError + # on a string longer than sys.get_int_max_str_digits() (default 4300 digits), + # and this runs inside post_form BEFORE its try/except, so an unbounded int() + # would let a hostile IdP / on-path proxy leak that raw ValueError past the + # module's typed-error contract (it is neither OidcError nor caught by the + # poll / refresh loops). A genuine Retry-After is a few digits; a >9-digit + # value (>31 years) is meaningless, so read it — and anything longer — as + # absent and let the caller's fixed back-off apply. + if not (text.isascii() and text.isdigit()) or len(text) > 9: return None - return secs if secs >= 0 else None + return int(text) class _PostResult(tuple): diff --git a/src/questdb/auth/_store.py b/src/questdb/auth/_store.py index 39e66274..db6a49a4 100644 --- a/src/questdb/auth/_store.py +++ b/src/questdb/auth/_store.py @@ -96,13 +96,16 @@ _LOCK_POLL_SLICE = 0.05 # Floor on a configured ``lock_stale``. A lock may legitimately be held for the # whole of one refresh under it: OidcDeviceAuth caps its HTTP timeout at 120s, -# applied per network leg, so the worst-case live hold is ~2x that. Reject a -# ``lock_stale`` at or below this floor — a shorter window would let ``_is_stale`` -# declare a LIVE holder's lock abandoned and a peer steal it mid-refresh, which -# the atomic steal cannot rescue (it guards two acquirers racing to break one -# *stale* lock, not a window so short a *live* lock reads as stale). The default -# (_DEFAULT_LOCK_STALE) sits comfortably above this. -_MIN_LOCK_STALE = 240.0 +# applied per network leg, so the worst-case live hold is ~2x that (~240s) PLUS +# the save's two fsyncs (token file + directory) and scheduling slack. Set the +# floor above that whole envelope — not at the bare 240s network figure, which a +# ``lock_stale`` configured just past it (240.001) would slip under on a slow +# host — and reject a value at or below it. A shorter window would let +# ``_is_stale`` declare a LIVE holder's lock abandoned and a peer steal it +# mid-refresh, which the atomic steal cannot rescue (it guards two acquirers +# racing to break one *stale* lock, not a window so short a *live* lock reads as +# stale). The default (_DEFAULT_LOCK_STALE) sits comfortably above this. +_MIN_LOCK_STALE = 300.0 # Set once if the platform cannot enforce owner-only POSIX permissions on the # token files (e.g. Windows), so the at-rest protection falls back to the @@ -448,11 +451,12 @@ def __init__( than stalling a sign-in. :param lock_stale: a lock older than this (seconds) is treated as abandoned by a crashed holder and stolen. It MUST exceed the longest - a live holder can hold the lock (one refresh under the lock), so a - value at or below the worst-case hold (~240s, twice - ``OidcDeviceAuth``'s 120s HTTP-timeout cap) is rejected to keep a - peer from stealing a live holder's lock mid-refresh; the default - (600s) stays safely above that. + a live holder can hold the lock (one refresh under the lock: up to + ~240s of network — twice ``OidcDeviceAuth``'s 120s HTTP-timeout cap — + plus the save's two fsyncs and scheduling slack), so a value at or + below that envelope (the ``_MIN_LOCK_STALE`` floor) is rejected to + keep a peer from stealing a live holder's lock mid-refresh; the + default (600s) stays safely above it. """ if not directory: raise OidcConfigError('the token store directory is required') @@ -604,9 +608,17 @@ def in_lock(self, key: TokenStoreKey, action: Callable[[], Any]) -> Any: held = self._acquire_lock(lock) except OSError: # Could not prepare the lock directory or file; run without the lock. - # Atomic replacement still keeps every reader consistent — only a - # rotating-refresh-token race across processes is left unguarded for - # this one refresh. + # Atomic replacement still keeps every reader CONSISTENT (no torn + # read), but a degraded lock-free refresh is no longer SERIALISED + # against a peer, so for this one refresh two cross-process races are + # unguarded: (1) a rotating-refresh-token race (two processes each + # refresh and one rotation is lost), and (2) a clear-vs-save race — + # because the clear()-generation re-check that normally guards a save + # is process-local (it lives in this process's in-memory cache), a + # save() here can re-create a file another process just clear()ed, + # resurrecting a cleared token until the next clear(). Both are + # best-effort by design; closing them across processes would need an + # on-disk epoch that save re-checks under the lock. held = False try: return action() @@ -626,10 +638,39 @@ def _lock_file(self, key: TokenStoreKey) -> str: return os.path.join(self._directory, key.hash() + '.lock') def _ensure_directory(self) -> None: + # Both os.path.isdir and os.chmod FOLLOW a symlink, so a symlink planted + # at the store path — by anyone with write access to its parent dir — + # would have us write the plaintext token files into, and chmod, the + # link's TARGET (outside any directory we own): re-asserting 0700 would + # then tighten the target, not close the exposure. lstat does not follow, + # so use it to detect a symlinked leaf and refuse it rather than operate + # through it. Only the final component is checked, so a symlinked PARENT + # (e.g. the whole store relocated to another volume via + # QUESTDB_CLIENT_OIDC_TOKEN_STORE_DIR) still works; a symlink AT the leaf + # does not. Refusal is best-effort like every other store failure: the + # sign-in still succeeds in memory, only persistence is skipped (with a + # warning) — see OidcDeviceAuth._warn_persistence. This narrows but cannot + # fully close the TOCTOU (a swap between lstat and the write needs precise + # timing plus parent write access); it defeats a persistently-planted + # symlink, the realistic case. + try: + leaf = os.lstat(self._directory) + except FileNotFoundError: + leaf = None + except OSError as e: + raise OidcError( + f'could not access the OIDC token store directory: {e}') from e + if leaf is not None and stat.S_ISLNK(leaf.st_mode): + raise OidcError( + 'the OIDC token store path is a symbolic link; refusing to use ' + 'it because the plaintext token files could be redirected ' + 'outside the owner-only directory. Point the store at a real ' + f'directory, or set {TOKEN_STORE_DIR_ENV} to one.') if os.path.isdir(self._directory): - # Re-assert owner-only permissions on a pre-existing directory: one - # left world/group-accessible by another tool, a permissive umask, or - # a hostile local pre-create would otherwise expose the token files. + # Re-assert owner-only permissions on a pre-existing (real) directory: + # one left world/group-accessible by another tool, a permissive + # umask, or a hostile local pre-create would otherwise expose the + # token files. The symlink variant is handled above. self._restrict_to_owner() return os.makedirs(self._directory, mode=0o700, exist_ok=True) @@ -819,7 +860,20 @@ def _is_stale(self, lock: str) -> bool: # atomic write still holds, and an attacker who can plant it already # has write access to the token directory.) return False - return (time.time() - mtime) > self._lock_stale + elapsed = time.time() - mtime + # Staleness rides the wall clock (st_mtime vs time.time()), unavoidable + # for a lock that may be shared across hosts with no common monotonic + # source: an NTP step still skews the age, and nothing file-only can fix + # that. Guard the one anomaly we CAN detect locally — a future-dated mtime + # (elapsed < 0), i.e. our clock currently reads BEHIND the lock's, whether + # from a backward step here or a holder whose clock runs ahead. The age is + # then untrustworthy and the lock may well be live, so treat it as fresh + # (do not steal) rather than break a live holder's lock; a genuinely + # abandoned lock is re-judged stale on a later poll once the clock catches + # up to it. + if elapsed < 0: + return False + return elapsed > self._lock_stale def _audience_matches(key_audience: Optional[str], file_audience: Any) -> bool: diff --git a/test/test_auth.py b/test/test_auth.py index 7f3750f6..5ebff908 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -90,9 +90,17 @@ class _FakeAuth: _ctx = None - def __init__(self, token='TKN'): + def __init__(self, token='TKN', interactive_required=False): self._value = token self.calls = 0 + # When True, mimic "no token acquired yet": a non-interactive fetch + # (allow_interactive=False — the pool-thread path) refuses with + # OidcInteractionRequired instead of returning a token, exactly as + # OidcDeviceAuth._token does when it would otherwise start a device flow. + self._interactive_required = interactive_required + # The allow_interactive value of the last _token() call, so a test can + # assert the adapter fetches the per-connection token non-interactively. + self.last_allow_interactive = None def token(self): self.calls += 1 @@ -104,6 +112,10 @@ def _token(self, *, allow_interactive=True): # thread, where an interactive prompt would block the pool). Mirror # OidcDeviceAuth and count it like token(). self.calls += 1 + self.last_allow_interactive = allow_interactive + if self._interactive_required and not allow_interactive: + raise OidcInteractionRequired( + 'Sign in first: no token has been acquired.') return self._value def headers(self): @@ -1019,6 +1031,38 @@ def test_access_mode_rejects_control_char_in_network_access_token(self): auth.token() self.assertIsNone(auth._tokens) + def test_blank_network_token_treated_as_missing(self): + # A served token that is blank (empty or whitespace-only) must read as + # ABSENT — not be cached and sent as "Bearer ". A run of spaces + # passes the printable-ASCII injection gate, so without an explicit blank + # check it would slip through and defeat the "fail once with a clear + # error, don't cache an unusable token" guarantee: the client would serve + # the blank token until expiry instead of surfacing the actionable "IdP + # returned no id_token" error. Mirrors the control-char pair above for + # both served kinds. + from questdb.auth._device import _safe_token_or_none + self.assertIsNone(_safe_token_or_none(' ')) # all spaces + self.assertIsNone(_safe_token_or_none('')) # empty + self.assertIsNone(_safe_token_or_none(' \t ')) # tab non-printable too + self.assertEqual( # inner space is kept + _safe_token_or_none('tok en'), 'tok en') + # groups mode serves the id_token: a blank one fails terminally, no cache. + self.state.token_script = [(200, { + 'access_token': ACCESS_TOKEN, 'id_token': ' ', + 'token_type': 'Bearer', 'expires_in': 3600})] + auth = self.make_auth(groups_in_token=True) + with self.assertRaises(OidcDeviceFlowError): + auth.token() + self.assertIsNone(auth._tokens) + # access mode serves the access_token: a blank one likewise fails. + self.state.token_script = [(200, { + 'access_token': ' ', 'id_token': ID_TOKEN, + 'token_type': 'Bearer', 'expires_in': 3600})] + auth = self.make_auth(groups_in_token=False) + with self.assertRaises(OidcDeviceFlowError): + auth.token() + self.assertIsNone(auth._tokens) + def test_access_token_headers(self): auth = self.make_auth(groups_in_token=False) self.assertEqual(auth.headers(), @@ -2811,6 +2855,66 @@ def create(**kw): events['fn'](None, None, [], cparams) self.assertEqual(cparams['password'], 'TKN') self.assertEqual(auth.calls - before, 2) + # ...and it fetches NON-interactively (via _token(allow_interactive= + # False), not token()): a regression to the interactive accessor would + # leave last_allow_interactive at None / True and fail here. See also + # test_sqlalchemy_engine_per_connect_refuses_interactive_signin. + self.assertIs(auth.last_allow_interactive, False) + + def test_sqlalchemy_engine_per_connect_refuses_interactive_signin(self): + # The per-connection token injection MUST be non-interactive: it runs on + # a pool thread, where launching a device-flow browser prompt would block + # the pool. The adapter fetches via auth._token(allow_interactive=False), + # so when no token has been acquired yet the pool sees OidcInteraction- + # Required rather than a hung prompt. Drive the real do_connect listener + # from a worker thread (the pool context) and assert the refusal + # propagates. A regression to auth.token() / allow_interactive=True would + # return 'TKN' here and NOT raise, failing this test. + auth = _FakeAuth('TKN', interactive_required=True) + events = {} + + fake_sa = types.ModuleType('sqlalchemy') + fake_sa.__path__ = [] + fake_sa.create_engine = lambda url, **kw: object() + + class _Event: + @staticmethod + def listens_for(target, name): + def deco(fn): + events.update(name=name, fn=fn) + return fn + return deco + + fake_sa.event = _Event + fake_eng = types.ModuleType('sqlalchemy.engine') + + class _URL: + @staticmethod + def create(**kw): + return 'URL' + + fake_eng.URL = _URL + with mock.patch.dict(sys.modules, { + 'sqlalchemy': fake_sa, + 'sqlalchemy.engine': fake_eng, + 'psycopg': types.ModuleType('psycopg')}): + sqlalchemy_engine(auth, 'https://db.example.com:9000') + + provide_token = events['fn'] + box = {} + + def run(): + try: + provide_token(None, None, [], {}) + except BaseException as e: # noqa: BLE001 - re-raised via the box + box['exc'] = e + + t = threading.Thread(target=run) + t.start() + t.join() + self.assertIsInstance(box.get('exc'), OidcInteractionRequired) + # The load-bearing half: it refused because the fetch was non-interactive. + self.assertIs(auth.last_allow_interactive, False) def test_sqlalchemy_engine_uses_bare_ipv6_host(self): # m6: SQLAlchemy's URL.create takes host and port separately and hands @@ -3465,6 +3569,28 @@ def test_normalize_url_malformed_ipv6_raises_config_error(self): with self.assertRaises(OidcConfigError): _normalize_url('https://[::1') + def test_normalize_url_rebrackets_ipv6_to_match_store_key(self): + # _normalize_url (the in-memory cache_key) must re-add the brackets + # urllib strips off an IPv6 literal, exactly as the on-disk + # _canonical_endpoint does. Without them "[::1]:9000" (host ::1, port + # 9000) and the DISTINCT host "[::1:9000]" (default port) both collapse to + # the ambiguous "::1:9000" and key two different IPv6 endpoints to ONE + # in-memory cache entry — while the bracketing disk store keeps them apart + # — i.e. a token keyed one way in memory and another on disk, the exact + # divergence this normalization exists to prevent. + from questdb.auth._device import _normalize_url + from questdb.auth._store import _canonical_endpoint + a = 'https://[::1]:9000/token' + b = 'https://[::1:9000]/token' + # Distinct in memory now (they previously collided) ... + self.assertNotEqual(_normalize_url(a), _normalize_url(b)) + # ... making the SAME distinction the on-disk key already made. + self.assertNotEqual(_canonical_endpoint(a), _canonical_endpoint(b)) + # Brackets preserved so the host:port boundary stays unambiguous. + self.assertEqual(_normalize_url(a), 'https://[::1]:9000/token') + self.assertEqual( + _normalize_url('http://[::1]/token'), 'http://[::1]/token') + def test_realm_path_distinguishes_key(self): # Multi-tenant IdP: same host, different realm path -> distinct keys # (the old origin-only key collided, leaking one realm's token). @@ -3649,6 +3775,42 @@ def test_post_form_attaches_retry_after_to_non_json_error(self): self.assertEqual(cm.exception.status, 429) self.assertEqual(cm.exception.retry_after, 30) + def test_parse_retry_after_rejects_lenient_int_forms(self): + # m7: int() is looser than the RFC 7231 delta-seconds it parses — it also + # accepts a leading sign ('+0010'), PEP 515 underscore separators ('1_0'), + # and non-ASCII Unicode decimal digits (e.g. Arabic-Indic '٠', mapped to + # 0). Only a bare run of ASCII digits is a valid Retry-After; anything else + # must read as absent (None) so the poll falls back to its fixed back-off + # rather than honor a malformed / attacker-crafted value. + from questdb.auth._http import _parse_retry_after + # Accepted: plain / zero-padded ASCII digits, surrounding whitespace, + # zero, and a case-insensitive (HTTP/2- or proxy-lowercased) header name. + self.assertEqual(_parse_retry_after({'Retry-After': '30'}), 30) + self.assertEqual(_parse_retry_after({'Retry-After': '0010'}), 10) + self.assertEqual(_parse_retry_after({'Retry-After': ' 5 '}), 5) + self.assertEqual(_parse_retry_after({'Retry-After': '0'}), 0) + self.assertEqual(_parse_retry_after({'retry-after': '7'}), 7) + # Rejected (all -> None): sign, underscores, Unicode digits, superscript, + # decimal / exponent, words, and the empty / whitespace-only value. + for bad in ('+0010', '-5', '1_0', '٠١', '²', + '10.0', '1e3', 'soon', '', ' '): + self.assertIsNone(_parse_retry_after({'Retry-After': bad}), bad) + # No matching header, an empty mapping, and None headers all read absent. + self.assertIsNone(_parse_retry_after({'X-Other': '9'})) + self.assertIsNone(_parse_retry_after({})) + self.assertIsNone(_parse_retry_after(None)) + # Length is bounded before int(): on Python >= 3.10.7 int() RAISES + # ValueError on a string longer than sys.get_int_max_str_digits() + # (default 4300 digits). This runs inside post_form before its own + # try/except, so an unbounded int() would leak a raw ValueError past the + # module's typed-error contract when a hostile IdP / on-path proxy sends a + # giant Retry-After. A >9-digit value (>31 years, meaningless) reads as + # absent; a 9-digit one is still accepted. Must return None, never raise. + self.assertEqual( + _parse_retry_after({'Retry-After': '9' * 9}), 999999999) + self.assertIsNone(_parse_retry_after({'Retry-After': '9' * 10})) + self.assertIsNone(_parse_retry_after({'Retry-After': '9' * 5000})) + def test_incomplete_error_body_maps_to_network_error(self): # A 4xx/5xx with a truncated CHUNKED body (the server announces a chunk, # sends fewer bytes, then closes) makes the error-body read raise @@ -5076,6 +5238,53 @@ def test_preexisting_loose_dir_is_tightened(self): self.store.save(self.key, self._pt()) self.assertEqual(os.stat(self.dir).st_mode & 0o777, 0o700) + @unittest.skipUnless(hasattr(os, 'symlink'), 'symlink support') + def test_symlinked_store_dir_is_refused(self): + # m8: os.path.isdir and os.chmod both FOLLOW a symlink, so a symlink + # planted at the store path (needs write access to the PARENT dir) would + # route the plaintext token files to — and chmod — the link's target, + # outside any directory we own; re-asserting 0700 would then tighten the + # target, not the exposure. _ensure_directory detects the symlinked leaf + # with lstat and refuses it, so save()/in_lock() raise (best-effort: the + # device flow degrades to no persistence) rather than write a credential + # through the link. + target = os.path.join(self.dir, 'real_target') + os.mkdir(target) + link = os.path.join(self.dir, 'link_store') + os.symlink(target, link) + store = FileTokenStore.at(link) + with self.assertRaises(OidcError): + store.save(self.key, self._pt()) + self.assertEqual(os.listdir(target), []) # nothing written through it + # in_lock likewise refuses to run its action through the link. + with self.assertRaises(OidcError): + store.in_lock(self.key, lambda: 'unreachable') + self.assertEqual(os.listdir(target), []) + # Only the LEAF is checked: a symlinked PARENT (e.g. the whole store + # relocated to another volume) is fine when the leaf itself is a real dir. + inner = FileTokenStore.at(os.path.join(link, 'tokens')) + inner.save(self.key, self._pt()) + self.assertIsNotNone(inner.load(self.key)) + self.assertIn('tokens', os.listdir(target)) # created under the real dir + + def test_is_stale_ignores_future_dated_lock(self): + # m2: staleness rides the wall clock (st_mtime vs time.time()), which is + # unavoidable for a cross-host lock. A FUTURE-dated mtime — our clock + # stepped back, or a holder's clock runs ahead — gives a negative age we + # cannot trust (the lock may be live), so _is_stale reads it as fresh and + # does NOT steal, rather than break a live holder's lock. + os.makedirs(self.dir, exist_ok=True) + lock = os.path.join(self.dir, self.key.hash() + '.lock') + open(lock, 'w').close() + future = time.time() + 100_000 + os.utime(lock, (future, future)) + self.assertFalse(self.store._is_stale(lock)) + # Guard didn't over-broaden: a genuinely old lock past the window is still + # stale. + past = time.time() - 100_000 + os.utime(lock, (past, past)) + self.assertTrue(self.store._is_stale(lock)) + def test_atomic_write_leaves_no_tmp(self): self.store.save(self.key, self._pt()) self.store.save(self.key, self._pt(refresh_token='RT2')) @@ -5290,7 +5499,7 @@ def test_in_lock_steals_a_stale_lock(self): # (os.utime, instant) rather than use a tiny window, so the lock reads as # abandoned without a real wait. store = FileTokenStore( - self.dir, lock_acquire_budget=1.0, lock_stale=300) + self.dir, lock_acquire_budget=1.0, lock_stale=_MIN_LOCK_STALE + 1) os.makedirs(self.dir, exist_ok=True) lock = os.path.join(self.dir, self.key.hash() + '.lock') open(lock, 'w').close() @@ -5314,7 +5523,7 @@ def test_concurrent_steal_stays_exclusive_two_threads(self): # test_in_lock_serializes_concurrent_acquirers, and no-hang/no-leak under # heavier steal contention by the test below.) store = FileTokenStore( - self.dir, lock_acquire_budget=2.0, lock_stale=300) + self.dir, lock_acquire_budget=2.0, lock_stale=_MIN_LOCK_STALE + 1) os.makedirs(self.dir, exist_ok=True) lock = os.path.join(self.dir, self.key.hash() + '.lock') open(lock, 'w').close() @@ -5352,7 +5561,7 @@ def test_concurrent_steal_does_not_hang_or_leak(self): # (see the two-thread test above), so this asserts the guarantees that # always hold. store = FileTokenStore( - self.dir, lock_acquire_budget=10.0, lock_stale=300) + self.dir, lock_acquire_budget=10.0, lock_stale=_MIN_LOCK_STALE + 1) os.makedirs(self.dir, exist_ok=True) lock = os.path.join(self.dir, self.key.hash() + '.lock') open(lock, 'w').close() @@ -5386,7 +5595,7 @@ def test_steal_recheck_does_not_delete_a_lock_that_became_fresh(self): # stubbed _is_stale: "stale" to the first (acquire-loop) check, "fresh" to # the post-rename re-check. store = FileTokenStore( - self.dir, lock_acquire_budget=0.2, lock_stale=300) + self.dir, lock_acquire_budget=0.2, lock_stale=_MIN_LOCK_STALE + 1) os.makedirs(self.dir, exist_ok=True) lock = os.path.join(self.dir, self.key.hash() + '.lock') with open(lock, 'w') as f: @@ -5739,6 +5948,23 @@ def test_persisted_token_with_control_char_is_rejected(self): self.assertEqual(auth.token(), ID_TOKEN) self.assertEqual(self.state.device_requests, 1) # rejected -> device flow + def test_persisted_blank_token_is_rejected(self): + # M1: the persistence path shares the wire path's blank-token guard. A + # served token that is empty or whitespace-only reads as absent (a run of + # spaces would otherwise pass the printable-ASCII gate), so the entry is + # ignored and a fresh sign-in follows rather than serving "Bearer + # ". Mirrors the control-char rejection above. + store = _FakeStore() + store.saved = PersistedToken( + access_token=ACCESS_TOKEN, id_token=' ', + refresh_token='REFRESH-1', + expires_at=FakeClock().now() + 3600, token_ttl=3600.0) + self._restart() + self._reset_server_counters() + auth = self.make_auth(token_store=store, clock=FakeClock()) # groups mode + self.assertEqual(auth.token(), ID_TOKEN) + self.assertEqual(self.state.device_requests, 1) # rejected -> device flow + def test_coordinated_refresh_adopts_peer_rotation(self): # Under the cross-process lock, re-reading the store sees a peer's freshly # rotated token and adopts it instead of POSTing a (now revoked) refresh. From 9912a51c182ec72867f8ca34430ad1ffc0aebd84 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 1 Jul 2026 10:59:23 +0100 Subject: [PATCH 094/104] fix(auth): sync persisted refresh token when adopting a cached token The slow-path shared-cache promotion set self._tokens from the shared MemoryCache without moving self._last_persisted_refresh_token, unlike every other adopter (_adopt, _store, clear). That desync made _refresh_under_lock's `refresh_token == _last_persisted_refresh_token` gate misread a peer-rotated adopted token as "our save failed, in-memory is newer than disk" and skip the token-store re-read -- so it refreshed a token a peer had already rotated away (revoked) and re-prompted, while the peer's valid refresh token sat unused on disk. Reachable with a rotating-refresh-token IdP, a FileTokenStore, and multiple OidcDeviceAuth instances sharing the process-global cache across processes; fail-safe (never serves a wrong/expired token) but defeats the silent cross-instance/restart resume the store exists to provide. Adopt the cache token's refresh token into the marker, exactly as _adopt does for a disk load. Adds a regression test that fails without the fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 12 ++++++++++++ test/test_auth.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 3d002348..ef9c55eb 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -925,6 +925,18 @@ def _obtain_tokens(self, *, allow_interactive: bool = True) -> TokenSet: or (cached.is_valid(self._now()) and self._has_required_token(cached))): self._tokens = cached + # Adopting a token from the shared cache must also move the + # "last persisted" marker, exactly as _adopt does for a disk + # load: the shared cache and the token store are written + # together by _store, so a cache token is believed to be on + # disk. Without this, _refresh_under_lock's + # `refresh_token == _last_persisted_refresh_token` gate would + # misread this adopted (peer-rotated) token as "newer than + # disk, our save failed" and skip the store re-read — then + # refresh a token a peer has already rotated away (revoked) + # and re-prompt, while the peer's valid refresh token sits + # unused on disk. + self._last_persisted_refresh_token = cached.refresh_token tokens = self._valid_cached() if tokens is not None: return tokens diff --git a/test/test_auth.py b/test/test_auth.py index 5ebff908..796efa85 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2754,6 +2754,39 @@ def test_stale_local_token_adopts_fresh_shared_cache_token(self): self.assertEqual(self.state.refresh_requests, 0) self.assertEqual(self.state.device_requests, 0) + def test_promoted_cache_token_syncs_last_persisted_marker(self): + # m1: adopting a fresh token from the shared cache (a peer refreshed and + # ROTATED the refresh token) must also advance this instance's + # _last_persisted_refresh_token marker, exactly as _adopt does for a disk + # load. Otherwise a later coordinated refresh reads the stale marker: its + # `refresh_token == _last_persisted_refresh_token` gate wrongly concludes + # "our save failed, in-memory is newer than disk", skips the token-store + # re-read, and refreshes the peer-rotated (now revoked) token — forcing a + # needless re-prompt while the peer's valid refresh token sits on disk. + clock = FakeClock() + a = self.make_auth(clock=clock) + b = self.make_auth(clock=clock) + self.assertEqual(a.cache_key, b.cache_key) + now = clock.now() + # a last saw 'r-old' as persisted (e.g. from its own earlier sign-in) and + # now holds a stale local token still carrying it: + a._tokens = TokenSet( + access_token='old', id_token='old-id', refresh_token='r-old', + expires_at=now - 10) + a._last_persisted_refresh_token = 'r-old' + # b refreshed and rotated the refresh token into the shared cache: + b._cache.store(b.cache_key, TokenSet( + access_token='fresh-access', id_token=ID_TOKEN, + refresh_token='r-new', issued_at=now, expires_at=now + 3600)) + # a.token() promotes the fresh cached token (no refresh / sign-in)... + self.assertEqual(a.token(), ID_TOKEN) + self.assertEqual(self.state.refresh_requests, 0) + self.assertEqual(self.state.device_requests, 0) + # ...and the persisted marker now tracks the ADOPTED refresh token, so a + # later _refresh_under_lock re-reads the store instead of replaying the + # revoked 'r-old'. + self.assertEqual(a._last_persisted_refresh_token, 'r-new') + class TestAdapters(unittest.TestCase): """PG-wire connection adapters: tested via injected fake modules (the real From 80cf3b752569160fab5601f38ecbc91ca615384d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 1 Jul 2026 10:59:31 +0100 Subject: [PATCH 095/104] docs(auth): export Renderer so its docstring xref resolves OidcDeviceAuth's docstrings reference :class:`~questdb.auth.Renderer` for the renderer= parameter, but Renderer was not exported, so the -nW (nitpicky, warnings-as-errors) docs build could not resolve the target and broke ./proj doc. Export Renderer from the package (__init__ + __all__) and autodoc it in api.rst, and document its callback contract (untrusted MITM-tamperable input, hold-the-lock/no-reentrancy, best-effort) so custom renderers are a proper public extension point. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/api.rst | 4 ++++ src/questdb/auth/__init__.py | 2 ++ src/questdb/auth/_render.py | 40 +++++++++++++++++++++++++++++++----- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index e50bc2bd..86dea4a7 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -112,6 +112,10 @@ See the :ref:`oidc_auth` guide for an overview. :undoc-members: :show-inheritance: +.. autoclass:: questdb.auth.Renderer + :members: + :show-inheritance: + .. autoexception:: questdb.auth.OidcError :show-inheritance: diff --git a/src/questdb/auth/__init__.py b/src/questdb/auth/__init__.py index 5c1d389d..e6aa0e72 100644 --- a/src/questdb/auth/__init__.py +++ b/src/questdb/auth/__init__.py @@ -53,6 +53,7 @@ from ._device import OidcDeviceAuth from ._discovery import OidcConfig from ._cache import TokenSet +from ._render import Renderer from ._errors import ( OidcError, OidcConfigError, @@ -80,6 +81,7 @@ 'OidcNetworkError', 'OidcTimeoutError', 'PersistedToken', + 'Renderer', 'TokenSet', 'TokenStore', 'TokenStoreKey', diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index a3ba5b47..ef4ddc27 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -446,19 +446,49 @@ def _fmt_minutes(seconds: float) -> int: class Renderer: - """No-op renderer interface; subclasses present the prompt to the user.""" + """No-op renderer interface; subclasses present the prompt to the user. + + Pass an instance as ``renderer=`` to :class:`~questdb.auth.OidcDeviceAuth` + to customise how the device-flow sign-in is shown (the built-ins are a + plain-text terminal renderer and a rich Jupyter one). Every callback is + optional — the base class no-ops each, so a subclass may override only the + ones it cares about. + + **The callbacks receive untrusted, MITM-tamperable IdP fields** + (``verification_uri``, ``user_code``, error strings, a JWT-derived + identity). A custom renderer that writes them to a terminal or a notebook + DOM must sanitise them itself (the built-ins strip control/bidi/zero-width + characters and vet the verification host); echoing them raw re-opens the + prompt-spoofing surface the built-in renderers close. + + **Concurrency.** The callbacks run while ``OidcDeviceAuth`` holds its + (non-reentrant) acquisition lock, so a callback must not call back into the + same instance's :meth:`~questdb.auth.OidcDeviceAuth.token` / + :meth:`~questdb.auth.OidcDeviceAuth.clear` (doing so raises rather than + deadlocks). Callbacks are best-effort: an exception raised by one is + swallowed and never aborts an otherwise-successful sign-in. + """ def on_prompt(self, resp: Dict[str, Any]) -> None: - pass + """Show the sign-in prompt at the start of the device flow. + + ``resp`` is the raw (untrusted) device-authorization response; the + verification URI and user code live under ``verification_uri`` / + ``verification_uri_complete`` / ``user_code``. + """ def on_waiting(self, seconds_left: float) -> None: - pass + """Report progress while polling; ``seconds_left`` is the time + remaining before the device code expires.""" def on_success(self, identity: Optional[str], expires_in: float) -> None: - pass + """Report a completed sign-in. ``identity`` is a best-effort, + unverified display name from the token's claims (or ``None``); + ``expires_in`` is the token's remaining lifetime in seconds.""" def on_failure(self, message: str) -> None: - pass + """Report a failed or expired sign-in with a human-readable + ``message`` (which may interpolate an untrusted IdP error string).""" class TerminalRenderer(Renderer): From d4710c79c18658501f8acc1b355ce7e86b5034d8 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 1 Jul 2026 11:43:56 +0100 Subject: [PATCH 096/104] fix(auth): harden token persistence and discovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the level-3 review findings on the questdb.auth module. Security: - _tokenset_from_persisted: screen BOTH wire-bindable tokens (access_token and id_token) through _safe_token_or_none, not just the served kind, so a control/CRLF char in a non-served persisted token can no longer land in the TokenSet (and be re-persisted verbatim) — symmetric with the network path. - discover_device_endpoint_from_idp: reject a discovery document whose own 'issuer' does not match the pinned issuer (RFC 8414 3.3), trailing-slash- insensitive; absent/non-string tolerated, so it is strictly a tightening. Robustness / nits: - _http: correct the head-read watchdog docstrings to describe the residual open()->release() window honestly (rare, recoverable via a retried OidcNetworkError) instead of overclaiming it cannot occur. - _store.save: close the mkstemp fd if os.fdopen raises before taking ownership; document the orphaned-lock window at the _steal_stale_lock restore site; add docstrings to the FileTokenStore method overrides. - _device._normalize_url: keep an explicit :0 port (compare against None, not truthiness) to match _canonical_endpoint's identity distinctions. - _adapters._coerce_port: reject a non-integral float port instead of silently truncating it through int(). Tests / CI: - Bound the previously-unbounded joins in the in_lock serialization and sqlalchemy per-connect tests and assert is_alive()==False, so a deadlock regression fails cleanly instead of hanging the suite. - ci/pip_install_deps.py: install cryptography so the behavioural TLS-rejection test runs in CI rather than silently skipping. - New regression tests: discovery issuer mismatch/trailing-slash, persisted non-served token screening, and a non-integral float pg_port. Verified each fails without its corresponding source fix. Full suite: 319 pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- ci/pip_install_deps.py | 7 ++++ src/questdb/auth/_adapters.py | 8 ++++ src/questdb/auth/_device.py | 38 +++++++++++------ src/questdb/auth/_discovery.py | 23 ++++++++++- src/questdb/auth/_http.py | 22 +++++++--- src/questdb/auth/_store.py | 37 ++++++++++++++++- test/test_auth.py | 74 +++++++++++++++++++++++++++++++++- 7 files changed, 186 insertions(+), 23 deletions(-) diff --git a/ci/pip_install_deps.py b/ci/pip_install_deps.py index e3ee25b4..321d4970 100644 --- a/ci/pip_install_deps.py +++ b/ci/pip_install_deps.py @@ -107,6 +107,13 @@ def main(args): try_pip_install('fastparquet>=2023.10.1') try_pip_install('pyarrow') + # For the questdb.auth OIDC tests: the behavioural TLS-rejection test + # (test_untrusted_server_certificate_is_rejected) generates a self-signed + # cert at runtime and skips without `cryptography`. Install it so the real + # handshake path is exercised in CI, not just the static-posture assertion. + # try_ (not required): on a platform with no wheel the test simply skips, as + # it already does locally. + try_pip_install('cryptography') on_linux_is_glibc = ( (not platform.system() == 'Linux') or diff --git a/src/questdb/auth/_adapters.py b/src/questdb/auth/_adapters.py index c10f9371..4c76cc9b 100644 --- a/src/questdb/auth/_adapters.py +++ b/src/questdb/auth/_adapters.py @@ -119,6 +119,14 @@ def _coerce_port(pg_port: Any) -> int: if isinstance(pg_port, bool): raise OidcConfigError( f'pg_port must be an integer port number, got {pg_port!r}.') + # A non-integral float silently truncates through int() (int(8812.9) == 8812) + # — never what the caller meant — so reject it explicitly. This also rejects + # inf/nan (is_integer() is False for both) with the clearer "integer port" + # message rather than the OverflowError/ValueError int() would raise. An + # integral float (8812.0) is still accepted as a convenience. + if isinstance(pg_port, float) and not pg_port.is_integer(): + raise OidcConfigError( + f'pg_port must be an integer port number, got {pg_port!r}.') try: port = int(pg_port) except (TypeError, ValueError, OverflowError) as e: diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index ef9c55eb..27ca5e4c 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -1168,20 +1168,27 @@ def _adopt(self, persisted: Optional[PersistedToken]) -> bool: def _tokenset_from_persisted( self, persisted: PersistedToken) -> Optional[TokenSet]: - # The file is attacker-writable, so treat the served token (the one - # token() puts verbatim into an Authorization header or a PG-wire - # password) as untrusted: reject a control/non-ASCII char — and the whole - # entry — rather than route a tampered credential onto the wire. A null - # served token is unusable. - access_token = _str_or_none(persisted.access_token) - id_token = _str_or_none(persisted.id_token) + # The file is attacker-writable, so treat BOTH wire-bindable tokens + # (access_token / id_token) as untrusted and run them through the SAME + # gate the network path applies (_safe_token_or_none in + # _tokenset_from_response): a control/non-ASCII char (a header / + # _sso-password injection vector) or a blank/whitespace-only value drops + # that token to None rather than landing in the TokenSet and being + # re-persisted verbatim by _snapshot. Gating BOTH — not just the currently + # served kind — keeps the persisted and network ingestion paths symmetric, + # so a tampered non-served token can't survive into the TokenSet (and back + # to disk) to be picked up by a later mode change or a future adapter that + # reads it. The refresh_token is only ever re-sent url-encoded to the IdP + # (never onto a header), so it keeps the plain str-or-None coercion, + # matching _tokenset_from_response. + access_token = _safe_token_or_none(persisted.access_token) + id_token = _safe_token_or_none(persisted.id_token) refresh_token = _str_or_none(persisted.refresh_token) served = id_token if self.config.groups_in_token else access_token - # A null, blank (whitespace-only), or control/non-ASCII served token is - # unusable: reject the whole entry rather than serve a tampered or empty - # credential. The blank check mirrors _safe_token_or_none on the wire - # path so a run of spaces can't slip through the printable-ASCII gate. - if not served or not served.strip() or not _has_only_token_chars(served): + # A null served token (absent, or dropped by the gate above as blank / + # control / non-ASCII) is unusable: reject the whole entry rather than + # serve a tampered or empty credential. + if not served: return None # The file is attacker-writable (and may have been written under a skewed # clock), so bound how long the loaded token is trusted exactly as a @@ -1715,7 +1722,12 @@ def _normalize_url(url: str) -> str: if ':' in host: host = f'[{host}]' default_port = {'https': 443, 'http': 80}.get(scheme) - if port and port != default_port: + # Compare against None, not truthiness, so an explicit :0 — falsy but a + # distinct (if unconnectable) port — is kept rather than collapsed onto the + # default, matching _store._canonical_endpoint (which keeps :0 too). The two + # must make the SAME port distinctions or a :0 endpoint would key one way in + # memory and another on disk. + if port is not None and port != default_port: netloc = f'{host}:{port}' else: netloc = host diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index ca022156..2a36bd39 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -449,7 +449,28 @@ def discover_device_endpoint_from_idp( # document (from a captive portal, bad proxy, or hostile IdP) to empty so # resolve_config's doc.get(...) yields a clear "could not resolve" error # rather than an AttributeError. Mirrors settings_config. - return doc if isinstance(doc, dict) else {} + doc = doc if isinstance(doc, dict) else {} + # RFC 8414 §3.3 / OpenID Connect Discovery: the document's own `issuer` MUST + # be identical to the issuer it was fetched from. We fetched from the + # out-of-band-pinned issuer's own origin over TLS, so a network attacker can't + # substitute the document — but enforcing the match still turns a + # misconfigured or wrong-tenant IdP (a document served at the pinned origin + # that self-declares a DIFFERENT issuer, whose token / device endpoints this + # client would otherwise trust cross-origin) into a clear failure instead of + # silently routing the device-code / refresh-token POSTs per that document. + # Compared trailing-slash-insensitively; a non-string value coerces to absent + # (via _str_setting) and a document that omits the field is tolerated (a + # minimal provider) — so this is strictly a tightening of the old behaviour. + doc_issuer = _str_setting(doc.get('issuer')) + if doc_issuer is not None and doc_issuer.rstrip('/') != issuer.rstrip('/'): + raise OidcConfigError( + f'The IdP discovery document declares issuer {doc_issuer!r}, which ' + f'does not match the pinned issuer {issuer!r}; refusing to use its ' + 'endpoints. A document that self-declares a different issuer ' + 'indicates a misconfigured or wrong-tenant identity provider. Pass ' + 'the endpoints explicitly (device_authorization_endpoint=..., ' + 'token_endpoint=...) to skip discovery.') + return doc def resolve_config( diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 31abdfa2..e4d0b479 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -189,9 +189,13 @@ class _DeadlineSocket: armed only once ``open()`` returns — cannot reach. The socket is shut down at most once, and never after :meth:`release` (called - when ``open()`` returns or raises), so a timer that fires in the instant the - head read finishes can't tear down the healthy socket the body read is about - to use. + when ``open()`` returns or raises), so a timer that fires *after* the head + read finishes is a no-op. One narrow race is irreducible with a wall-clock + timer: a fire in the instant between ``open()`` returning and :meth:`release` + running can still tear the healthy socket down — but only when the head read + completes right at the deadline, and it surfaces as a typed + ``OidcNetworkError`` the caller retries (one wasted round-trip, never a wrong + or truncated token), so it degrades safely rather than corrupting a read. """ __slots__ = ('_sock', '_done', '_lock') @@ -449,9 +453,15 @@ def request( try: resp = _opener(ctx, watch).open(req, timeout=timeout) finally: - # Head read finished (returned or raised): stop guarding so the body - # read's own watchdog owns the socket, and a late timer fire can't - # tear down the healthy socket. + # Head read finished (returned or raised): release() disarms the + # watchdog so any LATER timer fire is a no-op and the body read's own + # watchdog owns the socket. One irreducible race remains — a timer + # that fires in the instant between open() returning and release() + # running here can still shut a healthy socket down (only when the + # head read completes right at the deadline). It is rare and + # recoverable: the body read then surfaces a typed OidcNetworkError, + # which the poll loop retries and a refresh retries later, costing at + # most one extra round-trip, never a wrong token. watch.release() head_timer.cancel() with resp: diff --git a/src/questdb/auth/_store.py b/src/questdb/auth/_store.py index db6a49a4..9e325772 100644 --- a/src/questdb/auth/_store.py +++ b/src/questdb/auth/_store.py @@ -512,6 +512,10 @@ def at_default_location(cls) -> 'FileTokenStore': return cls(os.path.join(home, '.questdb', 'oidc-tokens')) def load(self, key: TokenStoreKey) -> Optional[PersistedToken]: + """Load and identity-verify the persisted token for ``key`` (see + :meth:`TokenStore.load`). Returns ``None`` — never raises — for a + missing, oversized, unreadable, non-regular, or wrong-identity file, so + an unusable entry falls back to a refresh / interactive sign-in.""" path = self._token_file(key) try: st = os.stat(path) @@ -555,6 +559,12 @@ def load(self, key: TokenStoreKey) -> Optional[PersistedToken]: return self._parse_and_verify(key, data) def save(self, key: TokenStoreKey, token: PersistedToken) -> None: + """Atomically persist ``token`` for ``key`` — write a sibling temp file + (mode ``0600``), fsync it, then ``os.replace`` it over the target — so a + concurrent reader in any process sees the whole old or whole new file, + never a torn credential. Raises :class:`~questdb.auth.OidcError` on an + I/O failure (which :class:`~questdb.auth.OidcDeviceAuth` treats as + non-fatal, continuing with the in-memory token).""" content = self._serialize(key, token) try: self._ensure_directory() @@ -564,11 +574,17 @@ def save(self, key: TokenStoreKey, token: PersistedToken) -> None: fd, tmp = tempfile.mkstemp( prefix=key.hash(), suffix='.tmp', dir=self._directory) moved = False + # os.fdopen takes ownership of fd and closes it on the `with` exit; if + # it were to raise BEFORE the wrapper is created (an unlikely + # allocation failure), fd would leak — so track whether ownership + # transferred and close it ourselves in the finally when it did not. + fd_owned = False try: # Force the payload to disk before the rename, so a crash between # the write and the atomic rename cannot leave the target # pointing at unflushed (zero/partial) bytes. with os.fdopen(fd, 'wb') as f: + fd_owned = True f.write(content) f.flush() os.fsync(f.fileno()) @@ -582,6 +598,9 @@ def save(self, key: TokenStoreKey, token: PersistedToken) -> None: # fsynced (Windows). self._fsync_directory() finally: + if not fd_owned: + with contextlib.suppress(OSError): + os.close(fd) if not moved: with contextlib.suppress(OSError): os.remove(tmp) @@ -591,6 +610,9 @@ def save(self, key: TokenStoreKey, token: PersistedToken) -> None: f'{e}') from e def clear(self, key: TokenStoreKey) -> None: + """Remove the persisted entry for ``key``; a no-op when none exists. + Raises :class:`~questdb.auth.OidcError` only on an unexpected I/O error + (not for an already-absent file).""" try: os.remove(self._token_file(key)) except FileNotFoundError: @@ -600,6 +622,12 @@ def clear(self, key: TokenStoreKey) -> None: f'could not remove the OIDC token store file: {e}') from e def in_lock(self, key: TokenStoreKey, action: Callable[[], Any]) -> Any: + """Run ``action`` under a per-identity ``O_CREAT|O_EXCL`` lock file (see + :meth:`TokenStore.in_lock`), so a refresh by another process sharing this + identity is serialised rather than raced. Steals a stale lock left by a + crashed holder, and degrades to running ``action`` without the lock — + integrity is still guarded by the atomic write — rather than stall a + sign-in if it cannot acquire one.""" lock = None held = False try: @@ -835,7 +863,14 @@ def _steal_stale_lock(self, lock: str) -> None: os.remove(private) else: # Moved a still-live lock by mistake; put it back. If the slot was - # retaken meanwhile, our private copy is redundant — drop it. + # retaken meanwhile, our private copy is redundant — drop it. (Edge + # case: if the genuine holder released and removed `lock` in the + # instant between our staleness re-check and this restore, we + # re-create an entry whose holder has already exited — a fresh-looking + # ORPHAN. It blocks no one incorrectly and the next acquirer reclaims + # it once it ages past lock_stale; for that window the identity + # degrades to lock-free coordination, the same best-effort fallback as + # running without the lock at all.) try: os.replace(private, lock) except OSError: diff --git a/test/test_auth.py b/test/test_auth.py index 796efa85..168f49cf 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1994,6 +1994,48 @@ def test_well_known_fallback_for_device_endpoint(self): self.assertEqual(auth.config.device_authorization_endpoint, self.base + '/device') + def test_discovery_doc_issuer_mismatch_rejected(self): + # RFC 8414 §3.3: the discovery document's own `issuer` MUST match the + # issuer it was fetched from. A document served at the pinned issuer's + # origin that self-declares a DIFFERENT issuer (a misconfigured or + # wrong-tenant IdP) is refused rather than having its cross-origin-trusted + # endpoints used to route the device-code / refresh-token POSTs. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.scope': 'openid', + 'acl.oidc.token.endpoint': self.base + '/token', + }} + self.state.well_known = { + 'issuer': 'https://other-tenant.example.com', + 'token_endpoint': self.base + '/token', + 'device_authorization_endpoint': self.base + '/device', + } + with self.assertRaises(OidcConfigError) as cm: + OidcDeviceAuth.from_questdb(self.base, issuer=self.base, + insecure=True, renderer=Renderer()) + self.assertIn('issuer', str(cm.exception)) + + def test_discovery_doc_issuer_trailing_slash_tolerated(self): + # The issuer match is trailing-slash-insensitive, so a document that + # declares the issuer with a trailing slash (a common IdP spelling) is + # accepted rather than spuriously rejected. + self.state.settings = {'config': { + 'acl.oidc.enabled': True, + 'acl.oidc.client.id': 'questdb', + 'acl.oidc.scope': 'openid', + 'acl.oidc.token.endpoint': self.base + '/token', + }} + self.state.well_known = { + 'issuer': self.base + '/', + 'token_endpoint': self.base + '/token', + 'device_authorization_endpoint': self.base + '/device', + } + auth = OidcDeviceAuth.from_questdb(self.base, issuer=self.base, + insecure=True, renderer=Renderer()) + self.assertEqual(auth.config.device_authorization_endpoint, + self.base + '/device') + def test_device_fallback_without_issuer_is_rejected(self): # M4: QuestDB advertises the token endpoint but not the device # endpoint, and no issuer is pinned. Discovery would otherwise be @@ -2944,7 +2986,10 @@ def run(): t = threading.Thread(target=run) t.start() - t.join() + t.join(5) + # A bounded join + is_alive check so a per-connect fetch that blocks + # (regression) fails cleanly instead of hanging the suite. + self.assertFalse(t.is_alive(), 'per-connect token fetch did not return') self.assertIsInstance(box.get('exc'), OidcInteractionRequired) # The load-bearing half: it refused because the fetch was non-interactive. self.assertIs(auth.last_allow_interactive, False) @@ -3081,12 +3126,14 @@ def test_pg_port_validation(self): # float('inf')/1e400 raise OverflowError (not ValueError) from int(); # float('nan') raises ValueError. Both must map to OidcConfigError. for bad in ('not-a-port', None, '88a2', True, 0, 70000, -1, + 8812.9, # a non-integral float would silently truncate float('inf'), float('-inf'), float('nan'), 1e400): with self.subTest(pg_port=bad): with self.assertRaises(OidcConfigError): _coerce_port(bad) self.assertEqual(_coerce_port(8812), 8812) self.assertEqual(_coerce_port('5432'), 5432) # str port coerced + self.assertEqual(_coerce_port(8812.0), 8812) # integral float accepted # Both adapter entry points reject it up front (no driver required). for fn in (sqlalchemy_engine, psycopg_connect): with self.subTest(fn=fn.__name__): @@ -5581,6 +5628,8 @@ def action(): t.start() for t in threads: t.join(timeout=30) + for t in threads: + self.assertFalse(t.is_alive(), 'a steal-contest thread deadlocked') self.assertEqual(ran[0], 2) # both actions ran self.assertEqual(max_seen[0], 1) # never two at once self.assertFalse(os.path.exists(lock)) # released; no leftover @@ -5674,7 +5723,11 @@ def action(): for t in threads: t.start() for t in threads: - t.join() + t.join(30) + # Bounded join + is_alive so a serialization/deadlock regression fails + # cleanly here instead of hanging the whole suite on an unbounded join. + for t in threads: + self.assertFalse(t.is_alive(), 'in_lock serialization deadlocked') self.assertEqual(ran[0], 6) # all actions ran self.assertEqual(max_seen[0], 1) # never two at once @@ -5998,6 +6051,23 @@ def test_persisted_blank_token_is_rejected(self): self.assertEqual(auth.token(), ID_TOKEN) self.assertEqual(self.state.device_requests, 1) # rejected -> device flow + def test_persisted_non_served_token_is_screened(self): + # The persisted path screens BOTH wire-bindable tokens (parity with the + # network path), not just the served one. In groups mode a valid id_token + # is served, but a control-char access_token (non-served) is dropped to + # None in the loaded TokenSet rather than kept and re-persisted verbatim by + # _snapshot — so it can never reach a header / _sso password via a later + # mode change or a future adapter that reads it. + store = _FakeStore() + store.saved = PersistedToken( + access_token='bad\r\naccess', id_token=ID_TOKEN, + refresh_token='REFRESH-1', + expires_at=FakeClock().now() + 3600, token_ttl=3600.0) + auth = self.make_auth(token_store=store, clock=FakeClock()) # groups mode + self.assertEqual(auth.token(), ID_TOKEN) # served id_token adopted + self.assertEqual(self.state.device_requests, 0) # entry usable, no flow + self.assertIsNone(auth._tokens.access_token) # non-served: screened + def test_coordinated_refresh_adopts_peer_rotation(self): # Under the cross-process lock, re-reading the store sees a peer's freshly # rotated token and adopts it instead of POSTing a (now revoked) refresh. From ccc400c2a626b1c761f29baa97bd6e7fc8d2a969 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 1 Jul 2026 18:52:29 +0100 Subject: [PATCH 097/104] fix(auth): reject malformed verification-URL port so the shown link can't diverge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _safe_link_url vetted the scheme, userinfo and host of the device-flow verification URL but never read its port, so a URL with a non-integer or out-of-range port (e.g. "https://host:70000/device") was returned verbatim as the clickable href / webbrowser.open() / QR target. _display_url, however, drops a junk port it can't render — so the user would read "https://host/device" while the click / browser / QR went to a different port on that host. That breaks the shown-vs-opened invariant _safe_target exists to hold (the port-stripped display had made the divergence invisible). Read .port inside the existing try so a malformed port raises ValueError and the URL is rejected: it is then shown as inert, port-stripped text and never made clickable/opened/scanned. Valid explicit ports are unaffected. Add a regression test pinning the invariant. Also correct the CHANGELOG: pyproject.toml's authoritative requires-python already declared >=3.10 in the released 4.1.0, so pip has rejected 3.8/3.9 since then; this release only syncs the stale setup.py python_requires. Scope the "nothing written to disk" note to the default (opt-in FileTokenStore persists a refresh token). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.rst | 22 ++++++++++++---------- src/questdb/auth/_render.py | 13 +++++++++++-- test/test_auth.py | 24 ++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3fe85d6e..79a67759 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -42,8 +42,9 @@ Highlights: Google: ``accounts.google.com`` issuer, ``oauth2.googleapis.com`` endpoints), while still pinning endpoints advertised over an untrusted ``/settings`` channel to the issuer. -* In-process token cache with silent refresh; in-memory only by default (no - token ever written to disk). +* In-process token cache with silent refresh; in-memory only by default + (nothing written to disk unless you opt into a + :class:`~questdb.auth.TokenStore` — see the next entry). * Opt-in **token persistence** (:class:`~questdb.auth.FileTokenStore`, passed as ``token_store=``) so a restarted process resumes from a saved refresh token instead of prompting again. The default file store keeps one owner-only @@ -66,14 +67,15 @@ See the :ref:`OIDC authentication guide ` for details. Breaking Changes ~~~~~~~~~~~~~~~~~ -* The minimum supported Python is now **3.10**, raised from 3.8 in ``setup.py``. - Python 3.8 (end-of-life 2024-10) and 3.9 (end-of-life 2025-10) are no longer - supported, and ``pip`` will refuse to install this release on them. This - affects the whole ``questdb`` package, not only the new :mod:`questdb.auth` - module. This is the first release to enforce the floor: although the 4.1.0 - changelog described dropping older versions, its ``setup.py`` still declared - ``>=3.8``, so 3.8 / 3.9 installs were not actually rejected until now. Users - still on 3.8 / 3.9 should pin to an earlier ``questdb`` release. +* The minimum supported Python is **3.10**. Python 3.8 (end-of-life 2024-10) and + 3.9 (end-of-life 2025-10) are no longer supported, and ``pip`` will refuse to + install this release on them. This affects the whole ``questdb`` package, not + only the new :mod:`questdb.auth` module. The authoritative ``requires-python`` + in ``pyproject.toml`` already declared ``>=3.10`` in 4.1.0 (so ``pip`` has + rejected 3.8 / 3.9 installs since then); this release also updates the stale + ``python_requires`` in ``setup.py``, which still read ``>=3.8``, to match — + removing the inconsistency for source builds that read ``setup.py`` directly. + Users still on 3.8 / 3.9 should pin to an earlier ``questdb`` release. 4.1.0 (2025-11-28) ------------------ diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index ef4ddc27..ed57a168 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -179,10 +179,19 @@ def _safe_link_url(url: Optional[str]) -> Optional[str]: try: parts = urllib.parse.urlparse(url) scheme = (parts.scheme or '').lower() - # `.username`/`.password`/`.hostname` parse the authority; `.port` (read - # indirectly via a malformed netloc) can raise ValueError — catch it. + # `.username`/`.password`/`.hostname`/`.port` parse the authority; a + # non-integer or out-of-range `.port` raises ValueError, caught below. + # Reading `.port` is essential, not incidental: without it a URL with a + # junk port (e.g. "https://host:70000/…") is returned verbatim for the + # href / webbrowser.open() / QR, while `_display_url` DROPS that port + # from the shown text — so the displayed link and the real target would + # diverge, the exact spoof this vetting (via `_safe_target`) exists to + # prevent. Rejecting it here keeps them identical: the URL is then shown + # as inert, port-stripped text and never made clickable/opened/scanned. + # A portless URL yields `.port is None` without raising. userinfo = parts.username is not None or parts.password is not None host = parts.hostname + _ = parts.port except (ValueError, TypeError): return None if scheme not in ('http', 'https'): diff --git a/test/test_auth.py b/test/test_auth.py index 168f49cf..7c5cbcdc 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -4576,6 +4576,30 @@ def test_safe_link_url_rejects_interior_tab_newline_cr(self): _safe_target('https://idp.example.com\n/device'), 'https://idp.example.com/device') + def test_safe_link_url_rejects_malformed_port_no_display_divergence(self): + # A non-integer / out-of-range port must make the URL non-clickable, so + # the displayed link cannot diverge from the href / webbrowser.open() / + # QR target. _display_url DROPS a junk port from the shown text (it can't + # render one), so if _safe_link_url returned the URL verbatim with the + # port intact, the user would read "https://idp.example.com/device" while + # the click / browser / QR went to a different port on that host — the + # exact shown-vs-opened spoof _safe_target exists to prevent. + from questdb.auth._render import ( + _safe_link_url, _safe_target, _display_url, _render_link) + for url in ('https://idp.example.com:70000/device', # out of range + 'https://idp.example.com:99999/device', # out of range + 'https://idp.example.com:0x50/device', # non-integer + 'https://idp.example.com:8080abc/device'): # non-integer + self.assertIsNone(_safe_link_url(url), f'should reject {url!r}') + self.assertIsNone(_safe_target(url), f'should reject {url!r}') + # Shown as inert text (no clickable ), so nothing can diverge. + self.assertNotIn(' Date: Wed, 1 Jul 2026 19:04:41 +0100 Subject: [PATCH 098/104] fix(auth): reject blank-after-strip device prompts; close test-coverage gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _request_device_code accepted a user_code / verification_uri that is a non-empty string of only control / zero-width / exotic-space characters: it passed the _str_or_none truthiness guard yet renders empty via _strip_control / _display_url, producing an "Open and enter code:" prompt with nothing to act on. Gate the two DISPLAYED fields on their post-strip value (via _verification_uri, exactly what the renderer shows) so such a response is rejected as non-conformant. device_code keeps the raw check (it is sent, not shown); user_code keeps _str_or_none first so a JSON number can't be coerced visible by _strip_control's str() fallback. Also close review-flagged test gaps: - strengthen the two clear() concurrency stress tests: seed a token whose id_token is DISTINCT from the mock-minted one, so a served value tells a stale cache-hit from a genuine re-acquisition — the seed == issued value made `token() != ID_TOKEN` a tautology a broken shared-cache CAS could pass; - FileTokenStore: a mid-write failure (fdopen/fsync/rename) cleans up its .tmp and raises; load() errno routing (ELOOP/ENOTDIR -> None, EACCES/EIO -> raise); a non-string file audience/issuer is rejected; a control-char refresh_token is kept (only url-encoded to the IdP, never a header); a real cross-PROCESS save/load round-trip over the on-disk format; - TerminalRenderer.on_waiting renders the MM:SS countdown; - import-check examples/oidc_device_auth.py (not in examples.manifest.yaml). Reword the test.py comment: the auth tests are pure-Python and run standalone, but test.py imports questdb.ingress first, so the aggregated CI run builds the extension. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 34 ++++-- test/test.py | 8 +- test/test_auth.py | 208 ++++++++++++++++++++++++++++++++++-- 3 files changed, 230 insertions(+), 20 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 27ca5e4c..b57041b2 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -62,6 +62,7 @@ from ._render import ( Renderer, _safe_target, + _strip_control, _verification_uri, _verification_uri_complete, detect_interactive, @@ -1482,25 +1483,36 @@ def _request_device_code(self) -> Dict[str, Any]: self.config.device_authorization_endpoint, form) # RFC 8628 §3.2 requires device_code, user_code AND a verification URI # (RFC spells it verification_uri; some IdPs, e.g. older Google, use - # verification_url). Require the URI too: without it the prompt would - # render a blank "Open and enter code" gap, so its absence is a - # non-conformant response, not a usable one. + # verification_url). The user_code and verification URI are DISPLAYED, so + # require them non-blank AFTER control-stripping: a value of only control + # / zero-width / exotic-space characters is a non-empty string (so it + # passes _str_or_none) yet renders empty via _strip_control / _display_url, + # producing an "Open and enter code:" prompt with nothing to act on. + # Gate on _verification_uri (exactly the value the renderer shows) so a + # response that would display blank is rejected as non-conformant rather + # than shown. The device_code is not displayed (it goes in the poll body), + # so it keeps the raw non-empty-string check; user_code still needs + # _str_or_none first so a non-string (a JSON number) can't be coerced + # visible by _strip_control's str() fallback. + verification_uri = _verification_uri(body) if (status == 200 and _str_or_none(body.get('device_code')) and _str_or_none(body.get('user_code')) - and (_str_or_none(body.get('verification_uri')) - or _str_or_none(body.get('verification_url')))): + and _strip_control(body.get('user_code')).strip() + and _strip_control(verification_uri).strip()): return body error = body.get('error') if status == 200: - # 200 but the guard above failed: a required field is missing or + # 200 but the guard above failed: a required field is missing, # non-string (coerced via _str_or_none, so a JSON number/list reads # as absent instead of being stringified into the prompt / poll - # request). A non-conformant body, not an HTTP failure — say so - # plainly rather than a contradictory "failed (HTTP 200)". + # request), or blank after control-stripping (only invisible chars, + # which would render an empty prompt). A non-conformant body, not an + # HTTP failure — say so plainly rather than a contradictory + # "failed (HTTP 200)". raise OidcDeviceFlowError( - 'The IdP returned a 200 device-authorization response missing a ' - 'required field (device_code, user_code, or verification_uri); ' - 'cannot start the device flow.', + 'The IdP returned a 200 device-authorization response with a ' + 'missing or blank required field (device_code, user_code, or ' + 'verification_uri); cannot start the device flow.', status=status, error=error, error_description=body.get('error_description')) diff --git a/test/test.py b/test/test.py index 8aa74cd3..f1f85561 100755 --- a/test/test.py +++ b/test/test.py @@ -33,8 +33,12 @@ from fixture import _parse_version -# OIDC auth tests (pure-Python; no compiled extension required). -# Imported here so they are picked up by ``unittest.main()`` in CI. +# OIDC auth tests. These test cases are themselves pure-Python and import no +# compiled extension, so they can be run standalone against an unbuilt checkout +# with ``PYTHONPATH=src python -m unittest test_auth``. They are imported here +# only so ``unittest.main()`` picks them up in the CI run alongside the ingress +# tests (this module imports ``questdb.ingress`` above, so the aggregated run +# does build the extension; the standalone invocation above does not). from test_auth import ( TestDeviceFlow, TestNonInteractive, diff --git a/test/test_auth.py b/test/test_auth.py index 7c5cbcdc..dfed08f0 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -37,11 +37,13 @@ import base64 import contextlib +import errno import importlib.util import io import json import os import shutil +import subprocess import sys import tempfile import threading @@ -742,6 +744,39 @@ def test_missing_verification_uri_is_rejected(self): self.state.token_script = [(200, None)] self.assertEqual(self.make_auth().token(), ID_TOKEN) + def test_blank_after_strip_user_code_or_uri_is_rejected(self): + # A user_code / verification_uri of ONLY control / zero-width / exotic- + # space chars is a non-empty string (so it passes the _str_or_none guard) + # yet renders empty after _strip_control / _display_url — an + # "Open and enter code:" prompt with nothing to act on. Such a response + # must be rejected as non-conformant (never started / polled), like a + # missing field, rather than shown blank. + for field, blank in ( + ('user_code', '​​'), # zero-width spaces + ('user_code', ' '), # NBSP -> folds to blank + ('verification_uri', '​​'), # zero-width only + ('verification_uri', '‮​')): # bidi + zero-width + resp = {'device_code': 'DEV-CODE', 'user_code': 'WDJB-MJHT', + 'verification_uri': 'https://idp.example.com/device', + 'expires_in': 600, 'interval': 5} + resp[field] = blank + self.state.device_response = resp + with self.assertRaises(OidcDeviceFlowError, + msg=f'{field}={blank!r} not rejected') as cm: + self.make_auth().token() + self.assertIn('blank', str(cm.exception).lower()) + self.assertEqual(self.state.token_requests, []) # never polled + self.state.token_requests = [] + # A real URL / code that merely carries a TRAILING zero-width char is + # still usable — the char is stripped, visible content remains — so the + # flow proceeds rather than over-rejecting. + self.state.device_response = { + 'device_code': 'DEV-CODE', 'user_code': 'WDJB-MJHT​', + 'verification_uri': 'https://idp.example.com/device​', + 'expires_in': 600, 'interval': 5} + self.state.token_script = [(200, None)] + self.assertEqual(self.make_auth().token(), ID_TOKEN) + def test_success_message_reports_real_jwt_lifetime(self): # Issue 7: the "expires in N min" message must report the token's REAL # lifetime (JWT exp), not the cache's clamped expires_at (_MAX_EXPIRES_IN, @@ -2654,8 +2689,15 @@ def test_token_clear_stress(self): auth = self.make_auth(clock=clock, open_browser=False) # Seed a valid token so the steady state is the lock-free fast path; a # device flow then runs ONLY when a clear() has just emptied the cache. + # The seed's id_token is DISTINCT from the one the mock mints on a device + # flow (ID_TOKEN), so a served token distinguishes a stale cache-hit + # (SEED_ID) from a genuine re-acquisition (ID_TOKEN) — a seed == issued + # value would make `token() != ID_TOKEN` a tautology that a broken CAS + # (which drops the shared-cache write) could pass unnoticed. + SEED_ID = 'SEED-ID-TOKEN' + self.assertNotEqual(SEED_ID, ID_TOKEN) seed = TokenSet( - access_token='a', id_token=ID_TOKEN, refresh_token='r', + access_token='a', id_token=SEED_ID, refresh_token='r', issued_at=clock.now(), expires_at=clock.now() + 3600) auth._cache.store(auth.cache_key, seed) @@ -2669,8 +2711,11 @@ def worker(): start.wait() try: for _ in range(iters): - if auth.token() != ID_TOKEN: - errors.append('wrong token kind served') + # Only the seed or the freshly-minted token are ever valid; + # anything else is a torn / wrong-context / CAS-corrupted read. + tok = auth.token() + if tok not in (SEED_ID, ID_TOKEN): + errors.append(f'unexpected token served: {tok!r}') return except Exception as e: # noqa: BLE001 errors.append(e) @@ -2701,7 +2746,11 @@ def clearer(): max(1, self.state.device_requests) * 10) # No leaked in-flight bookkeeping once the storm settles. self.assertEqual(_MEMORY_INFLIGHT.get(auth.cache_key, 0), 0) - # The auth is still usable afterwards. + # A final clear forces re-acquisition: the served token is now the FRESH + # mock id_token, DISTINCT from the seed — proving a cleared entry is + # genuinely re-acquired (not a stale seed served) and the CAS repopulated + # the shared cache with the fresh token. + auth.clear() self.assertEqual(auth.token(), ID_TOKEN) def test_cross_instance_clear_stress(self): @@ -2724,6 +2773,16 @@ def test_cross_instance_clear_stress(self): for _ in range(n_inst)] key = insts[0].cache_key self.assertTrue(all(a.cache_key == key for a in insts)) + # Seed the shared cache with a token whose id_token is DISTINCT from the + # one the mock mints on a device flow (ID_TOKEN), so a served value tells + # a stale cache-hit (SEED_ID) apart from a re-acquisition (ID_TOKEN) — the + # seed == issued tautology would let a broken cross-instance CAS (which + # drops the shared-cache write) pass unnoticed. + SEED_ID = 'SEED-ID-TOKEN-XINST' + self.assertNotEqual(SEED_ID, ID_TOKEN) + insts[0]._cache.store(key, TokenSet( + access_token='a', id_token=SEED_ID, refresh_token='r', + issued_at=clock.now(), expires_at=clock.now() + 3600)) n_workers = 6 iters = 80 @@ -2735,8 +2794,11 @@ def worker(wid): start.wait() try: for i in range(iters): - if insts[(wid + i) % n_inst].token() != ID_TOKEN: - errors.append('wrong token kind served') + # Only the seed or a freshly-minted token are ever valid; + # anything else is a torn / wrong-context / CAS-corrupted read. + tok = insts[(wid + i) % n_inst].token() + if tok not in (SEED_ID, ID_TOKEN): + errors.append(f'unexpected token served: {tok!r}') return except Exception as e: # noqa: BLE001 errors.append(e) @@ -2768,7 +2830,10 @@ def clearer(): # No leaked process-global bookkeeping once the storm settles. self.assertEqual(_MEMORY_INFLIGHT.get(key, 0), 0) self.assertNotIn(key, _MEMORY_GENERATION) - # Still usable. + # A final clear forces re-acquisition: the served token is now the FRESH + # mock id_token, DISTINCT from the seed — proving a cleared entry is + # re-acquired across instances (not a stale seed served). + insts[0].clear() self.assertEqual(insts[0].token(), ID_TOKEN) def test_stale_local_token_adopts_fresh_shared_cache_token(self): @@ -3340,6 +3405,23 @@ def test_settings_config_ignores_user_writable_preferences(self): self.assertEqual(settings_config({'acl.oidc.client.id': 'q'}), {'acl.oidc.client.id': 'q'}) + def test_example_oidc_device_auth_imports(self): + # examples/oidc_device_auth.py is NOT in examples.manifest.yaml — it needs + # a live IdP and an interactive sign-in, so it can't run as a system test + # — hence nothing else import-checks it. Import it here (main() is guarded + # by __name__, so importing runs no I/O / sign-in) to catch a syntax error + # or public-API drift: a renamed/removed questdb.auth symbol in its + # top-level import would fail this test. + example = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'examples', 'oidc_device_auth.py') + self.assertTrue(os.path.exists(example), example) + spec = importlib.util.spec_from_file_location( + 'oidc_device_auth_example', example) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # triggers `from questdb.auth import ...` + self.assertTrue(hasattr(module, 'main')) + class TestEndpointValidation(unittest.TestCase): def setUp(self): @@ -5261,6 +5343,23 @@ def _display(self, html_str): _Cap().on_prompt(resp) # must not raise self.assertNotIn(' "1:30"; a sub-minute value zero-pads. + from questdb.auth._render import TerminalRenderer + buf = io.StringIO() + r = TerminalRenderer(stream=buf) + r.on_waiting(90.0) + out = buf.getvalue() + self.assertIn('1:30', out) + self.assertIn('waiting', out.lower()) + buf.truncate(0) + buf.seek(0) + r.on_waiting(5.0) + self.assertIn('0:05', buf.getvalue()) + # A known cross-language hash vector: the lowercase-hex SHA-256 of the canonical # identity string for this exact identity, computed independently from the frozen @@ -5396,6 +5495,101 @@ def test_atomic_write_leaves_no_tmp(self): [n for n in os.listdir(self.dir) if n.endswith('.tmp')], []) self.assertEqual(self.store.load(self.key).refresh_token, 'RT2') + def test_save_failure_leaves_no_tmp_and_raises(self): + # A mid-write failure — at fdopen (fd never wrapped), fsync (fd wrapped), + # or the atomic rename — must raise OidcError (persistence is best-effort; + # OidcDeviceAuth then continues with the in-memory token) AND remove its + # sibling temp file, so a crashed save never litters the store with a + # torn/partial .tmp credential and never closes the fd twice. Exercises + # the fd_owned / moved cleanup branches that a successful save can't. + for point in ('fdopen', 'fsync', 'replace'): + with self.subTest(point=point): + with mock.patch(f'questdb.auth._store.os.{point}', + side_effect=OSError(errno.EIO, 'injected')): + with self.assertRaises(OidcError): + self.store.save(self.key, self._pt()) + leftover = [n for n in os.listdir(self.dir) + if n.endswith('.tmp')] + self.assertEqual(leftover, [], f'{point}: temp file leaked') + self.assertFalse(os.path.exists(self._file()), + f'{point}: target created despite failure') + + def test_load_errno_routing(self): + # load()'s os.stat can fail for different reasons. A path that is not a + # usable regular file — a symlink loop (ELOOP) or a non-directory path + # component (ENOTDIR) — is "no usable entry": return None and fall back to + # a refresh / fresh sign-in. A genuine I/O or permission error (EACCES, + # EIO) is NOT recoverable by re-prompting, so it must surface as OidcError + # rather than be silently swallowed as "no token". + self.store.save(self.key, self._pt()) + for err in (errno.ELOOP, errno.ENOTDIR): + with mock.patch('questdb.auth._store.os.stat', + side_effect=OSError(err, os.strerror(err))): + self.assertIsNone(self.store.load(self.key), + f'errno {err} should read as no entry') + for err in (errno.EACCES, errno.EIO): + with mock.patch('questdb.auth._store.os.stat', + side_effect=OSError(err, os.strerror(err))): + with self.assertRaises(OidcError): + self.store.load(self.key) + + def test_load_rejects_nonstring_audience_or_issuer_in_file(self): + # The token file is attacker-writable. A non-string audience / issuer + # (a JSON number/list from a hand-edited or hostile file) must never match + # the live identity: _audience_matches / _issuer_matches demand an exact + # string match (or both absent), so such a file is rejected (load -> None) + # rather than served as though the identity lined up. + for field in ('audience', 'issuer'): + self.store.save(self.key, self._pt()) + with open(self._file()) as fh: + obj = json.loads(fh.read()) + obj[field] = 12345 # non-string + with open(self._file(), 'w') as fh: + json.dump(obj, fh) + self.assertIsNone(self.store.load(self.key), + f'non-string {field} should be rejected') + + def test_load_keeps_control_char_refresh_token(self): + # Unlike the wire-bound access/id tokens (which OidcDeviceAuth screens with + # _safe_token_or_none because they go onto an Authorization header / _sso + # password, where a decoded CR/LF is an injection vector), the + # refresh_token is only ever url-encoded into the IdP token request, never + # a header — so the store loads it verbatim and the url-encoding at send + # time neutralizes any control char. Pin that deliberate asymmetry: a + # control char in the refresh_token is preserved, not silently dropped. + self.store.save(self.key, self._pt(refresh_token='r\r\ntoken')) + got = self.store.load(self.key) + self.assertEqual(got.refresh_token, 'r\r\ntoken') + + def test_cross_process_save_load_round_trip(self): + # The file store's raison d'etre is cross-PROCESS (and cross-language) + # sharing — the rest of the suite exercises it only with threads in one + # interpreter. Save in a CHILD process, load in this one, over a real + # process boundary: proves the on-disk format a separate process writes is + # readable here (the restart-resume path, and the Java-interop contract, + # in miniature) rather than only within one address space. + script = ( + 'import sys\n' + 'from questdb.auth import (' + 'FileTokenStore, TokenStoreKey, PersistedToken)\n' + 'k = TokenStoreKey("questdb", "https://idp:443/token",\n' + ' "https://idp:443/device", "openid", None, False)\n' + 'FileTokenStore.at(sys.argv[1]).save(k, PersistedToken(\n' + ' access_token="AT", id_token="IT", refresh_token="XPROC-RT",\n' + ' expires_at=1003600.0, token_ttl=3600.0))\n') + env = dict(os.environ) + env['PYTHONPATH'] = os.pathsep.join(p for p in sys.path if p) + res = subprocess.run( + [sys.executable, '-c', script, self.dir], + env=env, timeout=60, capture_output=True, text=True) + self.assertEqual(res.returncode, 0, + f'child process failed: {res.stderr}') + got = self.store.load(self.key) + self.assertIsNotNone( + got, 'a token saved by another process was not loadable here') + self.assertEqual(got.refresh_token, 'XPROC-RT') + self.assertEqual(got.access_token, 'AT') + def test_oversized_file_ignored(self): with open(self._file(), 'wb') as fh: fh.write(b'{' + b' ' * (1 << 20)) # > _MAX_FILE_BYTES From 75f27bbb14dc4d1fa5c1406673c8b07600eafee7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 1 Jul 2026 23:43:03 +0100 Subject: [PATCH 099/104] fix(auth): keep bad-arg errors typed and reuse a peer token before re-prompt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two edge cases found while reviewing the OIDC device-flow module. - _validate_positive_number interpolated the offending value with {!r}. repr() on an int with >4300 digits itself raises ValueError (CPython's int->str limit, active on the 3.10 floor), so a huge-int timeout / default_interval escaped the module's typed-error contract as a bare ValueError instead of OidcConfigError. Fall back to a type description when repr() can't run. - When a silent refresh was exhausted, _acquire evicted the shared-cache entry and ran the device flow unconditionally. A peer OidcDeviceAuth sharing the process-global cache may have stored a valid token for this identity during our refresh; evicting it and prompting was a needless re-prompt. Re-read the shared cache before evicting and adopt a valid peer token, syncing _last_persisted_refresh_token as the promotion path does. (Checking after the evict would not work — evict() drops the peer token first, so the re-read has to precede it.) Both are covered by regression tests that fail without the fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 45 +++++++++++++++++++++++++--------- test/test_auth.py | 49 ++++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 12 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index b57041b2..0f4dd279 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -258,9 +258,17 @@ def _validate_positive_number(value: Any, name: str) -> None: # (settimeout would raise its own OverflowError later). ok = False if not ok: + # repr() on an int with >4300 digits itself raises ValueError (CPython's + # integer-string-conversion limit, active on the 3.10 floor), which would + # escape the very typed-error contract this validation exists to uphold; + # fall back to a type description for such a pathological value so the + # raise stays an OidcConfigError. + try: + shown = repr(value) + except ValueError: + shown = f'a value of type {type(value).__name__}' raise OidcConfigError( - f'{name} must be a positive, finite number of seconds, ' - f'got {value!r}') + f'{name} must be a positive, finite number of seconds, got {shown}') def _validate_timeout(value: Any) -> None: @@ -994,15 +1002,30 @@ def _acquire( refreshed = self._try_refresh_coordinated(tokens, generation) if refreshed is not None: return refreshed - # The refresh path is exhausted: the refresh_token is proven useless - # (rejected, or the IdP won't re-issue the required kind), so the - # device flow below is the only way forward. Drop the stale token — - # from this instance AND the shared cache — before the flow, so that - # if it then FAILS (non-interactive, user cancels, IdP rejects the - # device request) the doomed token isn't left cached to be reloaded - # and re-refreshed fruitlessly on every later token() call. evict() - # doesn't bump the clear()-generation, so the _store below still - # lands `fresh` (unless a genuine concurrent clear() intervenes). + # The refresh path is exhausted: OUR refresh_token is proven useless + # (rejected, or the IdP won't re-issue the required kind). Before + # falling through to an interactive sign-in, re-consult the shared + # cache: a peer instance sharing the process-global store may have + # acquired or refreshed a VALID token for this identity while we + # awaited our own (failed) refresh — adopt and return that rather than + # evict it and prompt the user needlessly. Read the backend directly + # (self._tokens still holds the stale token we are about to drop), and + # sync _last_persisted_refresh_token exactly as the _obtain_tokens + # adoption does so a later refresh of the adopted token isn't misread + # as newer-than-disk. + peer = self._cache.load(self.cache_key) + if (peer is not None and peer.is_valid(self._now()) + and self._has_required_token(peer)): + self._tokens = peer + self._last_persisted_refresh_token = peer.refresh_token + return peer + # No usable peer token: drop the stale one — from this instance AND + # the shared cache — before the flow, so that if it then FAILS + # (non-interactive, user cancels, IdP rejects the device request) the + # doomed token isn't left cached to be reloaded and re-refreshed + # fruitlessly on every later token() call. evict() doesn't bump the + # clear()-generation, so the _store below still lands `fresh` (unless + # a genuine concurrent clear() intervenes). self._tokens = None self._cache.evict(self.cache_key) diff --git a/test/test_auth.py b/test/test_auth.py index dfed08f0..8405a224 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1165,7 +1165,13 @@ def test_constructor_rejects_bad_typed_args(self): # bare OverflowError, and a too-large int does the same; both # must raise the typed error up front, not escape from urllib. {'timeout': float('inf')}, {'timeout': float('-inf')}, - {'timeout': 10 ** 1000}, {'default_interval': 10 ** 1000}): + {'timeout': 10 ** 1000}, {'default_interval': 10 ** 1000}, + # An int with >4300 digits: the finite-check rejects it, but the + # error message must not repr() it — repr() on such an int itself + # raises ValueError (CPython's int->str limit), which would escape + # the typed-error contract. 10**1000 above is only 1001 digits, + # UNDER the limit, so it does not exercise this; 10**5000 does. + {'timeout': 10 ** 5000}, {'default_interval': 10 ** 5000}): with self.assertRaises(OidcConfigError): OidcDeviceAuth(**{**good, **bad}) # A float interval/timeout is fine (clamped / passed to the socket). @@ -2894,6 +2900,47 @@ def test_promoted_cache_token_syncs_last_persisted_marker(self): # revoked 'r-old'. self.assertEqual(a._last_persisted_refresh_token, 'r-new') + def test_peer_token_adopted_after_failed_refresh_avoids_reprompt(self): + # M2: OUR refresh_token is proven useless (rejected), but a peer instance + # sharing the process-global cache stored a VALID token for this identity + # while our refresh was in flight. _acquire must adopt and return that + # peer token rather than evict it and run a needless interactive device + # flow. (Before the fix _acquire dropped straight to the device flow, + # evicting the peer's fresh token and re-prompting the user.) + clock = FakeClock() + a = self.make_auth(clock=clock) + b = self.make_auth(clock=clock) + self.assertEqual(a.cache_key, b.cache_key) + now = clock.now() + # a holds a stale local token; the shared cache is empty, so a proceeds + # past the _obtain_tokens promotion into _acquire, where its refresh runs. + a._tokens = TokenSet( + access_token='old', id_token='old-id', refresh_token='r-old', + expires_at=now - 10) + peer_tok = TokenSet( + access_token='peer-access', id_token=ID_TOKEN, + refresh_token='r-new', issued_at=now, expires_at=now + 3600) + + # Model the race deterministically: a's coordinated refresh fails (our + # token is rejected), and while it is "in flight" a peer stores a valid + # token into the shared cache. + def failed_refresh(tokens, generation): + b._cache.store(b.cache_key, peer_tok) + return None + + a._try_refresh_coordinated = failed_refresh + self.assertEqual(a.token(), ID_TOKEN) + # No interactive device flow ran: the peer token was adopted, not evicted. + self.assertEqual(self.state.device_requests, 0) + # The peer token remains in the shared cache (a did not evict it). + cached = a._cache.load(a.cache_key) + self.assertIsNotNone(cached) + self.assertEqual(cached.id_token, ID_TOKEN) + self.assertEqual(cached.refresh_token, 'r-new') + # The adopted refresh token is tracked so a later coordinated refresh + # doesn't misread it as newer-than-disk (mirrors the promotion path). + self.assertEqual(a._last_persisted_refresh_token, 'r-new') + class TestAdapters(unittest.TestCase): """PG-wire connection adapters: tested via injected fake modules (the real From 5aac5e89a119882ca0761064bac2c40a1ff0f462 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 2 Jul 2026 00:12:59 +0100 Subject: [PATCH 100/104] fix(auth): sweep orphaned token temps, harden lock, tighten docs and tests Minor findings from the review round (the Moderate fixes landed in the previous commit). - FileTokenStore.clear() now sweeps orphaned `*.tmp` siblings. save() writes the plaintext token into a temp before its atomic rename; a hard crash in that window left it behind (0600, never read back) and clear() removed only the .json, so a forgotten credential lingered on disk. - _is_stale() uses lstat, so a symlink planted at the lock path is judged by the link's own mtime rather than a target a co-tenant keeps fresh; the residual future-dated-mtime case is documented as integrity-safe. - Doc accuracy: scope "auto-refreshed" to sqlalchemy_engine (psycopg_connect captures the token once at connect time); document the adapters' host / pg_port / database params and OidcConfigError; correct FileTokenStore.load "never raises" (it raises on a hard I/O error); frame the 0600 file mode (not the 0700 directory) as the content protection; note the cross-process clear-vs-save race is not limited to the lock-failed path. - Tests: _pg_module v3/v2/none selection order, an invalid-UTF-8 HTTP body via json(), a FIFO at the token path, and the groups_in_token=False refresh fall-back (mirroring the existing groups=True test). Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.rst | 6 ++- src/questdb/auth/__init__.py | 6 ++- src/questdb/auth/_adapters.py | 8 ++++ src/questdb/auth/_store.py | 84 ++++++++++++++++++++++++++--------- test/test_auth.py | 61 +++++++++++++++++++++++++ 5 files changed, 140 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 79a67759..4185f43e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -56,8 +56,10 @@ Highlights: refresh held under the file store's cross-process lock can't outlast its staleness window; a larger value raises ``OidcConfigError`` at construction. * Convenience adapters (:func:`~questdb.auth.sqlalchemy_engine`, - :func:`~questdb.auth.psycopg_connect`) that wire the auto-refreshed token into - PG-wire as the ``_sso`` password. + :func:`~questdb.auth.psycopg_connect`) that wire the token into PG-wire as the + ``_sso`` password — ``sqlalchemy_engine`` re-supplies a fresh, auto-refreshed + token on every new pooled connection, ``psycopg_connect`` captures it at + connect time. * ``token()`` / ``headers()`` require no dependencies beyond the standard library; ``sqlalchemy`` / ``psycopg`` / ``qrcode`` / ``IPython`` are imported lazily. diff --git a/src/questdb/auth/__init__.py b/src/questdb/auth/__init__.py index e6aa0e72..0f514ef8 100644 --- a/src/questdb/auth/__init__.py +++ b/src/questdb/auth/__init__.py @@ -38,8 +38,10 @@ token = auth.token() # device flow on first use headers = auth.headers() # {"Authorization": "Bearer .."} -For PG-wire there are two convenience adapters that wire the auto-refreshed -token in as the ``_sso`` password:: +For PG-wire there are two convenience adapters that wire the token in as the +``_sso`` password — ``sqlalchemy_engine`` re-supplies a fresh, auto-refreshed +token on every new pooled connection, while ``psycopg_connect`` captures the +current token once at connect time:: from questdb.auth import sqlalchemy_engine, psycopg_connect diff --git a/src/questdb/auth/_adapters.py b/src/questdb/auth/_adapters.py index 4c76cc9b..d4b9a4bc 100644 --- a/src/questdb/auth/_adapters.py +++ b/src/questdb/auth/_adapters.py @@ -170,11 +170,14 @@ def sqlalchemy_engine( :meth:`OidcDeviceAuth.from_questdb`. :param url: The QuestDB base URL; the PG host is derived from it unless ``host=`` is given. + :param host: Override the PG-wire host (otherwise taken from ``url``). :param pg_port: PG-wire port (default ``8812``). :param database: Database name (default ``"qdb"``). :param drivername: SQLAlchemy driver; defaults to ``postgresql+psycopg`` (v3) or ``postgresql+psycopg2`` depending on what is installed. :param engine_kwargs: Forwarded to ``create_engine``. + :raises OidcConfigError: if ``pg_port`` is not a valid TCP port, or the + resolved host is missing or carries connection-string metacharacters. """ pg_port = _coerce_port(pg_port) try: @@ -231,7 +234,12 @@ def psycopg_connect( :meth:`OidcDeviceAuth.from_questdb`. :param url: The QuestDB base URL; the PG host is derived from it unless ``host=`` is given. + :param host: Override the PG-wire host (otherwise taken from ``url``). + :param pg_port: PG-wire port (default ``8812``). + :param database: Database name (default ``"qdb"``). :param connect_kwargs: Forwarded to the driver's ``connect()``. + :raises OidcConfigError: if ``pg_port`` is not a valid TCP port, or the + resolved host is missing or carries connection-string metacharacters. """ pg_port = _coerce_port(pg_port) mod = _pg_module() diff --git a/src/questdb/auth/_store.py b/src/questdb/auth/_store.py index 9e325772..e9295dbe 100644 --- a/src/questdb/auth/_store.py +++ b/src/questdb/auth/_store.py @@ -31,10 +31,13 @@ token — one silent token-endpoint round-trip — instead of re-prompting. :class:`FileTokenStore` is the default implementation: one plaintext JSON file -per identity, protected at rest by file permissions (``0600`` file, ``0700`` -directory) rather than encryption — the same posture ``gcloud``, ``aws`` and -``gh`` take. Supply your own :class:`TokenStore` (backed by an OS keychain, a -KMS, or a vault) to encrypt the refresh token at rest. +per identity, protected at rest by file permissions rather than encryption — the +same posture ``gcloud``, ``aws`` and ``gh`` take. The token *content* is +protected by the ``0600`` file mode (set atomically at creation by ``mkstemp``); +the ``0700`` directory is defense-in-depth (listing/replacement resistance, +re-asserted best-effort and dropped silently when the directory is owned by +another principal). Supply your own :class:`TokenStore` (backed by an OS +keychain, a KMS, or a vault) to encrypt the refresh token at rest. The on-disk format (directory, file name, JSON schema, atomic-write and lock-file protocols) is a deliberately **language-neutral contract** so the Java @@ -513,9 +516,11 @@ def at_default_location(cls) -> 'FileTokenStore': def load(self, key: TokenStoreKey) -> Optional[PersistedToken]: """Load and identity-verify the persisted token for ``key`` (see - :meth:`TokenStore.load`). Returns ``None`` — never raises — for a - missing, oversized, unreadable, non-regular, or wrong-identity file, so - an unusable entry falls back to a refresh / interactive sign-in.""" + :meth:`TokenStore.load`). Returns ``None`` for a missing, oversized, + unreadable, non-regular, or wrong-identity file, so an unusable entry + falls back to a refresh / interactive sign-in; raises + :class:`~questdb.auth.OidcError` only on a genuine I/O or permission + error (e.g. ``EACCES``/``EIO``), which re-parsing cannot recover.""" path = self._token_file(key) try: st = os.stat(path) @@ -620,6 +625,29 @@ def clear(self, key: TokenStoreKey) -> None: except OSError as e: raise OidcError( f'could not remove the OIDC token store file: {e}') from e + # Also sweep any orphaned sibling temp files holding a now-forgotten + # credential (see _sweep_orphan_temps); best-effort, never fails clear(). + self._sweep_orphan_temps(key) + + def _sweep_orphan_temps(self, key: TokenStoreKey) -> None: + # Remove any leftover `*.tmp` files for this identity. save() writes + # the plaintext token into such a temp before its atomic rename; a hard + # crash (SIGKILL / power loss) in that window leaves it behind — 0600 and + # never read back by load(), but it still holds a refresh token the user + # is now asking to forget. os.replace consumes the temp on a normal save, + # so this usually finds nothing. List by prefix (not glob) so a glob + # metacharacter in the directory path is a non-issue. Best-effort: a + # sweep failure must not fail clear(). + prefix = key.hash() + try: + with os.scandir(self._directory) as entries: + names = [e.name for e in entries] + except OSError: + return + for name in names: + if name.startswith(prefix) and name.endswith('.tmp'): + with contextlib.suppress(OSError): + os.remove(os.path.join(self._directory, name)) def in_lock(self, key: TokenStoreKey, action: Callable[[], Any]) -> Any: """Run ``action`` under a per-identity ``O_CREAT|O_EXCL`` lock file (see @@ -638,15 +666,16 @@ def in_lock(self, key: TokenStoreKey, action: Callable[[], Any]) -> Any: # Could not prepare the lock directory or file; run without the lock. # Atomic replacement still keeps every reader CONSISTENT (no torn # read), but a degraded lock-free refresh is no longer SERIALISED - # against a peer, so for this one refresh two cross-process races are - # unguarded: (1) a rotating-refresh-token race (two processes each - # refresh and one rotation is lost), and (2) a clear-vs-save race — - # because the clear()-generation re-check that normally guards a save - # is process-local (it lives in this process's in-memory cache), a - # save() here can re-create a file another process just clear()ed, - # resurrecting a cleared token until the next clear(). Both are - # best-effort by design; closing them across processes would need an - # on-disk epoch that save re-checks under the lock. + # against a peer, so for this one refresh a rotating-refresh-token + # race is unguarded (two processes each refresh and one rotation is + # lost). A second cross-process race — clear-vs-save, where a save() + # re-creates a file another process just clear()ed — is NOT specific + # to this degraded branch: the clear()-generation re-check that guards + # a save is process-local (it lives in this process's in-memory + # cache), so a concurrent cross-process clear() is undone even when + # the lock IS held. Both are best-effort by design; closing them + # across processes would need an on-disk epoch that save re-checks + # under the lock. held = False try: return action() @@ -705,8 +734,9 @@ def _ensure_directory(self) -> None: self._restrict_to_owner() def _restrict_to_owner(self) -> None: - # Best-effort: the at-rest protection of the plaintext token files is - # exactly these owner-only directory permissions. On a non-POSIX + # Best-effort, defense-in-depth: the token CONTENT is protected by the + # 0600 file mode (mkstemp sets it atomically); these owner-only directory + # permissions add listing/replacement resistance on top. On a non-POSIX # filesystem (Windows) POSIX modes do not apply, so fall back to the # directory's inherited ACL and warn once. if os.name != 'posix': @@ -715,7 +745,9 @@ def _restrict_to_owner(self) -> None: try: os.chmod(self._directory, 0o700) except OSError: - # The directory is not ours to chmod: keep the existing permissions. + # The directory is not ours to chmod (owned by another principal): + # keep the existing permissions. Each token file's own 0600 mode still + # protects its content regardless of the directory mode. pass def _fsync_directory(self) -> None: @@ -887,7 +919,13 @@ def _holder_bytes(self) -> bytes: def _is_stale(self, lock: str) -> bool: try: - mtime = os.stat(lock).st_mtime + # lstat (not stat): judge a symlink by the LINK's own mtime, never + # its target's. A co-tenant who plants a symlink at the lock path + # (its O_EXCL create already refuses to follow one) and keeps the + # TARGET fresh cannot thereby keep the lock looking live — with lstat + # the link's own age is what counts, so it is stolen once it ages + # out. For a regular-file lock (what we create) lstat == stat. + mtime = os.lstat(lock).st_mtime except OSError: # Can't determine the age, so don't steal. (A pre-planted dangling # symlink at the lock path lands here forever, so that identity @@ -905,7 +943,11 @@ def _is_stale(self, lock: str) -> bool: # then untrustworthy and the lock may well be live, so treat it as fresh # (do not steal) rather than break a live holder's lock; a genuinely # abandoned lock is re-judged stale on a later poll once the clock catches - # up to it. + # up to it. A co-tenant with write access to the store dir can abuse this + # by future-dating a lock they plant (os.utime) to pin the identity to + # lock-free coordination indefinitely — but that is integrity-safe (the + # atomic write still holds) and no worse than the peer simply holding or + # deleting files, which their write access already permits. if elapsed < 0: return False return elapsed > self._lock_stale diff --git a/test/test_auth.py b/test/test_auth.py index 8405a224..cf02c6a6 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -1775,6 +1775,22 @@ def test_refresh_without_id_token_non_interactive_does_not_loop(self): self.assertEqual(self.state.refresh_requests, 1) self.assertEqual(self.state.device_requests, 0) + def test_refresh_without_access_token_falls_back_to_device_flow(self): + # m8: the symmetric case of the groups_in_token=True test above. + # groups_in_token=False, but the IdP's refresh omits the access_token + # (the kind _select returns in this mode): the refresh is unusable, so + # fall back to the interactive device flow rather than caching it and + # looping. + auth = self.make_auth(groups_in_token=False) + self._seed_expired(auth) + self.state.refresh_response = (200, { + 'id_token': ID_TOKEN, 'token_type': 'Bearer', + 'expires_in': 3600}) # no access_token + token = auth.token() + self.assertEqual(token, ACCESS_TOKEN) # from the device flow + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual(self.state.device_requests, 1) # fell back + def test_cached_token_missing_required_kind_is_refreshed(self): # A cached, non-expired token that lacks the required kind (here: # access_token in non-groups mode) must not pass the cache gate and @@ -3273,6 +3289,27 @@ def test_pg_module_missing_chains_cause(self): _pg_module() self.assertIsInstance(cm.exception.__cause__, ImportError) + def test_pg_module_selection_order(self): + # m8: _pg_module prefers psycopg v3, else psycopg2, else raises a chained + # ImportError. Force the module presence via sys.modules so the ordering + # is exercised regardless of which driver is actually installed (the + # missing-driver tests above skip when one is present). A None entry in + # sys.modules makes `import ` raise ImportError. + from questdb.auth._adapters import _pg_module + fake_v3 = types.ModuleType('psycopg') + fake_v2 = types.ModuleType('psycopg2') + with mock.patch.dict(sys.modules, + {'psycopg': fake_v3, 'psycopg2': fake_v2}): + self.assertIs(_pg_module(), fake_v3) # both present -> v3 wins + with mock.patch.dict(sys.modules, + {'psycopg': None, 'psycopg2': fake_v2}): + self.assertIs(_pg_module(), fake_v2) # only v2 -> fall back + with mock.patch.dict(sys.modules, + {'psycopg': None, 'psycopg2': None}): + with self.assertRaises(ImportError) as cm: # neither -> chained error + _pg_module() + self.assertIsInstance(cm.exception.__cause__, ImportError) + class TestConfigHelpers(unittest.TestCase): def test_as_bool_variants(self): @@ -4639,6 +4676,21 @@ def test_get_json_non_json_2xx_raises_oidc_error(self): b + '/.well-known/openid-configuration', timeout=5) self.assertEqual(cm.exception.status, 200) # status attached + def test_invalid_utf8_json_body_raises_oidc_error(self): + # m8: HttpResponse.json() decodes the body as utf-8 with NO + # errors='replace' (unlike text()), so an invalid-UTF-8 2xx body raises + # UnicodeDecodeError. Both call sites (post_form, get_json) must catch it + # and surface a typed error rather than let a raw UnicodeDecodeError + # escape the contract. (The existing 0x80..0x82 test targets the JWT + # base64 payload, not the HTTP body decode.) + from questdb.auth import _http + with _raw_response_server(200, 'application/json', b'\xff\xfe') as b: + with self.assertRaises(OidcError): + _http.post_form(b + '/token', {'a': 'b'}, timeout=5) + with _raw_response_server(200, 'application/json', b'\xff\xfe') as b: + with self.assertRaises(OidcConfigError): + _http.get_json(b + '/settings', timeout=5) + class TestRendererSecurity(unittest.TestCase): """The Jupyter prompt must never turn an IdP-supplied URL into a @@ -5664,6 +5716,15 @@ def test_directory_at_token_path_ignored(self): os.mkdir(self._file()) self.assertIsNone(self.store.load(self.key)) + @unittest.skipUnless(hasattr(os, 'mkfifo'), 'FIFO support') + def test_fifo_at_token_path_ignored(self): + # m8: a non-regular file other than a directory — e.g. a FIFO — planted + # at the token path is likewise not a usable entry (the S_ISREG guard + # covers every non-regular type). load() must return None; because it + # stats before opening, it never blocks reading the FIFO. + os.mkfifo(self._file()) + self.assertIsNone(self.store.load(self.key)) + def test_deeply_nested_json_file_ignored(self): # The token file is attacker-writable. A deeply-nested JSON document # (well under the size cap) makes json.loads raise RecursionError, which From c393dd92a878984140f0863a9c2e791d605f887e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 2 Jul 2026 12:08:27 +0100 Subject: [PATCH 101/104] fix(auth): reject multi-host PG hosts and truncated HTTP bodies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two credential-path hardening fixes found in review. PG host guard (_adapters.py): replace the deny-list of conninfo metacharacters with a positive allow-list of the characters a real hostname / IPv4 / IPv6-literal can contain. The deny-list missed ',' — a libpq MULTI-HOST separator ('host=a,b' tries both), so a tampered URL could steer the connection, and the '_sso' token sent as the password, to an attacker host that merely reads next to the real one — and '/', which libpq treats as a Unix-socket directory. The allow-list closes the whole redirection/injection class at once. Underscore stays allowed, matching _render._SAFE_HOST_RE. HTTP body read (_http.py): read1() (used for the chunked-dribble watchdog) does not enforce Content-Length — on a body that declares N bytes but delivers fewer then closes, it returns the short data then a clean EOF, so a truncated (yet still JSON-parseable) token / config response was handed back as a complete 200. http.client leaves the owed count on resp.length, so treat a truthy length at EOF as a truncation and raise OidcNetworkError. Chunked bodies (length None) and non -http.client readers (no length attr) are unaffected. Add regression tests for both (comma/unix-socket/userinfo host rejection; truncated Content-Length via a unit stub and a real socket) and update the cross-reference comments to the renamed _LEGAL_HOST_RE. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_adapters.py | 42 +++++++----- src/questdb/auth/_discovery.py | 5 +- src/questdb/auth/_http.py | 20 ++++++ src/questdb/auth/_render.py | 2 +- test/test_auth.py | 117 ++++++++++++++++++++++++++++++++- 5 files changed, 167 insertions(+), 19 deletions(-) diff --git a/src/questdb/auth/_adapters.py b/src/questdb/auth/_adapters.py index d4b9a4bc..08b739de 100644 --- a/src/questdb/auth/_adapters.py +++ b/src/questdb/auth/_adapters.py @@ -43,16 +43,25 @@ _DEFAULT_PG_PORT = 8812 _DEFAULT_DATABASE = 'qdb' -# Reject connection-string delimiters (';', '='), whitespace/control chars, and -# '%' in the host: a real hostname / IPv4 / IPv6-literal never has them, so their -# presence means a tampered URL trying to inject PG connection parameters -# (psycopg turns its kwargs into a libpq conninfo string). ':' is allowed — IPv6 -# literals contain it, and the PG drivers take host and port separately. '%' -# would only appear as an IPv6 zone-id (e.g. 'fe80::1%eth0'), meaningful only for -# a link-local address on the local machine and never for reaching a remote -# QuestDB; rejecting it keeps the guard a strict plain-host allowlist -# (defense-in-depth — '%' is not itself a conninfo delimiter). -_ILLEGAL_HOST_CHARS = re.compile(r'[\x00-\x20\x7f;=%]') +# Constrain the PG-wire host to exactly the characters a real hostname / IPv4 / +# IPv6-literal can contain — ASCII letters, digits, '.', '-', '_', and ':' (which +# an IPv6 literal carries once urlparse has stripped its brackets; the PG drivers +# take host and port separately) — and reject everything else. A positive +# allow-list (rather than a deny-list of known-bad chars) closes the WHOLE class +# of libpq conninfo-injection / connection-redirection vectors at once, because +# psycopg turns its kwargs into a libpq conninfo string: +# * ',' is the libpq MULTI-HOST separator ('host=a,b' tries both a and b), so a +# tampered URL could steer the connection — and the '_sso' token sent as the +# password — to an attacker host that merely reads next to the real one; +# * a '/' makes libpq treat the value as a Unix-socket DIRECTORY, redirecting +# to a local socket; +# * ';', '=', whitespace and control chars are conninfo delimiters; +# * '%' is only ever an IPv6 zone-id ('fe80::1%eth0'), meaningful for an on-host +# link-local address, never for reaching a remote QuestDB. +# None of these appears in a genuine host, so this is the choke point that keeps a +# malformed/tampered URL from redirecting the PG connection. Mirrors the host +# hygiene in _render._SAFE_HOST_RE and _discovery's authority checks. +_LEGAL_HOST_RE = re.compile(r'\A[A-Za-z0-9._:-]+\Z') def _pg_module(): @@ -96,12 +105,15 @@ def _require_host(url: str, host: Optional[str] = None) -> str: # driver (and any junk inside the brackets is still caught). if resolved.startswith('[') and resolved.endswith(']') and len(resolved) > 2: resolved = resolved[1:-1] - if _ILLEGAL_HOST_CHARS.search(resolved): + if not _LEGAL_HOST_RE.match(resolved): raise OidcConfigError( - f'The QuestDB host {resolved!r} contains an illegal character ' - "(';', '=', '%', whitespace or a control character). A hostname or " - 'IP address never does; this indicates a malformed or tampered URL. ' - '(Such a host could otherwise inject PG connection parameters.)') + f'The QuestDB host {resolved!r} contains an illegal character. A ' + 'hostname or IP address contains only letters, digits, ".", "-", ' + '"_" and ":" (IPv6); anything else — "," (a libpq multi-host ' + 'separator), "/" (a Unix-socket path), ";", "=", "%", whitespace or ' + 'a control character — indicates a malformed or tampered URL and ' + 'could otherwise redirect the PG connection or inject connection ' + 'parameters.') return resolved diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 2a36bd39..fb48e223 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -313,8 +313,9 @@ def _endpoint_path_under_issuer(endpoint: str, issuer: str) -> bool: # ``%`` (percent-encoding, or an IPv6 zone-id such as ``fe80::1%eth0`` — an # on-host link-local artifact, never a way to reach a remote IdP), so reject them # — fail closed — mirroring the host hygiene already enforced in -# ``_adapters._ILLEGAL_HOST_CHARS`` and ``_render._SAFE_HOST_RE`` (both of which -# also reject ``%``). (Non-ASCII is checked with ``str.isascii`` in the function, +# ``_adapters._LEGAL_HOST_RE`` and ``_render._SAFE_HOST_RE`` (both positive +# allow-lists, so both also reject ``%``). (Non-ASCII is checked with +# ``str.isascii`` in the function, # not this regex.) _UNSAFE_AUTHORITY_RE = re.compile(r'[\\\s\x00-\x1f\x7f%]') diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index e4d0b479..9d6f96c5 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -383,6 +383,26 @@ def _read_body(resp: Any, *, max_bytes: int, deadline: float) -> bytes: raise OidcNetworkError( 'Timed out reading the response body; the server is too ' 'slow or is dribbling data.') + # read1() (which we read through above) does NOT enforce + # Content-Length: on a body that DECLARES N bytes but delivers + # fewer then closes, it returns the short data and then this clean + # b'' EOF, with no exception — so a truncated response would + # otherwise be returned as a complete 200 (the plain read() would + # raise IncompleteRead on the same input). http.client leaves the + # still-owed byte count on `resp.length` — None for a chunked body + # (no declared length; the deadline watchdog bounds that path), 0 + # once a Content-Length body is fully delivered, and > 0 when it + # ended short. A truthy length at EOF therefore means the declared + # body was NOT fully received, so surface it as the network error + # it is rather than hand back a silently-truncated (and possibly + # still parseable) token / config JSON. A stream without a + # `length` attribute (getattr -> None) is treated as complete, so + # a non-http.client body reader degrades to the prior behaviour. + if getattr(resp, 'length', None): + raise OidcNetworkError( + 'The response body ended before its declared ' + 'Content-Length was received (truncated response); ' + 'refusing to treat a partial body as complete.') return b''.join(chunks) total += len(chunk) if total > max_bytes: diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index ed57a168..0c45e02e 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -132,7 +132,7 @@ def _verification_uri_complete(resp: Dict[str, Any]) -> Optional[str]: # confusable (e.g. a Cyrillic look-alike, or the fullwidth solidus ``U+FF0F``), # a stray control char, or a ``%`` (percent-encoding, or an IPv6 zone-id — # neither of which a remote verification host legitimately needs, matching the -# hygiene in ``_adapters._ILLEGAL_HOST_CHARS``) — can misrepresent the real +# hygiene in ``_adapters._LEGAL_HOST_RE``) — can misrepresent the real # destination host, so such a URL is never made clickable/auto-opened. _SAFE_HOST_RE = re.compile(r'\A[a-z0-9._:-]+\Z') diff --git a/test/test_auth.py b/test/test_auth.py index cf02c6a6..3c073b8d 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -3229,6 +3229,35 @@ def test_require_host_with_conf_metachars_rejected(self): _require_host('https://db.example.com:9000', 'questdb.example.com'), 'questdb.example.com') + def test_require_host_rejects_multihost_comma_and_unix_socket(self): + # Regression: the host guard is a positive allow-list, so characters that + # are NOT conninfo delimiters (which the old deny-list caught) but ARE + # libpq-meaningful must still be rejected — otherwise a tampered URL could + # redirect the PG connection (and the '_sso' token sent as the password): + # * ',' is the libpq MULTI-HOST separator ('host=a,b' tries both), and + # urlparse keeps it in .hostname; + # * a leading '/' makes libpq read the value as a Unix-socket directory; + # * '@' (userinfo) never belongs in a bare host. + for bad in ('https://good.questdb.com,evil.attacker.com:9000', + 'https://good.questdb.com,evil.attacker.com'): + with self.subTest(url=bad): + with self.assertRaises(OidcConfigError): + _require_host(bad) + for bad_host in ('good.questdb.com,evil.attacker.com', # multi-host + '[good,evil]', # comma smuggled in brackets + '/var/run/postgresql', # Unix-socket dir + '/tmp', + 'good@evil'): # userinfo + with self.subTest(host=bad_host): + with self.assertRaises(OidcConfigError): + _require_host('https://db.example.com:9000', bad_host) + # A hostname carrying an underscore (accepted in practice, and by the + # render / discovery host checks) is still allowed — the allow-list must + # not over-reject a legitimate host. + self.assertEqual( + _require_host('https://db:9000', 'my_host.internal'), + 'my_host.internal') + def test_require_host_unbrackets_explicit_ipv6(self): # m2: psycopg / SQLAlchemy take a BARE address. The URL-derived path is # already unbracketed by urlparse, but an explicit host="[::1]" override @@ -3672,7 +3701,7 @@ def test_percent_authority_rejected(self): # artifact, never a way to reach a remote IdP) or percent-encoding (which # urlparse keeps in .hostname but a resolver/transport may decode, # diverging the validated host from the connected one). This mirrors the - # host hygiene in _adapters._ILLEGAL_HOST_CHARS and _render._SAFE_HOST_RE, + # host hygiene in _adapters._LEGAL_HOST_RE and _render._SAFE_HOST_RE, # which both reject '%' too. from questdb.auth._discovery import _reject_confusable_authority for url in ('https://[fe80::1%25eth0]/token', # IPv6 zone-id (%25 == %) @@ -4252,6 +4281,92 @@ def test_read_body_rejects_oversized(self): with self.assertRaises(OidcNetworkError): _read_body(resp, max_bytes=100, deadline=1e18) + def test_read_body_rejects_truncated_content_length(self): + # Regression: read1() (which _read_body reads through, for the chunked- + # dribble watchdog) does NOT enforce Content-Length — on a body that + # DECLARES N bytes but delivers fewer then EOFs, it returns the short data + # then a clean b'', with no exception. Without the guard, that truncated + # (yet still JSON-parseable) body was handed back as a complete 200. Since + # http.client leaves the still-owed count on resp.length, _read_body must + # treat a truthy length at EOF as a truncation and raise. + from questdb.auth._http import _read_body + + class _LenResp: + # Faithfully mimics http.client.HTTPResponse on a Content-Length body: + # read1(n) yields up to n buffered bytes and DECREMENTS the owed + # count, so .length hits 0 exactly when the declared body is fully + # delivered and stays > 0 if the peer closed early. + def __init__(self, body, declared): + self._body = body + self.length = declared + + def read1(self, n): + if not self._body: + return b'' + chunk, self._body = self._body[:n], self._body[n:] + self.length -= len(chunk) + return chunk + + truncated = _LenResp(b'{"access_token":"REAL"}', declared=5000) + with self.assertRaises(OidcNetworkError): + _read_body(truncated, max_bytes=10 ** 6, deadline=1e18) + + # A body that delivers exactly its declared length drains .length to 0 and + # must still be accepted — the guard must not over-reject. + body = b'{"access_token":"REAL"}' + complete = _LenResp(body, declared=len(body)) + self.assertEqual( + _read_body(complete, max_bytes=10 ** 6, deadline=1e18), body) + + # A chunked body has no declared length (no `length` attribute, so + # getattr -> None), so the guard must NOT fire — the deadline watchdog + # bounds that path instead. + chunked = _ChunkStream(b'{"a":', b'1}') + self.assertEqual( + _read_body(chunked, max_bytes=10 ** 6, deadline=1e18), b'{"a":1}') + + def test_request_rejects_truncated_content_length_body(self): + # Regression against the REAL socket stack (the _LenResp unit test above + # cannot catch a mismatch with real http.client behaviour). A server that + # DECLARES a large Content-Length but sends a short, valid-JSON body then + # closes must NOT be handed back as a complete 200 — request() must raise + # OidcNetworkError rather than let a hostile/flaky peer pass off a + # truncated token / config response as whole. + import socket + from questdb.auth import _http + + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(('127.0.0.1', 0)) + srv.listen(1) + port = srv.getsockname()[1] + + def serve(): + try: + conn, _ = srv.accept() + except OSError: + return + try: + conn.recv(65536) # consume the request line/headers + # Declare 5000 bytes, send a ~40-byte valid-JSON body, then close + # — a clean early EOF (not a dribble), well inside the deadline. + conn.sendall( + b'HTTP/1.1 200 OK\r\n' + b'Content-Type: application/json\r\n' + b'Content-Length: 5000\r\n\r\n' + b'{"access_token":"REAL","refresh_token":"R"}') + finally: + conn.close() + + server_thread = threading.Thread(target=serve, daemon=True) + server_thread.start() + try: + with self.assertRaises(OidcNetworkError): + _http.request('GET', f'http://127.0.0.1:{port}/x', timeout=5.0) + finally: + srv.close() + server_thread.join(2.0) + def test_read_body_aborts_on_slow_dribble(self): # A steady dribble that never trips the per-read socket timeout must # still abort once the whole-read wall-clock deadline passes — the gap From 5a70301d5ed1906dc3b82c08bde1c1e1395494ea Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 2 Jul 2026 13:48:47 +0100 Subject: [PATCH 102/104] fix(auth): close credential-routing bypass and two token-store gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolve_config tracked endpoint provenance by identity (`is not None`) but chose the value by truthiness (`or _resolve_endpoint(...)`), so an empty-string override — a common "unset" sentinel like token_endpoint=os.environ.get("TOK", "") — was stamped caller-explicit (trusted) while its value came from the untrusted /settings response. That false provenance skipped the plaintext-channel guard and both issuer pins, letting a tampered /settings route the device code and refresh token to an attacker origin. Normalize empty->None up front so an empty override behaves exactly like an omitted one. FileTokenStore.load() stat'd the path then did a blocking open(): a co-tenant swapping the regular file for a FIFO in that window hung the open forever, pinning the acquisition-lock-holding thread. Open O_NONBLOCK and re-validate the opened fd with fstat, closing the TOCTOU. _try_refresh_coordinated's post-failure fall-through re-refreshed with the stale tokens argument; a custom TokenStore whose in_lock raised after its action already rotated the refresh token would replay the spent token and trip the IdP's reuse detection. Re-consult the freshest in-memory token first, and document the in_lock contract OidcDeviceAuth relies on. The bundled FileTokenStore was already immune to both. Adds regression tests for all three. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 22 +++++-- src/questdb/auth/_discovery.py | 18 ++++++ src/questdb/auth/_store.py | 56 +++++++++++++++-- test/test_auth.py | 111 +++++++++++++++++++++++++++++++++ 4 files changed, 199 insertions(+), 8 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 0f4dd279..19991709 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -1101,10 +1101,24 @@ def _try_refresh_coordinated( self._warn_persistence('lock', e) finally: self._store_lock_held = False - # Reached only when in_lock raised a non-network store failure above: the - # flag is now cleared, so this lock-free refresh's own persist takes its - # normal lock-acquiring path. - return self._try_refresh_locally(tokens, generation) + # Reached only when in_lock raised a NON-network store failure above (a + # custom store's lock backend; the bundled FileTokenStore degrades + # internally and never reaches here). The in-lock action may ALREADY have + # refreshed — and, on a rotating IdP, CONSUMED our refresh token — before + # in_lock raised (e.g. a custom store whose lock RELEASE failed after + # action() succeeded). So re-consult our own freshest view rather than + # replay the now-stale `tokens` argument: if that refresh landed a valid + # token, return it as-is (it is already committed to self._tokens and, + # generation permitting, the shared cache); otherwise refresh with the + # freshest tokens we hold — never the stale argument, whose refresh_token + # may be spent (replaying it trips the IdP's refresh-token reuse detection + # and revokes the just-minted token). The flag is now cleared, so this + # fall-through's own persist takes its normal lock-acquiring path. + current = self._tokens if self._tokens is not None else tokens + if (current is not None and current.is_valid(self._now()) + and self._has_required_token(current)): + return current + return self._try_refresh_locally(current, generation) def _try_refresh_locally( self, tokens: TokenSet, generation: int) -> Optional[TokenSet]: diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index fb48e223..73468565 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -494,6 +494,24 @@ def resolve_config( from QuestDB ``/settings`` (if ``questdb_url`` is given) and, as a last resort for the device endpoint, the IdP discovery document. """ + # Normalize an empty-string override to absent BEFORE any provenance is + # derived below. Provenance is tracked by identity (`... is not None`) while + # the value is chosen by truthiness (`... or _resolve_endpoint(...)`), so an + # empty string — the natural sentinel a caller layer produces for an unset + # override, e.g. `token_endpoint=os.environ.get("TOK", "")` — would otherwise + # be stamped caller-explicit (trusted) while its VALUE is taken from the + # untrusted /settings response. That false provenance disables every + # /settings guard at once (the plaintext-channel pin, the issuer-origin pin + # and the issuer-path pin all skip a "caller-explicit" endpoint), silently + # routing the device-code and refresh-token POSTs to whatever endpoint a + # tampered /settings advertises. Collapsing empty to None here makes + # `... is not None` a faithful "explicit and usable" signal, so an empty + # override behaves exactly like an omitted one. (The direct OidcDeviceAuth + # constructor already rejects an empty endpoint; this closes the from_questdb + # path, which consumes these before any non-empty check.) + token_endpoint = token_endpoint or None + device_authorization_endpoint = device_authorization_endpoint or None + issuer = issuer or None cfg: Dict[str, Any] = {} if questdb_url: cfg = fetch_settings( diff --git a/src/questdb/auth/_store.py b/src/questdb/auth/_store.py index e9295dbe..5e919ef8 100644 --- a/src/questdb/auth/_store.py +++ b/src/questdb/auth/_store.py @@ -405,6 +405,20 @@ def in_lock(self, key: TokenStoreKey, action: Callable[[], Any]) -> Any: ``action`` re-enters this same store (it calls :meth:`load` and :meth:`save` on it). An implementation that cannot acquire the lock should run ``action`` anyway (degrade) rather than fail a sign-in. + + **Contract ``OidcDeviceAuth`` relies on.** ``action`` MUST be invoked + exactly once, **synchronously, on the calling thread**, with the lock + held for the whole call, and ``in_lock`` MUST NOT raise *after* ``action`` + has run — swallow a release failure rather than propagate it. + ``OidcDeviceAuth`` reads an instance flag while ``action`` runs to know + the disk lock is held (so the save it performs writes inline rather than + recursively re-locking); a store that runs ``action`` on another thread + could make that flag observed true while the lock is not held, racing a + concurrent :meth:`clear`. And on a rotating IdP, a refresh inside + ``action`` may have already consumed the refresh token by the time a late + release fails, so raising then would (absent the caller's post-failure + re-consult) replay a spent token and get the freshly minted one revoked. + The bundled :class:`FileTokenStore` satisfies this contract. """ return action() @@ -547,18 +561,52 @@ def load(self, key: TokenStoreKey) -> Optional[PersistedToken]: # rather than read it into memory. if st.st_size <= 0 or st.st_size > _MAX_FILE_BYTES: return None + # Open O_NONBLOCK and re-validate the OPENED fd, not the earlier stat: a + # hostile co-tenant with write access to the store dir could swap the + # regular file for a FIFO between the stat above and this open, and a + # blocking open() of a FIFO hangs forever waiting for a writer — pinning + # the calling thread, which may hold the acquisition lock (load() runs + # under it, including inside the store's cross-process in_lock). That is + # the very "peer pins the lock-holding thread" failure mode the HTTP + # layer builds watchdogs against, so guard it here too. O_NONBLOCK makes + # the FIFO open return at once; fstat on the fd — the object actually + # opened, closing the stat->open TOCTOU — then rejects a non-regular or + # resized file. O_NONBLOCK is a no-op on a regular file; Windows lacks it + # (and has no FIFO-at-path exposure), so it degrades to a plain open. try: - with open(path, 'rb') as f: - data = f.read(_MAX_FILE_BYTES + 1) + fd = os.open(path, os.O_RDONLY | getattr(os, 'O_NONBLOCK', 0)) except FileNotFoundError: return None except IsADirectoryError: - # The regular file became a directory between the stat above and - # this open (a TOCTOU); treat it as no usable entry, as above. + # Became a directory between the stat above and this open (a TOCTOU). return None + except OSError as e: + if e.errno in (errno.ELOOP, errno.ENOTDIR): + return None + raise OidcError( + f'could not read the OIDC token store file: {e}') from e + # os.fdopen takes ownership of fd and closes it on the `with` exit; track + # whether ownership transferred (mirrors save()) so an early reject or an + # fdopen failure closes fd exactly once in the finally. + fd_owned = False + try: + fst = os.fstat(fd) + # Re-check on the fd: a non-regular file (a FIFO / device / directory + # swapped in after the stat) or one that grew past the cap since the + # stat is not a usable entry — return None rather than read it. + if not stat.S_ISREG(fst.st_mode) or not ( + 0 < fst.st_size <= _MAX_FILE_BYTES): + return None + with os.fdopen(fd, 'rb') as f: + fd_owned = True + data = f.read(_MAX_FILE_BYTES + 1) except OSError as e: raise OidcError( f'could not read the OIDC token store file: {e}') from e + finally: + if not fd_owned: + with contextlib.suppress(OSError): + os.close(fd) if len(data) > _MAX_FILE_BYTES: return None return self._parse_and_verify(key, data) diff --git a/test/test_auth.py b/test/test_auth.py index 3c073b8d..aab971b1 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -43,6 +43,7 @@ import json import os import shutil +import stat import subprocess import sys import tempfile @@ -2484,6 +2485,41 @@ def test_issuer_path_scope_rejects_dot_segment_traversal(self): issuer=kc + '/prod') self.assertIn('issuer', str(cm.exception).lower()) + def test_empty_string_endpoint_override_does_not_launder_settings(self): + # C1: an empty-string endpoint override is a common "unset" sentinel + # (e.g. token_endpoint=os.environ.get("QDB_TOKEN_ENDPOINT", "")). It must + # behave exactly like an OMITTED (None) override -- never be treated as + # caller-explicit (trusted) while its VALUE is silently taken from the + # untrusted /settings response. Were it treated as explicit, the + # provenance flags would stamp the /settings-advertised (attacker) + # endpoint as caller-supplied and skip BOTH the plaintext-channel guard + # and the issuer origin/path pins, routing the device code / refresh token + # to the attacker. resolve_config normalizes empty->None up front so this + # can't happen. + # Plaintext channel, no pin: the guard must still fire for any empty combo + # (both empty, or one empty + one omitted). + for tok, dev in (('', ''), ('', None), (None, '')): + with self.assertRaises(OidcConfigError) as cm: + self._resolve(self._TAMPERED, + questdb_url='http://qdb.internal.example:9000', + insecure=True, + token_endpoint=tok, + device_authorization_endpoint=dev) + self.assertIn( + 'issuer', str(cm.exception), + f'empty override ({tok!r}, {dev!r}) bypassed the plaintext guard') + # https channel WITH an issuer pinned to a DIFFERENT origin: an empty + # override must not skip the issuer-origin pin the way a genuine explicit + # endpoint (intentionally) does -- the /settings attacker endpoints stay + # pinned and rejected. + with self.assertRaises(OidcConfigError) as cm: + self._resolve(self._TAMPERED, + questdb_url='https://qdb.example.com:9000', + issuer='https://idp.good.example', + token_endpoint='', + device_authorization_endpoint='') + self.assertIn('issuer', str(cm.exception).lower()) + class TestConcurrency(AuthTestBase): def test_valid_cached_token_does_not_block_during_signin(self): @@ -5840,6 +5876,37 @@ def test_fifo_at_token_path_ignored(self): os.mkfifo(self._file()) self.assertIsNone(self.store.load(self.key)) + @unittest.skipUnless( + hasattr(os, 'mkfifo') and os.name == 'posix', 'FIFO support') + def test_fifo_swapped_in_after_stat_does_not_hang(self): + # TOCTOU: the S_ISREG/size guards run on load()'s INITIAL os.stat, but a + # hostile co-tenant with write access to the store dir could swap the + # regular file for a FIFO between that stat and the open. A blocking + # open() of a FIFO hangs forever waiting for a writer, pinning the calling + # thread (which may hold the acquisition lock). Simulate the swap by + # making the initial stat report a plausible REGULAR file while the real + # path is a FIFO: load() must open O_NONBLOCK and reject it via the fstat + # re-check on the opened fd, returning None promptly rather than hanging. + os.mkfifo(self._file()) + fake_reg = os.stat_result(( + stat.S_IFREG | 0o600, 0, 0, 1, os.getuid(), os.getgid(), + 64, 0, 0, 0)) + result = {} + + def run(): + # Patch only os.stat (not os.fstat), so the initial guard sees a + # regular file while the fd-based re-check sees the real FIFO. + with mock.patch('questdb.auth._store.os.stat', + return_value=fake_reg): + result['r'] = self.store.load(self.key) + + t = threading.Thread(target=run, daemon=True) + t.start() + t.join(timeout=10) + self.assertFalse( + t.is_alive(), 'load() hung on a FIFO swapped in after the stat') + self.assertIsNone(result['r']) + def test_deeply_nested_json_file_ignored(self): # The token file is attacker-writable. A deeply-nested JSON document # (well under the size cap) makes json.loads raise RecursionError, which @@ -6307,6 +6374,22 @@ def in_lock(self, key, action): return super().in_lock(key, action) +class _RaiseAfterActionStore(_FakeStore): + """A custom store whose in_lock RAISES after action() has already run. + + Models a real-world custom TokenStore whose lock-RELEASE fails after the + coordinated refresh already succeeded (e.g. a Redis/Consul lock whose + release call throws). The action's refresh has, on a rotating IdP, already + consumed the old refresh token by then, so the caller must NOT re-refresh + with it. Exercises the post-action fall-through in _try_refresh_coordinated. + """ + + def in_lock(self, key, action): + self.in_locks += 1 + action() # the coordinated refresh runs (and rotates the token) here + raise OidcError('simulated lock-release failure after refresh') + + class TestPersistence(AuthTestBase): """OidcDeviceAuth wired to a TokenStore (opt-in persistence).""" @@ -6421,6 +6504,34 @@ def test_transient_refresh_error_propagates_through_lock(self): self.assertEqual(self.state.device_requests, 1) # only the sign-in self.assertGreaterEqual(store.in_locks, 1) + def test_custom_store_lock_failure_after_refresh_does_not_replay_token(self): + # M2: a custom TokenStore whose in_lock RAISES after action() already ran + # the coordinated refresh (a lock-release failure on a rotating IdP). The + # refresh has already consumed REFRESH-1 and minted REFRESH-2, so the + # fall-through must NOT re-refresh with the now-stale REFRESH-1 — replaying + # a spent refresh token trips the IdP's reuse detection and revokes the + # fresh one. It must instead return the already-refreshed token, so + # REFRESH-1 is sent to the token endpoint exactly ONCE. + store = _RaiseAfterActionStore() + clock = FakeClock() + auth = self.make_auth(token_store=store, clock=clock) + auth.token() # sign in; persists REFRESH-1 + self._reset_server_counters() + # Rotate on refresh, so a replay would be observable as a 2nd REFRESH-1. + self.state.refresh_response = (200, { + 'access_token': ACCESS_TOKEN, 'id_token': ID_TOKEN, + 'refresh_token': 'REFRESH-2', 'token_type': 'Bearer', + 'expires_in': 3600}) + clock.wall += 10_000 # expire the access/id token + # Succeeds via the in-lock refresh despite the post-action lock failure. + self.assertEqual(auth.token(), ID_TOKEN) + # The spent REFRESH-1 was sent exactly once — never replayed. (Without the + # fall-through's re-consult it would be sent a second time here.) + self.assertEqual(self.state.refresh_requests, 1) + self.assertEqual( + [f['refresh_token'] for f in self.state.refresh_forms], ['REFRESH-1']) + self.assertEqual(self.state.device_requests, 0) # no needless re-prompt + def test_clear_removes_persisted_entry_and_reprompts(self): store = _FakeStore() auth = self.make_auth(token_store=store) From 797ceae6e7a529b9219bb027b37a353023f932a4 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 2 Jul 2026 16:29:57 +0100 Subject: [PATCH 103/104] fix(auth): keep non-string issuer typed; canonicalize endpoint confirmation M1: a non-string issuer= reached urlparse (via resolve_config's authority vetting and the discovery-URL build) before OidcDeviceAuth.__init__ could type-check it, so from_questdb leaked a raw AttributeError/TypeError instead of an OidcError, escaping the module's typed-error contract. safe_urlparse now maps TypeError/AttributeError too (also covering a non-string QuestDB URL), and resolve_config type-checks issuer early with the same clear message the direct constructor already raises. M2: the /settings-vs-IdP-discovery endpoint confirmation used raw string equality, so a legitimate split-origin IdP (Google/Auth0/Azure) whose two sources spelled one endpoint with a trailing slash, an explicit default port, or a case difference was wrongly rejected by the issuer-origin pin. Compare on the canonical endpoint form instead (reusing _canonical_endpoint, the same canonicalization the cache key and store key use), so a differing query stays a distinct routing target and an unparseable value fails closed rather than raising. Adds regression tests for both. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_discovery.py | 44 +++++++++++++++++++- src/questdb/auth/_http.py | 9 +++- test/test_auth.py | 76 ++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 4 deletions(-) diff --git a/src/questdb/auth/_discovery.py b/src/questdb/auth/_discovery.py index 73468565..101a8853 100644 --- a/src/questdb/auth/_discovery.py +++ b/src/questdb/auth/_discovery.py @@ -43,6 +43,7 @@ from ._errors import OidcConfigError from ._http import get_json, safe_urlparse, _is_loopback +from ._store import _canonical_endpoint # QuestDB /settings keys (see EntPropServerConfiguration.exportConfiguration()). _K_ENABLED = 'acl.oidc.enabled' @@ -188,6 +189,29 @@ def _origin_str(url: str) -> str: return f'{scheme}://{host}:{port}' if port else f'{scheme}://{host}' +def _same_endpoint(url_a: str, url_b: Optional[str]) -> bool: + """True if two endpoint URLs are the SAME credential-routing target. + + Compared on the canonical endpoint form (:func:`_store._canonical_endpoint` + — scheme/host/port/path/query, trailing-slash-insensitive), NOT raw string + equality, so a ``/settings`` endpoint and the IdP discovery document's + spelling of that same endpoint still match despite a trailing slash, an + explicit-vs-default port, or a scheme/host case difference (e.g. ``/token`` + vs ``/token/``). Reuses the token store's endpoint canonicalisation so this + "same endpoint" test makes the same distinctions as the in-memory cache key + and the on-disk store key. A differing query is a different routing target, + so it is kept (not the same). A value that will not parse counts as NOT the + same — the caller then falls through to the issuer-origin/path pins, the + fail-closed direction — rather than raising. + """ + if url_b is None: + return False + try: + return _canonical_endpoint(url_a) == _canonical_endpoint(url_b) + except OidcConfigError: + return False + + def _settings_channel_is_plaintext(questdb_url: str) -> bool: """ True if QuestDB ``/settings`` was fetched over plaintext http to a @@ -512,6 +536,17 @@ def resolve_config( token_endpoint = token_endpoint or None device_authorization_endpoint = device_authorization_endpoint or None issuer = issuer or None + # Type-check issuer HERE, not only in OidcDeviceAuth.__init__: resolve_config + # PARSES it below (_reject_confusable_authority -> safe_urlparse, and the + # discovery-URL build) before __init__ ever runs, and a non-string would make + # urlparse raise a raw AttributeError/TypeError. safe_urlparse now maps those + # too, but check here as well so from_questdb fails with the SAME clear, + # early message as the direct constructor rather than a generic "malformed + # URL". The other caller endpoints are only parsed here when they come from + # /settings (always strings); a caller-explicit non-string endpoint is caught + # by __init__'s isinstance guards, so issuer is the one that needs it here. + if issuer is not None and not isinstance(issuer, str): + raise OidcConfigError('issuer must be a string or None') cfg: Dict[str, Any] = {} if questdb_url: cfg = fetch_settings( @@ -652,7 +687,12 @@ def resolve_config( # steering credentials to a different realm on the same host. # # Both checks are waived for an endpoint the IdP's OWN (authoritative, - # TLS-fetched) discovery document advertised verbatim (url == confirmed_by_idp): + # TLS-fetched) discovery document advertised (_same_endpoint(url, + # confirmed_by_idp), compared on the canonical endpoint form so a trailing + # slash / default port / case difference between the /settings spelling and + # the IdP document's spelling still counts as confirmed — an exact-string + # test wrongly rejected a legitimate split-origin IdP whose two sources + # spelled the one endpoint slightly differently): # that confirms it independently of /settings, exactly as trustworthy as the # pinned IdP. So this runs AFTER discovery — a /settings endpoint the IdP # confirms is accepted consistently under BOTH pins (the issuer-PATH check @@ -676,7 +716,7 @@ def resolve_config( ('device-authorization endpoint', device_authorization_endpoint, device_from_settings, doc_device_endpoint)): - if not from_settings or url == confirmed_by_idp: + if not from_settings or _same_endpoint(url, confirmed_by_idp): # Caller-explicit / IdP-discovered / discovery-confirmed: trusted. continue if _normalized_origin(url) != issuer_origin: diff --git a/src/questdb/auth/_http.py b/src/questdb/auth/_http.py index 9d6f96c5..8f975244 100644 --- a/src/questdb/auth/_http.py +++ b/src/questdb/auth/_http.py @@ -117,12 +117,17 @@ def safe_urlparse(url: str) -> tuple: Both ``urlparse`` (malformed IPv6 literal) and ``ParseResult.port`` (non-integer port) raise a bare ``ValueError``; re-raise as :class:`OidcConfigError` to keep a malformed URL within the error contract. - Returns ``(parts, port)``. + A non-``str``/``bytes`` ``url`` instead makes ``urlparse`` raise a bare + ``TypeError``/``AttributeError`` (it tries to ``.decode`` / slice the value); + map those too, so a non-string endpoint / issuer / QuestDB URL that reaches + here — e.g. via ``resolve_config``, which parses ``issuer`` before + ``OidcDeviceAuth.__init__`` can type-check it — stays within the typed-error + contract rather than escaping raw. Returns ``(parts, port)``. """ try: parts = urllib.parse.urlparse(url) return parts, parts.port - except ValueError as e: + except (ValueError, TypeError, AttributeError) as e: raise OidcConfigError( f'Malformed endpoint URL {url!r}: {e}.') from e diff --git a/test/test_auth.py b/test/test_auth.py index aab971b1..c74b5b44 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -2228,6 +2228,62 @@ def test_settings_endpoint_off_issuer_origin_confirmed_by_discovery(self): self.assertEqual(auth.config.device_authorization_endpoint, 'https://oauth2.idp.example/device') + def test_settings_endpoint_confirmed_despite_trailing_slash(self): + # M2 regression: a split-origin IdP whose /settings token endpoint sits + # off the issuer ORIGIN is confirmed by the IdP's own (TLS-fetched) + # discovery document — but the two sources SPELL the one endpoint slightly + # differently (an explicit :443 and a trailing slash here). The + # confirmation is compared on the canonical endpoint form, not by raw + # string equality, so the trivial spelling difference still counts as + # confirmed and the endpoint is ACCEPTED. An exact-string test wrongly + # rejected this (a real Google / Auth0 / Azure deployment whose /settings + # spelling differs from the IdP document's). + from questdb.auth import _discovery + issuer = 'https://accounts.idp.example' + settings = { + 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': 'https://oauth2.idp.example/token'} + well_known = { # SAME endpoint, spelled with explicit :443 + trailing '/' + 'issuer': issuer, + 'token_endpoint': 'https://oauth2.idp.example:443/token/', + 'device_authorization_endpoint': + 'https://oauth2.idp.example/device'} + with mock.patch.object(_discovery, 'fetch_settings', + return_value=settings), \ + mock.patch.object(_discovery, 'discover_device_endpoint_from_idp', + return_value=well_known): + cfg = _discovery.resolve_config( + questdb_url='https://qdb.example.com:9000', issuer=issuer) + # The /settings spelling is kept as the resolved value; it was accepted + # because the IdP document confirmed the same canonical endpoint. + self.assertEqual(cfg.token_endpoint, 'https://oauth2.idp.example/token') + + def test_settings_endpoint_confirmation_keeps_query_distinct(self): + # M2: the canonical confirmation still treats a DIFFERING QUERY STRING as + # a different credential-routing target — a /settings token endpoint whose + # query differs from the IdP document's is NOT confirmed, so the + # off-issuer-origin pin rejects it. Guards the widened confirmation + # against becoming too loose. + from questdb.auth import _discovery + issuer = 'https://accounts.idp.example' + settings = { + 'acl.oidc.enabled': True, 'acl.oidc.client.id': 'questdb', + 'acl.oidc.token.endpoint': + 'https://oauth2.idp.example/token?tenant=EVIL'} + well_known = { + 'issuer': issuer, + 'token_endpoint': 'https://oauth2.idp.example/token?tenant=good', + 'device_authorization_endpoint': + 'https://oauth2.idp.example/device'} + with mock.patch.object(_discovery, 'fetch_settings', + return_value=settings), \ + mock.patch.object(_discovery, 'discover_device_endpoint_from_idp', + return_value=well_known): + with self.assertRaises(OidcConfigError) as cm: + _discovery.resolve_config( + questdb_url='https://qdb.example.com:9000', issuer=issuer) + self.assertIn('issuer', str(cm.exception).lower()) + def test_settings_off_origin_token_not_confirmed_by_discovery_rejected(self): # The flip side of the confirmed case: /settings advertises an # off-issuer-origin token endpoint that the IdP discovery document does @@ -3783,6 +3839,26 @@ def test_issuer_validated_in_direct_constructor(self): device_authorization_endpoint='https://idp.good/device', scope='openid', issuer=bad) + def test_non_string_issuer_maps_to_config_error(self): + # M1: resolve_config (the from_questdb path) PARSES the issuer — to vet + # its authority and, when needed, build the IdP discovery URL — BEFORE + # OidcDeviceAuth.__init__ can type-check it. urlparse raises a raw + # AttributeError / TypeError on a non-str/bytes value, so without an early + # guard a non-string issuer escaped the module's typed-error contract. + # It must now map to OidcConfigError here too, matching the direct + # constructor (test_issuer_validated_in_direct_constructor). This is the + # only caller kwarg resolve_config parses before __init__ validates it — + # a non-string client_id / endpoint is caught by __init__'s isinstance + # guards, and a /settings-sourced value is always a string. + from questdb.auth._discovery import resolve_config + for bad in (123, b'https://idp', ['https://idp'], 12.5): + with self.assertRaises(OidcConfigError): + resolve_config( + client_id='questdb', + token_endpoint='https://idp.good/token', + device_authorization_endpoint='https://idp.good/device', + issuer=bad) + def test_endpoint_path_under_issuer(self): # M1: segment-aware path containment used to isolate path-based realms. from questdb.auth._discovery import _endpoint_path_under_issuer as under From e50f490cfec2af8caad2d1d8b82925103e4c7dad Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 2 Jul 2026 16:51:15 +0100 Subject: [PATCH 104/104] fix(auth): honor slow_down on a 429, coerce groups flag, strip hidden marks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit m1 (RFC 8628 §3.5): a non-conformant `429 {"error":"slow_down"}` was handled by the transient 429/5xx arm, which runs before the dedicated slow_down arm, so a low Retry-After made the poll interval DROP right after the IdP asked to slow down. The 429/5xx arm now passes at_least_increment when the body carries slow_down, so the interval still increases by at least 5 (floor-bounded either way); a plain 429 keeps honoring Retry-After verbatim. m2: groups_in_token reached the OidcDeviceAuth constructor un-coerced. A truthy non-bool (e.g. 2, from an env read without a cast) is used truthily in memory, but the store hashed the file as groups=1 while _parse_and_verify compared the raw value, so a persisted entry failed its OWN reload (`True != 2`) and re-prompted every restart. Coerce it to bool in __init__ (mirroring from_questdb), and make the store self-consistent for a direct TokenStoreKey: _serialize writes a boolean and _parse_and_verify compares bool-to-bool. m3: seven invisible Default_Ignorable non-spacing marks (category Mn) — the combining grapheme joiner (U+034F), the Mongolian free variation selectors (U+180B-U+180D, U+180F) and the Khmer inherent vowels (U+17B4, U+17B5) — survived _strip_control's "keep accents" rule and could hide payload in a user_code / identity / URL, like the FE00-FE0F variation selectors already stripped. Add them to _STRIP_EXTRA. Adds regression tests for all three. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/questdb/auth/_device.py | 22 +++++++++- src/questdb/auth/_render.py | 10 +++++ src/questdb/auth/_store.py | 10 ++++- test/test_auth.py | 87 +++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+), 3 deletions(-) diff --git a/src/questdb/auth/_device.py b/src/questdb/auth/_device.py index 19991709..6651abc6 100644 --- a/src/questdb/auth/_device.py +++ b/src/questdb/auth/_device.py @@ -479,6 +479,16 @@ def __init__( audience = None if issuer is not None and not isinstance(issuer, str): raise OidcConfigError('issuer must be a string or None') + # Coerce groups_in_token to a real bool. It is used truthily everywhere in + # memory (cache_key, TokenStoreKey.hash, _select), so a truthy non-bool + # (e.g. 2, from an env-var read without a cast) already behaves correctly + # there — but TokenStoreKey.hash buckets the file as groups=1 while + # _serialize writes the raw value and _parse_and_verify compares + # `bool(file) != key.groups_in_token`, so a raw 2 makes the entry fail its + # OWN reload (`True != 2`) and re-prompt on every restart. Normalizing + # here (mirroring from_questdb, which already bool()s it) keeps the config + # a real bool so the in-memory and on-disk identities agree. + groups_in_token = bool(groups_in_token) # default_interval feeds the poll-interval clamp and timeout every IdP # socket call; a non-numeric value would otherwise escape as a bare # TypeError rather than the typed error this block exists to raise. @@ -1679,7 +1689,17 @@ def _poll_for_token(self, resp: Dict[str, Any]) -> TokenSet: # not a rate-limit, so its cadence is unchanged absent a Retry-After). if status >= 500 or status == 429: if status == 429 or retry_after is not None: - interval = _backoff_interval(interval, retry_after) + # If this transient reply ALSO carries a slow_down body, + # RFC 8628 §3.5 still applies — the interval MUST increase by + # at least 5, never drop below current+5 just because the + # 429's Retry-After happened to be lower. Without this a + # non-conformant `429 {"error":"slow_down"}` (the RFC returns + # slow_down with HTTP 400, handled by the dedicated arm below, + # which never sees a 429/5xx) with a low Retry-After would + # poll FASTER right after the IdP asked it to slow down. + interval = _backoff_interval( + interval, retry_after, + at_least_increment=(body.get('error') == 'slow_down')) continue # A 3xx with a JSON body is still a redirect these endpoints never diff --git a/src/questdb/auth/_render.py b/src/questdb/auth/_render.py index 0c45e02e..ae745e59 100644 --- a/src/questdb/auth/_render.py +++ b/src/questdb/auth/_render.py @@ -351,8 +351,18 @@ def _render_link(url: Optional[str], *, text: Optional[str] = None) -> str: # (U+E0100–U+E01EF) — category Mn (so the "keep accents" rule below would keep # them), invisible, and able to carry hidden payload through a user_code / URL # / identity or flip an adjacent glyph's text/emoji presentation. +# - the remaining invisible Default_Ignorable non-spacing marks (also category +# Mn, so likewise kept by the "keep accents" rule): the combining grapheme +# joiner (U+034F), the Mongolian free variation selectors (U+180B–U+180D and +# U+180F) and the Khmer inherent vowels (U+17B4, U+17B5) — same hazard class +# as the variation selectors above, invisible and able to hide payload in a +# user_code / URL / identity. (The Cf/Cn/Lo Default_Ignorables — soft hyphen, +# U+180E, the zero-width/bidi runs, the tag chars — are already dropped by the +# category rule.) _STRIP_EXTRA = frozenset( '\u115f\u1160\u3164\uffa0' + + ''.join(chr(c) for c in ( + 0x034F, 0x17B4, 0x17B5, 0x180B, 0x180C, 0x180D, 0x180F)) + ''.join(chr(c) for c in range(0xFE00, 0xFE10)) + ''.join(chr(c) for c in range(0xE0100, 0xE01F0))) diff --git a/src/questdb/auth/_store.py b/src/questdb/auth/_store.py index 5e919ef8..a033a346 100644 --- a/src/questdb/auth/_store.py +++ b/src/questdb/auth/_store.py @@ -836,7 +836,12 @@ def _serialize(self, key: TokenStoreKey, token: PersistedToken) -> bytes: # interoperates unchanged. if key.issuer is not None: obj['issuer'] = key.issuer - obj['groups_in_token'] = key.groups_in_token + # bool() it: TokenStoreKey is public, so a direct caller could pass a + # truthy non-bool. hash() already buckets it truthily ('1'/'0'), so write + # the matching boolean into the payload (a boolean field should hold a + # boolean) rather than a raw 2 that reads oddly and diverges from the + # name. OidcDeviceAuth already normalizes it, so this is a no-op there. + obj['groups_in_token'] = bool(key.groups_in_token) if token.access_token is not None: obj['access_token'] = token.access_token if token.id_token is not None: @@ -874,7 +879,8 @@ def _parse_and_verify( or obj.get('scope') != key.scope or not _audience_matches(key.audience, obj.get('audience')) or not _issuer_matches(key.issuer, obj.get('issuer')) - or bool(obj.get('groups_in_token')) != key.groups_in_token): + or bool(obj.get('groups_in_token')) + != bool(key.groups_in_token)): return None return PersistedToken( access_token=_nonempty_str(obj.get('access_token')), diff --git a/test/test_auth.py b/test/test_auth.py index c74b5b44..523f6b37 100644 --- a/test/test_auth.py +++ b/test/test_auth.py @@ -464,6 +464,36 @@ def fake(url, form): # dropping back to the 5s floor. self.assertEqual(self._clock.sleeps, [5, 10, 15]) + def test_json_429_slow_down_body_still_increases_interval(self): + # m1 (RFC 8628 §3.5): a NON-conformant `429 {"error":"slow_down"}` (the + # RFC returns slow_down with HTTP 400) is caught by the 429/5xx transient + # arm, which runs BEFORE the dedicated slow_down arm. It must still obey + # the slow_down rule — raise the interval by >=5 — not honor a low + # Retry-After and poll FASTER right after the IdP asked it to slow down. + # A plain 429 with no slow_down body still honors Retry-After verbatim + # (test_poll_honors_retry_after). + from questdb.auth._http import _PostResult + auth = self.make_auth() + real = auth._idp_post + n = {'tok': 0} + + def fake(url, form): + if url.endswith('/token'): + n['tok'] += 1 + if n['tok'] == 1: + return _PostResult(400, {'error': 'slow_down'}, None) # 5->10 + if n['tok'] == 2: + # 429 status AND a slow_down body, with a low Retry-After. + return _PostResult(429, {'error': 'slow_down'}, 1) + return real(url, form) # success + return real(url, form) + + auth._idp_post = fake + self.assertEqual(auth.token(), ID_TOKEN) + # 5 -> (+5) 10 -> 429+slow_down Retry-After:1 stays >= 10+5 = 15, never + # dropping to the 5s floor. + self.assertEqual(self._clock.sleeps, [5, 10, 15]) + def test_non_json_429_retry_after_honored_in_poll(self): # m2: a non-JSON 429 from a proxy/WAF now carries its Retry-After # (post_form attaches it to the OidcError), so the poll loop's exception @@ -1187,6 +1217,25 @@ def test_constructor_rejects_bad_typed_args(self): OidcDeviceAuth.from_questdb( 'https://db.example.com:9000', default_interval=-1) + def test_groups_in_token_coerced_to_bool(self): + # m2: a truthy non-bool groups_in_token (e.g. 2, from an env read without + # a cast) is used truthily everywhere in memory, but the on-disk store + # keyed the file as groups=1 while _parse_and_verify compared the raw + # value (`bool(file) != 2`), so a persisted entry failed its OWN reload + # and re-prompted every restart. The constructor now coerces it to a real + # bool so the in-memory and on-disk identities agree. + base = dict( + client_id='c', + device_authorization_endpoint='https://idp.example.com/device', + token_endpoint='https://idp.example.com/token', + scope='openid', renderer=Renderer()) + self.assertIs( + OidcDeviceAuth(**base, groups_in_token=2).config.groups_in_token, + True) + self.assertIs( + OidcDeviceAuth(**base, groups_in_token=0).config.groups_in_token, + False) + def test_zero_expires_in_is_treated_as_unknown(self): # A non-positive expires_in must not mark the just-issued token expired. self.state.token_script = [(200, { @@ -5411,6 +5460,24 @@ def test_strip_control_caps_combining_run(self): self.assertEqual(_strip_control('e' + acute), 'e' + acute) self.assertEqual(_strip_control('café 北京'), 'café 北京') + def test_strip_control_removes_invisible_default_ignorable_marks(self): + # m3: invisible Default_Ignorable non-spacing marks (category Mn) that the + # "keep accents" rule would otherwise keep — the combining grapheme joiner + # (U+034F), the Mongolian free variation selectors (U+180B-U+180D, U+180F) + # and the Khmer inherent vowels (U+17B4, U+17B5) — can hide payload in a + # user_code / identity / URL exactly like the FE00-FE0F variation + # selectors. They must be stripped; a legitimate accent is still kept. + from questdb.auth._render import _strip_control + for cp in (0x034F, 0x180B, 0x180C, 0x180D, 0x180F, 0x17B4, 0x17B5): + self.assertEqual(_strip_control('A' + chr(cp) + 'B'), 'AB', + f'U+{cp:04X} not stripped') + # A run of them can't smuggle a hidden gap into a user_code. + self.assertEqual( + _strip_control('WDJB' + chr(0x034f) + chr(0x180b) + 'MJHT'), + 'WDJBMJHT') + # A legitimate accent (also category Mn) is still preserved. + self.assertEqual(_strip_control('e' + chr(0x0301)), 'e' + chr(0x0301)) + def test_strip_control_folds_exotic_whitespace_to_ascii_space(self): # An invisible-as-space separator (NBSP, ideographic space, ...) is a # phishing primitive: it can pad a user_code / identity / error to hide @@ -5726,6 +5793,26 @@ def test_round_trip(self): self.assertEqual(got.expires_at, 1_003_600.0) self.assertEqual(got.token_ttl, 3600.0) + def test_non_bool_groups_in_token_round_trips(self): + # m2 (store-side defense-in-depth): TokenStoreKey is public, so a direct + # caller could pass a truthy non-bool groups_in_token. hash() buckets it + # truthily ('1'/'0'), so save/load must agree — _serialize writes the + # boolean and _parse_and_verify compares bool-to-bool — rather than the + # raw value failing its own reload (`True != 2`). (OidcDeviceAuth also + # coerces it; this guards the direct-key path.) + key2 = TokenStoreKey( + 'questdb', 'https://idp:443/token', 'https://idp:443/device', + 'openid', None, 2) # truthy non-bool + self.store.save(key2, self._pt()) + self.assertIsNotNone(self.store.load(key2)) + # It buckets to the same file as a real bool-True key, and that key — + # whose payload the fix wrote as a clean boolean — loads it too. + key_true = TokenStoreKey( + 'questdb', 'https://idp:443/token', 'https://idp:443/device', + 'openid', None, True) + self.assertEqual(key2.hash(), key_true.hash()) + self.assertIsNotNone(self.store.load(key_true)) + def test_missing_file_returns_none(self): self.assertIsNone(self.store.load(self.key))