From 1c058e29e72da59f915d23289a5ad84e1af82111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Thu, 11 Jun 2026 09:31:41 -0400 Subject: [PATCH 001/289] fix(stamphog): explore PR head in isolated worktree --- .github/workflows/pr-approval-agent.yml | 66 +++++++++++- tools/pr-approval-agent/README.md | 35 +++++++ tools/pr-approval-agent/github.py | 59 ++++++++--- tools/pr-approval-agent/review_pr.py | 122 +++++++++++++++------- tools/pr-approval-agent/reviewer.py | 43 +++++++- tools/pr-approval-agent/test_github.py | 48 ++++++++- tools/pr-approval-agent/test_review_pr.py | 87 ++++++++++++++- 7 files changed, 399 insertions(+), 61 deletions(-) diff --git a/.github/workflows/pr-approval-agent.yml b/.github/workflows/pr-approval-agent.yml index ede57a1dd582..d0f4fe78aac8 100644 --- a/.github/workflows/pr-approval-agent.yml +++ b/.github/workflows/pr-approval-agent.yml @@ -2,7 +2,7 @@ name: PR Approval Agent on: pull_request: - types: [labeled, ready_for_review, synchronize] + types: [labeled, ready_for_review, synchronize, edited] permissions: contents: read @@ -19,7 +19,7 @@ jobs: # Triggers: explicit `stamphog` label, ready_for_review with the # label already present, or `synchronize` where decide-delta # asked for re-review (or itself failed — fail closed for safety). - needs: [decide-delta, dismiss] + needs: [decide-delta, dismiss, dismiss-on-retarget] if: >- always() && !github.event.pull_request.draft @@ -28,6 +28,7 @@ jobs: || (github.event.action == 'ready_for_review' && contains(github.event.pull_request.labels.*.name, 'stamphog')) || needs.decide-delta.outputs.run_review == 'true' || needs.decide-delta.result == 'failure' + || (github.event.action == 'edited' && github.event.changes.base != null && contains(github.event.pull_request.labels.*.name, 'stamphog')) ) runs-on: ubuntu-latest timeout-minutes: 10 @@ -172,8 +173,19 @@ jobs: filter: blob:none fetch-depth: 0 - - name: Fetch PR head - run: git fetch --filter=blob:none origin pull/${{ github.event.pull_request.number }}/head + - name: Fetch PR head and base + # base.ref is author-controlled, so it goes through an env var + # (never interpolated into the command). A stacked PR's base is + # its parent branch, which the master checkout doesn't fetch; + # dismiss_check needs it to classify merge commits. Best-effort — + # a missing base just fails the merge check closed to re-review. + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + BASE_REF_NAME: ${{ github.event.pull_request.base.ref }} + run: | + set -euo pipefail + git fetch --filter=blob:none origin "pull/${PR_NUMBER}/head" + git fetch --filter=blob:none origin "$BASE_REF_NAME" || true - name: Install uv uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 @@ -260,3 +272,49 @@ jobs: -f message="New commits pushed (delta classified \`$REASON\`) — stamphog approval dismissed; re-review running automatically." \ -f event=DISMISS done + + # A base-branch retarget changes the effective diff without a push, so no + # `synchronize` fires and the decide-delta/dismiss path above never runs. + # This is the normal Graphite flow: when a stack's parent PR merges, the + # child is retargeted from the parent branch onto master, and its diff is + # recomputed against a different base. Under the master ruleset + # (dismiss_stale_reviews_on_push=false), a prior bot approval would + # silently carry onto that new surface — and a malicious retarget could + # exploit the same gap. Dismiss the stale approval here; the review job + # re-runs (gated on the label) against the new base. This job is ordered + # before review via `needs`, so it can't dismiss the fresh approval the + # re-review is about to post. + dismiss-on-retarget: + if: >- + github.event.action == 'edited' + && github.event.changes.base != null + && !github.event.pull_request.draft + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + # The dismissal loop below mirrors the `dismiss` job's. Neither job + # checks out the repo, so factoring it into a shared script would + # force a checkout into both — more cost than the duplication saves. + # If you change the bot-approval selection (the github-actions[bot] + # + APPROVED filter), update both jobs in lockstep. + - name: Dismiss stale bot approvals on base change + env: + # Same identity (github-actions[bot]) that posted the approval. + GH_TOKEN: ${{ github.token }} + PR: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + set -euo pipefail + + mapfile -t REVIEW_IDS < <( + gh api "repos/$REPO/pulls/$PR/reviews" --paginate \ + --jq '.[] | select(.user.login == "github-actions[bot]" and .state == "APPROVED") | .id' + ) + + for id in "${REVIEW_IDS[@]}"; do + [ -z "$id" ] && continue + gh api -X PUT "repos/$REPO/pulls/$PR/reviews/$id/dismissals" \ + -f message="Base branch retargeted — stamphog approval dismissed; re-review runs automatically if the label is present." \ + -f event=DISMISS + done diff --git a/tools/pr-approval-agent/README.md b/tools/pr-approval-agent/README.md index 5b7ee817e6bb..e375756ea7f7 100644 --- a/tools/pr-approval-agent/README.md +++ b/tools/pr-approval-agent/README.md @@ -75,6 +75,41 @@ Final verdict → GitHub review (approve or comment) The bot never posts request-changes — only approves or comments. +## Stacked PRs (Graphite / git stacks) + +A stacked PR targets its parent branch, not master, and depends on code the +parent introduces but hasn't merged yet. Two parts make stamphog correct on +these: + +- **Exploration sees the post-stack tree.** The workflow checks out master + (hardcoded, so a PR can't swap the review script), but the LLM reviewer's + `Read`/`Grep`/`Glob` run in a detached **worktree at the PR head** instead. + The head tree already contains the parent PRs' code, so symbols from a + not-yet-merged parent resolve and aren't flagged as broken imports. The diff + itself is still computed `base_sha...head_sha`, so the review is scoped to + exactly this PR's changes. Worktree creation falls back to reviewing from + master if it fails. + - **Security:** the worktree is PR-authored content. The reviewer runs the + Agent SDK with `setting_sources=[]` (isolation mode), so it does **not** + load `.claude/settings.json` hooks (command execution) or `CLAUDE.md` + (injected instructions) from the head tree. Those files are still readable + as untrusted _content_ under the anti-injection notice — never as + configuration. + +- **Base retarget dismisses the stale approval.** When a stack's parent merges, + the child PR is retargeted from the parent branch onto master, changing its + effective diff **without a push** — so no `synchronize` fires and the normal + push-dismiss path is skipped. Under the master ruleset + (`dismiss_stale_reviews_on_push=false`), a prior bot approval would silently + carry onto the new base. The workflow listens for the `edited` event and, when + the base changed, dismisses the bot approval and re-reviews against the new + base (if the label is still present). + +The base commit of a stacked PR is its parent branch tip, which the master +checkout doesn't fetch by default — `github.ensure_commits` and the +`decide-delta` job both fetch the base branch so `git diff base_sha...head_sha` +and the dismiss-time merge classification resolve it. + ## Tiers ### T0 — deterministic diff --git a/tools/pr-approval-agent/github.py b/tools/pr-approval-agent/github.py index 2a8adeb199f7..3936d842e1d3 100644 --- a/tools/pr-approval-agent/github.py +++ b/tools/pr-approval-agent/github.py @@ -23,6 +23,7 @@ class PRData: mergeable_state: str author: str labels: list[str] + base_ref: str base_sha: str head_sha: str files: list[dict] @@ -232,35 +233,60 @@ def _git_diff_files(base_sha: str, head_sha: str, repo_root: Path) -> list[dict] return files -def ensure_commits(pr_number: int, head_sha: str, repo_root: Path) -> None: - """Fetch PR commits if not available locally.""" - result = subprocess.run( - ["git", "cat-file", "-t", head_sha], - cwd=repo_root, - capture_output=True, - timeout=5, - ) - if result.returncode == 0: - return - subprocess.run( - ["git", "fetch", "origin", f"pull/{pr_number}/head"], - cwd=repo_root, - capture_output=True, - timeout=30, +def _have_commit(sha: str, repo_root: Path) -> bool: + return ( + subprocess.run( + ["git", "cat-file", "-t", sha], + cwd=repo_root, + capture_output=True, + timeout=5, + ).returncode + == 0 ) +def ensure_commits(pr_number: int, head_sha: str, base_ref: str, base_sha: str, repo_root: Path) -> None: + """Make the PR head and its base commit available locally. + + The workflow checks out master and fetches `pull//head`, which covers + the common case. Two stacked-PR cases need more: + - The head: fetched explicitly if the merge commit isn't present. + - The base: a stacked PR targets its parent branch, not master, so + `base_sha` is the parent's tip. It's usually an ancestor of the head + (reachable once the head is fetched), but a rebased/force-pushed stack + can leave it unreachable — fetch the base branch by name to be sure. + `git diff base_sha...head_sha` (and dismiss_check's ancestry walk) + need that object present. Best-effort: a missing base surfaces later + as a diff error rather than a silent wrong scope. + """ + if not _have_commit(head_sha, repo_root): + subprocess.run( + ["git", "fetch", "--filter=blob:none", "origin", f"pull/{pr_number}/head"], + cwd=repo_root, + capture_output=True, + timeout=60, + ) + if not _have_commit(base_sha, repo_root): + subprocess.run( + ["git", "fetch", "--filter=blob:none", "origin", base_ref], + cwd=repo_root, + capture_output=True, + timeout=60, + ) + + def fetch_pr(pr_number: int, repo: str, repo_root: Path | None = None) -> PRData: """Fetch PR data: metadata from API, file stats from local git.""" pr = _gh_api(f"repos/{repo}/pulls/{pr_number}") reviews_raw = _gh_api(f"repos/{repo}/pulls/{pr_number}/reviews", paginate=True) + base_ref = pr["base"]["ref"] base_sha = pr["base"]["sha"] head_sha = pr["head"]["sha"] check_runs_resp = _gh_api(f"repos/{repo}/commits/{head_sha}/check-runs") git_root = repo_root or Path.cwd() - ensure_commits(pr_number, head_sha, git_root) + ensure_commits(pr_number, head_sha, base_ref, base_sha, git_root) files = _git_diff_files(base_sha, head_sha, git_root) review_comments = _fetch_review_threads(repo, pr_number) @@ -274,6 +300,7 @@ def fetch_pr(pr_number: int, repo: str, repo_root: Path | None = None) -> PRData mergeable_state=pr.get("mergeable_state", "unknown"), author=pr["user"]["login"], labels=[label["name"] for label in pr.get("labels", [])], + base_ref=base_ref, base_sha=base_sha, head_sha=head_sha, files=files, diff --git a/tools/pr-approval-agent/review_pr.py b/tools/pr-approval-agent/review_pr.py index c250c879486e..e6e3aadfccd7 100644 --- a/tools/pr-approval-agent/review_pr.py +++ b/tools/pr-approval-agent/review_pr.py @@ -22,7 +22,11 @@ import json import time +import uuid import argparse +import tempfile +import subprocess +from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path @@ -341,9 +345,55 @@ def _check_tier(self) -> tuple[bool, str]: return True, f"T0 auto-approve: {summary}" return True, summary + @contextmanager + def _pr_head_worktree(self): + """Yield a detached worktree at the PR head, or None on failure. + + The agent's filesystem tools need to see the codebase as it will look + after the PR lands. For a stacked PR that includes code from parent + PRs that aren't on the base branch yet — without it, those parents' + symbols look like broken imports and the reviewer false-refuses. The + main checkout stays master (the workflow hardcodes that so a PR can't + swap the review script), so the worktree is the only place the head + tree is materialized. Cleaned up on exit; falls back to None (review + from master) if creation fails. + + SECURITY: the worktree is PR-authored content. The agent is isolated + from it as *configuration* by setting_sources=[] in Reviewer — see the + note there. It still reads the files as untrusted *content*, which is + the whole point of a review. + """ + worktree_dir = Path(tempfile.gettempdir()) / f"pr-review-{self.pr_number}-{uuid.uuid4().hex[:8]}" + created = False + try: + result = subprocess.run( + ["git", "worktree", "add", "--detach", str(worktree_dir), self.pr.head_sha], + capture_output=True, + text=True, + timeout=120, + cwd=REPO_ROOT, + ) + if result.returncode == 0: + created = True + print(_dim(f" Exploring PR head in worktree: {worktree_dir}")) + else: + print(_warn(f"Worktree creation failed, reviewing from master: {result.stderr.strip()}")) + except subprocess.TimeoutExpired: + print(_warn("Worktree creation timed out, reviewing from master")) + + try: + yield worktree_dir if created else None + finally: + if created: + subprocess.run( + ["git", "worktree", "remove", "--force", str(worktree_dir)], + capture_output=True, + timeout=30, + cwd=REPO_ROOT, + ) + def _llm_review(self, gate_verdict: str) -> None: print(f"\n{_bold('LLM Review')}") - reviewer = Reviewer(REPO_ROOT, verbose=self.verbose) gate_context = { "gate_verdict": gate_verdict, @@ -353,41 +403,43 @@ def _llm_review(self, gate_verdict: str) -> None: print(_dim(" Calling reviewer...")) max_retries = 3 reviewer_unavailable = False - for attempt in range(max_retries): - try: - self.reviewer_output = reviewer.review( - self.pr, - self.classification, - gate_context, - ) - break - except Exception as e: - if attempt < max_retries - 1: - wait = 2 ** (attempt + 1) - print(_warn(f"Reviewer failed (attempt {attempt + 1}/{max_retries}): {e}")) - print(_dim(f" Retrying in {wait}s...")) - time.sleep(wait) - else: - print(_fail(f"Reviewer failed after {max_retries} attempts: {e}")) - print( - _warn( - " This is an LLM backend failure (credentials, credit, or outage), " - "not a verdict on the PR. Check the STAMPHOG_ANTHROPIC_API_KEY " - "secret (or local ANTHROPIC_API_KEY)." - ) + with self._pr_head_worktree() as explore_root: + reviewer = Reviewer(REPO_ROOT, explore_root=explore_root, verbose=self.verbose) + for attempt in range(max_retries): + try: + self.reviewer_output = reviewer.review( + self.pr, + self.classification, + gate_context, ) - reviewer_unavailable = True - self.reviewer_output = { - "verdict": "ERROR", - "reasoning": ( - "The review agent couldn't reach its LLM backend — an infrastructure " - "or credentials issue, not a problem with this PR. The `stamphog` label " - "has been kept; the review retries automatically on the next push, or " - "re-apply the label once the backend recovers." - ), - "risk": "unknown", - "issues": [str(e)], - } + break + except Exception as e: + if attempt < max_retries - 1: + wait = 2 ** (attempt + 1) + print(_warn(f"Reviewer failed (attempt {attempt + 1}/{max_retries}): {e}")) + print(_dim(f" Retrying in {wait}s...")) + time.sleep(wait) + else: + print(_fail(f"Reviewer failed after {max_retries} attempts: {e}")) + print( + _warn( + " This is an LLM backend failure (credentials, credit, or outage), " + "not a verdict on the PR. Check the STAMPHOG_ANTHROPIC_API_KEY " + "secret (or local ANTHROPIC_API_KEY)." + ) + ) + reviewer_unavailable = True + self.reviewer_output = { + "verdict": "ERROR", + "reasoning": ( + "The review agent couldn't reach its LLM backend — an infrastructure " + "or credentials issue, not a problem with this PR. The `stamphog` label " + "has been kept; the review retries automatically on the next push, or " + "re-apply the label once the backend recovers." + ), + "risk": "unknown", + "issues": [str(e)], + } llm_verdict = self.reviewer_output.get("verdict", "UNKNOWN") print(f" Verdict: {llm_verdict}") diff --git a/tools/pr-approval-agent/reviewer.py b/tools/pr-approval-agent/reviewer.py index 140a64821ee0..2fe03ec4ad49 100644 --- a/tools/pr-approval-agent/reviewer.py +++ b/tools/pr-approval-agent/reviewer.py @@ -204,8 +204,12 @@ def _validate_verdict(result: dict) -> dict: class Reviewer: """LLM reviewer using Agent SDK.""" - def __init__(self, repo_root: Path, *, verbose: bool = False): + def __init__(self, repo_root: Path, *, explore_root: Path | None = None, verbose: bool = False): self.repo_root = repo_root + # Where the agent's Read/Grep/Glob look. For stacked PRs this is a + # worktree at the PR head so imports from not-yet-merged parent PRs + # resolve (see review_pr._pr_head_worktree). Falls back to repo_root. + self.explore_root = explore_root or repo_root self.verbose = verbose def review(self, pr: PRData, classification: dict, gate_context: dict) -> dict: @@ -224,7 +228,18 @@ async def _review(self, pr: PRData, classification: dict, gate_context: dict) -> system_prompt=REVIEWER_SYSTEM, allowed_tools=["Read", "Grep", "Glob"], disallowed_tools=["Write", "Edit", "NotebookEdit", "Bash", "Agent", "WebFetch", "WebSearch"], - cwd=str(self.repo_root), + cwd=str(self.explore_root), + # SECURITY: explore_root holds PR-authored content (a worktree at + # the PR head for stacked PRs). With the default (None) the SDK + # loads filesystem settings from cwd like the CLI does — including + # .claude/settings.json hooks (arbitrary command execution) and + # CLAUDE.md (injected as instructions). A PR could ship either. + # [] is SDK isolation mode: no filesystem settings, no hooks, no + # CLAUDE.md autoload. The agent can still Read those files, but as + # untrusted content under the anti-injection notice, never as + # configuration. This is the guardrail that makes pointing cwd at + # PR-controlled files safe. + setting_sources=[], max_turns=3 if quick else 20, model=MODEL, permission_mode="dontAsk", @@ -318,8 +333,14 @@ def _log_tool_call(self, block: ToolUseBlock) -> None: print(f"\033[2m {name} {json.dumps(inp)[:100]}\033[0m", flush=True) def _write_diff_file(self, pr: PRData) -> Path: - """Write the PR diff to a temp file so the LLM can Read it on demand.""" - diff_path = self.repo_root / ".pr-review-diff.patch" + """Write the PR diff to a temp file so the LLM can Read it on demand. + + Written into explore_root so the agent (whose cwd is explore_root) + can read it by relative path. The diff itself is always computed from + base_sha...head_sha in the main repo, so it shows only this PR's + changes even when explore_root is a worktree at the full-stack head. + """ + diff_path = self.explore_root / ".pr-review-diff.patch" result = subprocess.run( ["git", "diff", f"{pr.base_sha}...{pr.head_sha}"], capture_output=True, @@ -380,6 +401,19 @@ def _build_review_prompt(self, pr: PRData, cl: dict, gate_context: dict, diff_pa elif gate_verdict == "AUTO-APPROVED": constraint = "\nGates auto-approved (T0). Confirm or flag concerns." + # Stacked PRs target a parent branch, not master. The working tree is + # the PR head, so it already contains code from not-yet-merged parent + # PRs — that's why Read/Grep/Glob resolve symbols that aren't in the + # diff. Tell the agent so it doesn't flag those as missing. + stack_note = "" + if pr.base_ref != "master": + stack_note = ( + f"\nStacked PR: this targets `{pr.base_ref}`, not master. The working tree reflects the " + "codebase as it will look after the whole stack lands, so symbols defined in parent PRs " + "resolve via Read/Grep/Glob even though they're absent from the diff below. Review only the " + "diff's changes; do not flag imports or references that resolve in the tree as missing." + ) + file_list = "\n".join( f" {f['filename']} (+{f['additions']}/-{f['deletions']})" + (" [NEW]" if f.get("status") == "A" else "") for f in pr.files @@ -401,6 +435,7 @@ def _build_review_prompt(self, pr: PRData, cl: dict, gate_context: dict, diff_pa {chr(10).join(gate_lines)} Gate verdict: {gate_verdict} {constraint} + {stack_note} The full diff is at: {diff_path} Read this file to review the changes, then submit your verdict. diff --git a/tools/pr-approval-agent/test_github.py b/tools/pr-approval-agent/test_github.py index 87aaa5a4617b..e81da9dacba4 100644 --- a/tools/pr-approval-agent/test_github.py +++ b/tools/pr-approval-agent/test_github.py @@ -1,8 +1,11 @@ """Tests for GitHub review normalization used by the PR approval agent.""" +from pathlib import Path + import pytest -from github import _normalize_reviews_for_prompt +import github +from github import _normalize_reviews_for_prompt, ensure_commits def test_normalize_reviews_marks_current_head_and_preserves_stale_reviews() -> None: @@ -75,3 +78,46 @@ def test_normalize_reviews_filters_by_trust_source( ) assert len(normalized) == expected_count + + +class _Result: + def __init__(self, returncode: int) -> None: + self.returncode = returncode + + +@pytest.mark.parametrize( + "present, expected_fetches", + [ + pytest.param({"HEAD_SHA", "BASE_SHA"}, [], id="both-present-no-fetch"), + pytest.param({"BASE_SHA"}, ["pull/9/head"], id="head-missing-fetches-pr-head"), + pytest.param({"HEAD_SHA"}, ["query-validations"], id="base-missing-fetches-base-branch"), + pytest.param(set(), ["pull/9/head", "query-validations"], id="both-missing-fetches-both"), + ], +) +def test_ensure_commits_fetches_missing_head_and_base( + monkeypatch: pytest.MonkeyPatch, present: set[str], expected_fetches: list[str] +) -> None: + """Stacked PRs target a parent branch, so the base commit may not be + reachable from the master checkout. ensure_commits fetches whatever is + missing — head via the pull ref, base via the base branch name.""" + fetched: list[str] = [] + + def fake_run(cmd: list[str], **kwargs: object) -> _Result: + if cmd[:3] == ["git", "cat-file", "-t"]: + return _Result(0 if cmd[3] in present else 1) + if "fetch" in cmd: + fetched.append(cmd[-1]) + return _Result(0) + return _Result(0) + + monkeypatch.setattr(github.subprocess, "run", fake_run) + + ensure_commits( + pr_number=9, + head_sha="HEAD_SHA", + base_ref="query-validations", + base_sha="BASE_SHA", + repo_root=Path("/repo"), + ) + + assert fetched == expected_fetches diff --git a/tools/pr-approval-agent/test_review_pr.py b/tools/pr-approval-agent/test_review_pr.py index e58da758fbf1..5a1432eb798e 100644 --- a/tools/pr-approval-agent/test_review_pr.py +++ b/tools/pr-approval-agent/test_review_pr.py @@ -10,12 +10,13 @@ sys.modules.setdefault("claude_agent_sdk", MagicMock()) sys.modules.setdefault("claude_agent_sdk.types", MagicMock()) +import reviewer as reviewer_mod # noqa: E402 import review_pr # noqa: E402 from github import PRData # noqa: E402 from review_pr import GateResult, Pipeline # noqa: E402 -def _fake_pr(head_sha: str) -> PRData: +def _fake_pr(head_sha: str, base_ref: str = "master") -> PRData: return PRData( number=1, repo="PostHog/posthog", @@ -25,6 +26,7 @@ def _fake_pr(head_sha: str) -> PRData: mergeable_state="clean", author="alice", labels=[], + base_ref=base_ref, base_sha="def456", head_sha=head_sha, files=[], @@ -88,3 +90,86 @@ def test_backend_failure_yields_error_except_when_gates_deny( if expected_final == "ERROR": assert pipeline.reviewer_output is not None assert pipeline.reviewer_output["verdict"] == "ERROR" + + +class _FakeCompleted: + def __init__(self, returncode: int, stderr: str = "") -> None: + self.returncode = returncode + self.stderr = stderr + + +def test_pr_head_worktree_yields_path_and_cleans_up(monkeypatch: pytest.MonkeyPatch) -> None: + """On success the context manager yields the worktree path and removes it on exit.""" + calls: list[list[str]] = [] + + def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: + calls.append(cmd) + return _FakeCompleted(0) + + monkeypatch.setattr(review_pr.subprocess, "run", fake_run) + + pipeline = Pipeline(pr_number=42, repo="PostHog/posthog") + pipeline.pr = _fake_pr(head_sha="cafe123") + + with pipeline._pr_head_worktree() as explore_root: + assert explore_root is not None + assert "pr-review-42-" in explore_root.name + add = next(c for c in calls if "add" in c) + assert "--detach" in add and "cafe123" in add + + # Cleanup ran with --force after the block exited. + remove = next(c for c in calls if "remove" in c) + assert "--force" in remove + + +def test_pr_head_worktree_falls_back_to_none_on_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """If worktree creation fails, yield None (review from master) and skip removal.""" + calls: list[list[str]] = [] + + def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: + calls.append(cmd) + return _FakeCompleted(1, stderr="fatal: invalid reference") + + monkeypatch.setattr(review_pr.subprocess, "run", fake_run) + + pipeline = Pipeline(pr_number=7, repo="PostHog/posthog") + pipeline.pr = _fake_pr(head_sha="deadbeef") + + with pipeline._pr_head_worktree() as explore_root: + assert explore_root is None + + # No worktree was created, so none is removed. + assert not any("remove" in c for c in calls) + + +def test_reviewer_explore_root_defaults_to_repo_root() -> None: + from pathlib import Path + + repo = Path("/repo") + assert reviewer_mod.Reviewer(repo).explore_root == repo + other = Path("/tmp/wt") + assert reviewer_mod.Reviewer(repo, explore_root=other).explore_root == other + + +@pytest.mark.parametrize( + "base_ref, expect_stack_note", + [ + ("master", False), + ("query-validations", True), + ], +) +def test_reviewer_prompt_stack_note(base_ref: str, expect_stack_note: bool) -> None: + """A stacked PR (base != master) gets a note telling the agent that + parent-PR symbols resolve in the tree and aren't missing.""" + from pathlib import Path + + reviewer = reviewer_mod.Reviewer(Path("/repo")) + pr = _fake_pr(head_sha="abc123", base_ref=base_ref) + classification = {"tier": "T1-agent", "t1_subclass": "T1b-small", "breadth": "single-area", "commit_type": "feat"} + gate_context = {"gate_verdict": "PENDING", "gates": []} + + prompt = reviewer._build_review_prompt(pr, classification, gate_context, Path("/tmp/diff.patch")) + + assert ("Stacked PR" in prompt) is expect_stack_note + if expect_stack_note: + assert base_ref in prompt From 4bb3fab5503a7bd0b8b8f0df3930a9c386b4078f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Tue, 7 Jul 2026 15:30:41 -0400 Subject: [PATCH 002/289] refactor(stamphog): harden mcp isolation, tidy stacked-pr code --- .github/workflows/pr-approval-agent.yml | 17 +-- tools/pr-approval-agent/github.py | 2 +- tools/pr-approval-agent/review_pr.py | 126 +++++++++++--------- tools/pr-approval-agent/reviewer.py | 13 +- tools/pr-approval-agent/test_familiarity.py | 1 + tools/pr-approval-agent/test_policy.py | 2 + tools/pr-approval-agent/test_reviewer.py | 1 + 7 files changed, 87 insertions(+), 75 deletions(-) diff --git a/.github/workflows/pr-approval-agent.yml b/.github/workflows/pr-approval-agent.yml index a04a39ef0626..47200371a12d 100644 --- a/.github/workflows/pr-approval-agent.yml +++ b/.github/workflows/pr-approval-agent.yml @@ -382,17 +382,12 @@ jobs: -f event=DISMISS done - # A base-branch retarget changes the effective diff without a push, so no - # `synchronize` fires and the decide-delta/dismiss path above never runs. - # This is the normal Graphite flow: when a stack's parent PR merges, the - # child is retargeted from the parent branch onto master, and its diff is - # recomputed against a different base. Under the master ruleset - # (dismiss_stale_reviews_on_push=false), a prior bot approval would - # silently carry onto that new surface — and a malicious retarget could - # exploit the same gap. Dismiss the stale approval here; the review job - # re-runs (gated on the label) against the new base. This job is ordered - # before review via `needs`, so it can't dismiss the fresh approval the - # re-review is about to post. + # A base retarget (Graphite moving a child onto master when its parent + # merges) changes the diff without a push, so no `synchronize` fires and a + # prior bot approval carries onto the new base under the master ruleset + # (dismiss_stale_reviews_on_push=false). Dismiss it; review re-runs against + # the new base. Ordered before review via `needs`, so it can't dismiss the + # fresh approval that re-review is about to post. See README for the full rationale. dismiss-on-retarget: if: >- github.event.action == 'edited' diff --git a/tools/pr-approval-agent/github.py b/tools/pr-approval-agent/github.py index ef18d7453af8..f3d6c20c15d3 100644 --- a/tools/pr-approval-agent/github.py +++ b/tools/pr-approval-agent/github.py @@ -24,13 +24,13 @@ class PRData: mergeable_state: str author: str labels: list[str] + base_ref: str base_sha: str head_sha: str files: list[dict] reviews: list[dict] review_comments: list[dict] check_runs: list[dict] - base_ref: str = "master" author_is_bot: bool = False pr_reactions: list[dict] = field(default_factory=list) body: str = "" diff --git a/tools/pr-approval-agent/review_pr.py b/tools/pr-approval-agent/review_pr.py index ba5205281170..64bc12259319 100644 --- a/tools/pr-approval-agent/review_pr.py +++ b/tools/pr-approval-agent/review_pr.py @@ -653,10 +653,8 @@ def _pr_head_worktree(self): tree is materialized. Cleaned up on exit; falls back to None (review from master) if creation fails. - SECURITY: the worktree is PR-authored content. The agent is isolated - from it as *configuration* by setting_sources=[] in Reviewer — see the - note there. It still reads the files as untrusted *content*, which is - the whole point of a review. + SECURITY: the worktree is PR-authored content; isolation from it as + *configuration* is enforced by setting_sources=[] in Reviewer. """ worktree_dir = Path(tempfile.gettempdir()) / f"pr-review-{self.pr_number}-{uuid.uuid4().hex[:8]}" created = False @@ -687,6 +685,70 @@ def _pr_head_worktree(self): cwd=REPO_ROOT, ) + def _run_reviewer_with_retries(self, reviewer: Reviewer, gate_context: dict, diff_path: Path) -> bool: + """Call the reviewer with backoff; set self.reviewer_output. + + Returns True when the reviewer never produced a verdict (an ERROR + stand-in was synthesized instead) so the caller retains the label. + Retryable failures (LLM backend) back off; non-retryable ones (e.g. + turn-limit) fail immediately with a distinct message. + """ + max_retries = 3 + for attempt in range(max_retries): + try: + self.reviewer_output = reviewer.review( + self.pr, + self.classification, + gate_context, + diff_path=diff_path, + ) + return False + except Exception as e: + err_str = str(e) + is_retryable = _is_retryable_error(err_str) + + if is_retryable and attempt < max_retries - 1: + wait = 2 ** (attempt + 1) + print(_warn(f"Reviewer failed (attempt {attempt + 1}/{max_retries}): {e}")) + print(_dim(f" Retrying in {wait}s...")) + time.sleep(wait) + continue + + if is_retryable: + print(_fail(f"Reviewer failed after {max_retries} attempts: {e}")) + print( + _warn( + " This is an LLM backend failure (credentials, credit, or outage), " + "not a verdict on the PR. Check the STAMPHOG_ANTHROPIC_API_KEY " + "secret (or local ANTHROPIC_API_KEY)." + ) + ) + self.reviewer_output = { + "verdict": "ERROR", + "reasoning": ( + "The review agent couldn't reach its LLM backend — an infrastructure " + "or credentials issue, not a problem with this PR. The `stamphog` label " + "has been kept; the review retries automatically on the next push, or " + "re-apply the label once the backend recovers." + ), + "risk": "unknown", + "issues": [err_str], + } + else: + print(_fail(f"Reviewer hit a non-retryable error: {e}")) + self.reviewer_output = { + "verdict": "ERROR", + "reasoning": ( + "The review agent could not complete its analysis for this PR " + "(likely too complex for the allocated turn budget). " + "The `stamphog` label has been kept; a human review is needed." + ), + "risk": "unknown", + "issues": [err_str], + } + return True + return True + def _llm_review(self, gate_verdict: str) -> None: print(f"\n{_bold('LLM Review')}") # Outside the retry loop: a diff-write hiccup must not masquerade as a @@ -699,63 +761,9 @@ def _llm_review(self, gate_verdict: str) -> None: } print(_dim(" Calling reviewer...")) - max_retries = 3 - reviewer_unavailable = False with self._pr_head_worktree() as explore_root: reviewer = Reviewer(REPO_ROOT, explore_root=explore_root, verbose=self.verbose) - for attempt in range(max_retries): - try: - self.reviewer_output = reviewer.review( - self.pr, - self.classification, - gate_context, - diff_path=diff_path, - ) - break - except Exception as e: - err_str = str(e) - is_retryable = _is_retryable_error(err_str) - - if is_retryable and attempt < max_retries - 1: - wait = 2 ** (attempt + 1) - print(_warn(f"Reviewer failed (attempt {attempt + 1}/{max_retries}): {e}")) - print(_dim(f" Retrying in {wait}s...")) - time.sleep(wait) - else: - reviewer_unavailable = True - if is_retryable: - print(_fail(f"Reviewer failed after {max_retries} attempts: {e}")) - print( - _warn( - " This is an LLM backend failure (credentials, credit, or outage), " - "not a verdict on the PR. Check the STAMPHOG_ANTHROPIC_API_KEY " - "secret (or local ANTHROPIC_API_KEY)." - ) - ) - self.reviewer_output = { - "verdict": "ERROR", - "reasoning": ( - "The review agent couldn't reach its LLM backend — an infrastructure " - "or credentials issue, not a problem with this PR. The `stamphog` label " - "has been kept; the review retries automatically on the next push, or " - "re-apply the label once the backend recovers." - ), - "risk": "unknown", - "issues": [err_str], - } - else: - print(_fail(f"Reviewer hit a non-retryable error: {e}")) - self.reviewer_output = { - "verdict": "ERROR", - "reasoning": ( - "The review agent could not complete its analysis for this PR " - "(likely too complex for the allocated turn budget). " - "The `stamphog` label has been kept; a human review is needed." - ), - "risk": "unknown", - "issues": [err_str], - } - break + reviewer_unavailable = self._run_reviewer_with_retries(reviewer, gate_context, diff_path) llm_verdict = self.reviewer_output.get("verdict", "UNKNOWN") print(f" Verdict: {llm_verdict}") diff --git a/tools/pr-approval-agent/reviewer.py b/tools/pr-approval-agent/reviewer.py index e5a23ab63fe4..7a99dc5e7a8c 100644 --- a/tools/pr-approval-agent/reviewer.py +++ b/tools/pr-approval-agent/reviewer.py @@ -307,6 +307,13 @@ async def _review( # configuration. This is the guardrail that makes pointing cwd at # PR-controlled files safe. setting_sources=[], + # setting_sources=[] covers settings.json but not .mcp.json, which + # has its own discovery. The CLI's project-trust gate already + # refuses an unapproved .mcp.json in headless mode, but pin it: + # strict config + empty server map ignore any .mcp.json the PR + # ships in the head tree, regardless of CLI defaults. + mcp_servers={}, + strict_mcp_config=True, max_turns=5 if quick else 20, model=MODEL, permission_mode="dontAsk", @@ -493,10 +500,8 @@ def _build_review_prompt(self, pr: PRData, cl: dict, gate_context: dict, diff_pa "scripts or lifecycle hooks changed." ) - # Stacked PRs target a parent branch, not master. The working tree is - # the PR head, so it already contains code from not-yet-merged parent - # PRs — that's why Read/Grep/Glob resolve symbols that aren't in the - # diff. Tell the agent so it doesn't flag those as missing. + # For a stacked PR the working tree is the PR head, so parent-PR symbols + # resolve in Read/Grep/Glob though absent from the diff; tell the agent. if pr.base_ref != "master": constraint += ( f"\nStacked PR: this targets `{pr.base_ref}`, not master. The working tree reflects the " diff --git a/tools/pr-approval-agent/test_familiarity.py b/tools/pr-approval-agent/test_familiarity.py index b1807253d536..a90518f9451c 100644 --- a/tools/pr-approval-agent/test_familiarity.py +++ b/tools/pr-approval-agent/test_familiarity.py @@ -315,6 +315,7 @@ def _prompt_fixture() -> tuple[PRData, dict, dict]: mergeable_state="clean", author="alice", labels=[], + base_ref="master", base_sha="base", head_sha="head", files=[{"filename": "src/foo.py", "additions": 3, "deletions": 1, "status": "M"}], diff --git a/tools/pr-approval-agent/test_policy.py b/tools/pr-approval-agent/test_policy.py index a3658e95ec38..da1a1acbed9d 100644 --- a/tools/pr-approval-agent/test_policy.py +++ b/tools/pr-approval-agent/test_policy.py @@ -393,6 +393,7 @@ def test_size_gate_applies_mixed_leniency(n_global: int, expected_ok: bool) -> N mergeable_state="clean", author="alice", labels=[], + base_ref="master", base_sha="base", head_sha="head", files=vr_files + global_files, @@ -537,6 +538,7 @@ def _body_pipeline(fam) -> "review_pr.Pipeline": mergeable_state="clean", author="alice", labels=[], + base_ref="master", base_sha="base", head_sha="91c4be2aaaa", files=[{"filename": "products/visual_review/a.py", "additions": 3, "deletions": 1, "status": "M"}], diff --git a/tools/pr-approval-agent/test_reviewer.py b/tools/pr-approval-agent/test_reviewer.py index e564e26d6961..57d762e4b01a 100644 --- a/tools/pr-approval-agent/test_reviewer.py +++ b/tools/pr-approval-agent/test_reviewer.py @@ -25,6 +25,7 @@ def _pr(**overrides: object) -> PRData: "mergeable_state": "clean", "author": "alice", "labels": [], + "base_ref": "master", "base_sha": "a", "head_sha": "h", "files": [], From c1b717018f7ad56a85ac188542e68f6710cb4953 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Tue, 7 Jul 2026 15:35:02 -0400 Subject: [PATCH 003/289] perf(stamphog): skip head worktree for non-stacked PRs --- tools/pr-approval-agent/review_pr.py | 22 +++++++++++++--------- tools/pr-approval-agent/test_review_pr.py | 22 +++++++++++++++++++--- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/tools/pr-approval-agent/review_pr.py b/tools/pr-approval-agent/review_pr.py index 64bc12259319..966331872406 100644 --- a/tools/pr-approval-agent/review_pr.py +++ b/tools/pr-approval-agent/review_pr.py @@ -642,20 +642,24 @@ def _check_tier(self) -> tuple[bool, str]: @contextmanager def _pr_head_worktree(self): - """Yield a detached worktree at the PR head, or None on failure. + """Yield a detached worktree at the PR head, or None to review from master. - The agent's filesystem tools need to see the codebase as it will look - after the PR lands. For a stacked PR that includes code from parent - PRs that aren't on the base branch yet — without it, those parents' - symbols look like broken imports and the reviewer false-refuses. The - main checkout stays master (the workflow hardcodes that so a PR can't - swap the review script), so the worktree is the only place the head - tree is materialized. Cleaned up on exit; falls back to None (review - from master) if creation fails. + Only stacked PRs need this: their head contains code from parent PRs + that aren't on the base branch yet, so without materializing the head + tree those parents' symbols look like broken imports and the reviewer + false-refuses. A non-stacked PR (base is master) reviews from the master + checkout exactly as before — yield None and skip the full-tree checkout. + The main checkout stays master (the workflow hardcodes that so a PR can't + swap the review script), so the worktree is the only place the head tree + is materialized. Cleaned up on exit; falls back to None if creation fails. SECURITY: the worktree is PR-authored content; isolation from it as *configuration* is enforced by setting_sources=[] in Reviewer. """ + if self.pr.base_ref == "master": + yield None + return + worktree_dir = Path(tempfile.gettempdir()) / f"pr-review-{self.pr_number}-{uuid.uuid4().hex[:8]}" created = False try: diff --git a/tools/pr-approval-agent/test_review_pr.py b/tools/pr-approval-agent/test_review_pr.py index ec1ff7086210..33025033be77 100644 --- a/tools/pr-approval-agent/test_review_pr.py +++ b/tools/pr-approval-agent/test_review_pr.py @@ -424,7 +424,7 @@ def __init__(self, returncode: int, stderr: str = "") -> None: def test_pr_head_worktree_yields_path_and_cleans_up(monkeypatch: pytest.MonkeyPatch) -> None: - """On success the context manager yields the worktree path and removes it on exit.""" + """On success (stacked PR) the context manager yields the worktree path and removes it on exit.""" calls: list[list[str]] = [] def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: @@ -434,7 +434,7 @@ def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: monkeypatch.setattr(review_pr.subprocess, "run", fake_run) pipeline = Pipeline(pr_number=42, repo="PostHog/posthog") - pipeline.pr = _fake_pr(head_sha="cafe123") + pipeline.pr = _fake_pr(head_sha="cafe123", base_ref="feat/parent-branch") with pipeline._pr_head_worktree() as explore_root: assert explore_root is not None @@ -458,10 +458,26 @@ def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: monkeypatch.setattr(review_pr.subprocess, "run", fake_run) pipeline = Pipeline(pr_number=7, repo="PostHog/posthog") - pipeline.pr = _fake_pr(head_sha="deadbeef") + pipeline.pr = _fake_pr(head_sha="deadbeef", base_ref="feat/parent-branch") with pipeline._pr_head_worktree() as explore_root: assert explore_root is None # No worktree was created, so none is removed. assert not any("remove" in c for c in calls) + + +def test_pr_head_worktree_skipped_for_non_stacked(monkeypatch: pytest.MonkeyPatch) -> None: + """A non-stacked PR (base is master) reviews from master — no worktree, + so git is never invoked and the full-tree checkout cost is skipped.""" + + def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: + raise AssertionError(f"non-stacked PR must not touch git: {cmd}") + + monkeypatch.setattr(review_pr.subprocess, "run", fake_run) + + pipeline = Pipeline(pr_number=7, repo="PostHog/posthog") + pipeline.pr = _fake_pr(head_sha="cafe123", base_ref="master") + + with pipeline._pr_head_worktree() as explore_root: + assert explore_root is None From 37c44b680736fba1105619b68db6ebe658ebc2b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Thu, 9 Jul 2026 14:15:25 -0400 Subject: [PATCH 004/289] fix(stamphog): harden stacked reviews --- .github/workflows/pr-approval-agent.yml | 34 ++++++---- tools/pr-approval-agent/README.md | 8 ++- tools/pr-approval-agent/review_pr.py | 71 ++++++++++++++++----- tools/pr-approval-agent/reviewer.py | 26 +++++++- tools/pr-approval-agent/test_review_pr.py | 77 ++++++++++++++++++++--- tools/pr-approval-agent/test_reviewer.py | 17 +++++ 6 files changed, 189 insertions(+), 44 deletions(-) diff --git a/.github/workflows/pr-approval-agent.yml b/.github/workflows/pr-approval-agent.yml index 0c93f2893251..baeaf38c79f4 100644 --- a/.github/workflows/pr-approval-agent.yml +++ b/.github/workflows/pr-approval-agent.yml @@ -29,6 +29,7 @@ jobs: needs: [decide-delta, dismiss, dismiss-on-retarget] if: >- always() + && !cancelled() && !github.event.pull_request.draft && github.event.pull_request.user.type != 'Bot' && !contains(github.event.pull_request.user.login, '[bot]') @@ -94,7 +95,7 @@ jobs: --output-json /tmp/review.json - name: Post review - if: always() + if: always() && !cancelled() env: # Use GITHUB_TOKEN for approvals so github-actions[bot] is the # reviewer — its approvals count toward branch protection rules, @@ -112,6 +113,7 @@ jobs: # bullets + folded gate mechanics); fall back to the bare # reasoning when the script predates the field. REASONING=$(jq -r '.review_body // .reviewer.reasoning // ""' /tmp/review.json 2>/dev/null || echo "") + REVIEWED_BASE_SHA=$(jq -r '.base_sha // ""' /tmp/review.json 2>/dev/null || echo "") REVIEWED_SHA=$(jq -r '.head_sha // ""' /tmp/review.json 2>/dev/null || echo "") # Lock the review to the sha the LLM actually saw — `gh pr @@ -123,11 +125,18 @@ jobs: fi if [ "$VERDICT" = "APPROVED" ]; then + CURRENT_REFS=$(gh api "repos/$REPO/pulls/$PR" --jq '[.base.sha, .head.sha] | @tsv') + IFS=$'\t' read -r CURRENT_BASE_SHA CURRENT_HEAD_SHA <<< "$CURRENT_REFS" + if [ "$CURRENT_BASE_SHA" != "$REVIEWED_BASE_SHA" ] || [ "$CURRENT_HEAD_SHA" != "$REVIEWED_SHA" ]; then + echo "PR base or head changed after review; skipping stale approval." + exit 0 + fi + APPROVAL_BODY="${REASONING}"$'\n\n' GH_TOKEN="$GH_TOKEN_APPROVE" gh api \ -X POST "repos/$REPO/pulls/$PR/reviews" \ "${SHA_ARGS[@]}" \ -f event=APPROVE \ - -f body="$REASONING" + -f body="$APPROVAL_BODY" else # Non-approve verdicts share ONE sticky comment, updated in # place, instead of a fresh COMMENT review per run. Review @@ -230,7 +239,7 @@ jobs: fi - name: Upload evidence - if: always() + if: always() && !cancelled() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: review-${{ github.event.pull_request.number }} @@ -348,10 +357,11 @@ jobs: # still carry a stale github-actions[bot] approval from before the # bot-author gate landed, and it must get dismissed on push. decide-delta # is skipped for bots, which routes here via the 'skipped' fail-closed - # path. This only ever touches github-actions[bot] approvals, so it's a - # no-op when there's nothing to dismiss. + # path. This only touches stamphog approvals posted by github-actions[bot] + # and is a no-op when there are none. if: >- always() + && !cancelled() && github.event.action == 'synchronize' && !github.event.pull_request.draft && ( @@ -373,11 +383,12 @@ jobs: run: | set -euo pipefail - # Only dismiss APPROVED reviews made by github-actions[bot] — - # human reviews and non-approval reviews are untouched. + # New approvals carry a hidden marker. Legacy stamphog reviews + # have the gate-mechanics table row, which keeps the transition + # from accidentally retaining an old approval. mapfile -t REVIEW_IDS < <( gh api "repos/$REPO/pulls/$PR/reviews" --paginate \ - --jq '.[] | select(.user.login == "github-actions[bot]" and .state == "APPROVED") | .id' + --jq '.[] | select(.user.login == "github-actions[bot]" and .state == "APPROVED" and ((.body // "") | contains("") or contains("| stamphog "))) | .id' ) for id in "${REVIEW_IDS[@]}"; do @@ -397,7 +408,6 @@ jobs: if: >- github.event.action == 'edited' && github.event.changes.base != null - && !github.event.pull_request.draft runs-on: ubuntu-latest timeout-minutes: 5 @@ -405,8 +415,8 @@ jobs: # The dismissal loop below mirrors the `dismiss` job's. Neither job # checks out the repo, so factoring it into a shared script would # force a checkout into both — more cost than the duplication saves. - # If you change the bot-approval selection (the github-actions[bot] - # + APPROVED filter), update both jobs in lockstep. + # If you change the stamphog-approval selection, update both + # dismissal jobs in lockstep. - name: Dismiss stale bot approvals on base change env: # Same identity (github-actions[bot]) that posted the approval. @@ -418,7 +428,7 @@ jobs: mapfile -t REVIEW_IDS < <( gh api "repos/$REPO/pulls/$PR/reviews" --paginate \ - --jq '.[] | select(.user.login == "github-actions[bot]" and .state == "APPROVED") | .id' + --jq '.[] | select(.user.login == "github-actions[bot]" and .state == "APPROVED" and ((.body // "") | contains("") or contains("| stamphog "))) | .id' ) for id in "${REVIEW_IDS[@]}"; do diff --git a/tools/pr-approval-agent/README.md b/tools/pr-approval-agent/README.md index 14189ee264b2..8a5717162d4c 100644 --- a/tools/pr-approval-agent/README.md +++ b/tools/pr-approval-agent/README.md @@ -145,14 +145,16 @@ these: The head tree already contains the parent PRs' code, so symbols from a not-yet-merged parent resolve and aren't flagged as broken imports. The diff itself is still computed `base_sha...head_sha`, so the review is scoped to - exactly this PR's changes. Worktree creation falls back to reviewing from - master if it fails. + exactly this PR's changes. If the worktree cannot be created, stamphog + returns `ERROR` and retains the label rather than reviewing against the wrong + source tree. - **Security:** the worktree is PR-authored content. The reviewer runs the Agent SDK with `setting_sources=[]` (isolation mode), so it does **not** load `.claude/settings.json` hooks (command execution) or `CLAUDE.md` (injected instructions) from the head tree. Those files are still readable as untrusted _content_ under the anti-injection notice — never as - configuration. + configuration. Stacked PR heads with tracked symbolic links fail closed, + so a PR path cannot resolve outside the worktree. - **Base retarget dismisses the stale approval.** When a stack's parent merges, the child PR is retargeted from the parent branch onto master, changing its diff --git a/tools/pr-approval-agent/review_pr.py b/tools/pr-approval-agent/review_pr.py index 966331872406..7a87afec789e 100644 --- a/tools/pr-approval-agent/review_pr.py +++ b/tools/pr-approval-agent/review_pr.py @@ -123,6 +123,10 @@ def _dim(msg: str) -> str: ) +class WorktreeUnavailableError(RuntimeError): + """The PR head tree required for a stacked review could not be created.""" + + def _is_retryable_error(err_msg: str) -> bool: """Return True if the error looks like an infrastructure/transient issue that is worth retrying (API timeouts, rate limits, overload). @@ -642,7 +646,7 @@ def _check_tier(self) -> tuple[bool, str]: @contextmanager def _pr_head_worktree(self): - """Yield a detached worktree at the PR head, or None to review from master. + """Yield a detached worktree at the PR head, or None for non-stacked PRs. Only stacked PRs need this: their head contains code from parent PRs that aren't on the base branch yet, so without materializing the head @@ -651,7 +655,8 @@ def _pr_head_worktree(self): checkout exactly as before — yield None and skip the full-tree checkout. The main checkout stays master (the workflow hardcodes that so a PR can't swap the review script), so the worktree is the only place the head tree - is materialized. Cleaned up on exit; falls back to None if creation fails. + is materialized. Cleaned up on exit; stacked PRs fail closed if creation + fails rather than reviewing against the wrong source tree. SECURITY: the worktree is PR-authored content; isolation from it as *configuration* is enforced by setting_sources=[] in Reviewer. @@ -661,7 +666,21 @@ def _pr_head_worktree(self): return worktree_dir = Path(tempfile.gettempdir()) / f"pr-review-{self.pr_number}-{uuid.uuid4().hex[:8]}" - created = False + try: + symlink_check = subprocess.run( + ["git", "ls-tree", "-r", "--full-tree", self.pr.head_sha], + capture_output=True, + text=True, + timeout=30, + cwd=REPO_ROOT, + ) + except subprocess.TimeoutExpired as exc: + raise WorktreeUnavailableError("symlink check timed out") from exc + if symlink_check.returncode != 0: + raise WorktreeUnavailableError(f"symlink check failed: {symlink_check.stderr.strip()}") + if any(line.startswith("120000 ") for line in symlink_check.stdout.splitlines()): + raise WorktreeUnavailableError("PR head contains symbolic links") + try: result = subprocess.run( ["git", "worktree", "add", "--detach", str(worktree_dir), self.pr.head_sha], @@ -670,24 +689,28 @@ def _pr_head_worktree(self): timeout=120, cwd=REPO_ROOT, ) - if result.returncode == 0: - created = True - print(_dim(f" Exploring PR head in worktree: {worktree_dir}")) - else: - print(_warn(f"Worktree creation failed, reviewing from master: {result.stderr.strip()}")) - except subprocess.TimeoutExpired: - print(_warn("Worktree creation timed out, reviewing from master")) + except subprocess.TimeoutExpired as exc: + raise WorktreeUnavailableError("worktree creation timed out") from exc + if result.returncode != 0: + raise WorktreeUnavailableError(f"worktree creation failed: {result.stderr.strip()}") + + print(_dim(f" Exploring PR head in worktree: {worktree_dir}")) try: - yield worktree_dir if created else None + yield worktree_dir finally: - if created: - subprocess.run( + try: + cleanup = subprocess.run( ["git", "worktree", "remove", "--force", str(worktree_dir)], capture_output=True, + text=True, timeout=30, cwd=REPO_ROOT, ) + if cleanup.returncode != 0: + print(_warn(f"Worktree cleanup failed (ignored): {cleanup.stderr.strip()}")) + except (OSError, subprocess.TimeoutExpired) as exc: + print(_warn(f"Worktree cleanup failed (ignored): {exc}")) def _run_reviewer_with_retries(self, reviewer: Reviewer, gate_context: dict, diff_path: Path) -> bool: """Call the reviewer with backoff; set self.reviewer_output. @@ -751,7 +774,8 @@ def _run_reviewer_with_retries(self, reviewer: Reviewer, gate_context: dict, dif "issues": [err_str], } return True - return True + + raise AssertionError("review retry loop exhausted without a verdict") def _llm_review(self, gate_verdict: str) -> None: print(f"\n{_bold('LLM Review')}") @@ -765,9 +789,21 @@ def _llm_review(self, gate_verdict: str) -> None: } print(_dim(" Calling reviewer...")) - with self._pr_head_worktree() as explore_root: - reviewer = Reviewer(REPO_ROOT, explore_root=explore_root, verbose=self.verbose) - reviewer_unavailable = self._run_reviewer_with_retries(reviewer, gate_context, diff_path) + try: + with self._pr_head_worktree() as explore_root: + reviewer = Reviewer(REPO_ROOT, explore_root=explore_root, verbose=self.verbose) + reviewer_unavailable = self._run_reviewer_with_retries(reviewer, gate_context, diff_path) + except WorktreeUnavailableError as exc: + reviewer_unavailable = True + self.reviewer_output = { + "verdict": "ERROR", + "reasoning": ( + "The review agent could not create the isolated worktree required to review this stacked PR. " + "The `stamphog` label has been kept; retry after the checkout issue is resolved." + ), + "risk": "unknown", + "issues": [str(exc)], + } llm_verdict = self.reviewer_output.get("verdict", "UNKNOWN") print(f" Verdict: {llm_verdict}") @@ -902,6 +938,7 @@ def to_dict(self) -> dict: "repo": self.pr.repo, "title": self.pr.title, "author": self.pr.author, + "base_sha": self.pr.base_sha, "head_sha": self.pr.head_sha, "classification": { # .get() not [] — the bot-author REFUSE returns before _classify(), diff --git a/tools/pr-approval-agent/reviewer.py b/tools/pr-approval-agent/reviewer.py index 191b6e6eb1f1..c2be362de378 100644 --- a/tools/pr-approval-agent/reviewer.py +++ b/tools/pr-approval-agent/reviewer.py @@ -9,6 +9,7 @@ import json import shutil import asyncio +import tempfile import textwrap from pathlib import Path @@ -285,6 +286,22 @@ def review(self, pr: PRData, classification: dict, gate_context: dict, diff_path """ return asyncio.run(self._review(pr, classification, gate_context, diff_path)) + def _copy_diff_into_explore_root(self, diff_path: Path) -> Path: + """Copy the diff to a runner-created path the agent can read. + + A predictable worktree path could be a tracked symlink. ``mkstemp`` + creates a fresh regular file, so PR content cannot redirect this copy. + """ + fd, copied_path = tempfile.mkstemp(prefix=".pr-review-diff-", suffix=".patch", dir=self.explore_root) + os.close(fd) + copied_diff_path = Path(copied_path) + try: + shutil.copyfile(diff_path, copied_diff_path) + except OSError: + copied_diff_path.unlink(missing_ok=True) + raise + return copied_diff_path + async def _review( self, pr: PRData, classification: dict, gate_context: dict, diff_path: Path | None = None ) -> dict: @@ -292,12 +309,13 @@ async def _review( if diff_path is None: diff_path = self._write_diff_file(pr) original_diff = diff_path + copied_diff_path: Path | None = None if self.explore_root != self.repo_root: # The agent's file access is scoped to cwd (explore_root); a diff # sitting in the master checkout would need an out-of-cwd read the - # dontAsk permission mode never grants. Copy it into the worktree — - # worktree teardown removes the copy. - diff_path = Path(shutil.copyfile(diff_path, self.explore_root / ".pr-review-diff.patch")) + # dontAsk permission mode never grants. + copied_diff_path = self._copy_diff_into_explore_root(diff_path) + diff_path = copied_diff_path prompt = self._build_review_prompt(pr, classification, gate_context, diff_path) # Gate denials and trivial PRs don't need deep exploration — @@ -417,6 +435,8 @@ async def _review( if isinstance(block, ToolUseBlock) and self.verbose: self._log_tool_call(block) + if copied_diff_path is not None: + copied_diff_path.unlink(missing_ok=True) if owns_diff: original_diff.unlink(missing_ok=True) diff --git a/tools/pr-approval-agent/test_review_pr.py b/tools/pr-approval-agent/test_review_pr.py index 33025033be77..69e0c498bc5f 100644 --- a/tools/pr-approval-agent/test_review_pr.py +++ b/tools/pr-approval-agent/test_review_pr.py @@ -1,6 +1,7 @@ """Tests for the review_pr.py output format.""" import sys +from pathlib import Path import pytest from unittest.mock import MagicMock @@ -61,12 +62,10 @@ def test_summarize_assurance_counts_threads_not_flattened_replies() -> None: assert pipeline._summarize_assurance()["unresolved_threads"] == 1 -def test_to_dict_includes_head_sha() -> None: - """The post-review workflow step reads head_sha from the JSON output to - lock the resulting GitHub review to the sha the LLM actually saw — see - `.github/workflows/pr-approval-agent.yml`'s "Post review" step.""" +def test_to_dict_includes_reviewed_base_and_head_shas() -> None: pipeline = Pipeline(pr_number=1, repo="PostHog/posthog") pipeline.pr = _fake_pr(head_sha="07dfeff14d95be1247e4c8c1065fd958a367389e") + pipeline.pr.base_sha = "b5412a26ec97b9d97367c7356cfe9d9b836ae3cb" pipeline.classification = {"tier": "T1-trivial", "breadth": "narrow"} pipeline.gate_results = [] pipeline.reviewer_output = None @@ -74,6 +73,7 @@ def test_to_dict_includes_head_sha() -> None: output = pipeline.to_dict() + assert output["base_sha"] == "b5412a26ec97b9d97367c7356cfe9d9b836ae3cb" assert output["head_sha"] == "07dfeff14d95be1247e4c8c1065fd958a367389e" @@ -418,9 +418,10 @@ def review(self, *args: object, **kwargs: object) -> dict: class _FakeCompleted: - def __init__(self, returncode: int, stderr: str = "") -> None: + def __init__(self, returncode: int, stderr: str = "", stdout: str = "") -> None: self.returncode = returncode self.stderr = stderr + self.stdout = stdout def test_pr_head_worktree_yields_path_and_cleans_up(monkeypatch: pytest.MonkeyPatch) -> None: @@ -447,12 +448,13 @@ def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: assert "--force" in remove -def test_pr_head_worktree_falls_back_to_none_on_failure(monkeypatch: pytest.MonkeyPatch) -> None: - """If worktree creation fails, yield None (review from master) and skip removal.""" +def test_pr_head_worktree_fails_closed_on_creation_failure(monkeypatch: pytest.MonkeyPatch) -> None: calls: list[list[str]] = [] def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: calls.append(cmd) + if "ls-tree" in cmd: + return _FakeCompleted(0) return _FakeCompleted(1, stderr="fatal: invalid reference") monkeypatch.setattr(review_pr.subprocess, "run", fake_run) @@ -460,13 +462,70 @@ def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: pipeline = Pipeline(pr_number=7, repo="PostHog/posthog") pipeline.pr = _fake_pr(head_sha="deadbeef", base_ref="feat/parent-branch") - with pipeline._pr_head_worktree() as explore_root: - assert explore_root is None + with pytest.raises(review_pr.WorktreeUnavailableError, match="invalid reference"): + with pipeline._pr_head_worktree(): + pass # No worktree was created, so none is removed. assert not any("remove" in c for c in calls) +def test_pr_head_worktree_rejects_symlinks(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[list[str]] = [] + + def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: + calls.append(cmd) + if "ls-tree" in cmd: + return _FakeCompleted(0, stdout="120000 blob abcdef\tlink\n") + raise AssertionError(f"symlinked PR must not create a worktree: {cmd}") + + monkeypatch.setattr(review_pr.subprocess, "run", fake_run) + + pipeline = Pipeline(pr_number=7, repo="PostHog/posthog") + pipeline.pr = _fake_pr(head_sha="deadbeef", base_ref="feat/parent-branch") + + with pytest.raises(review_pr.WorktreeUnavailableError, match="symbolic links"): + with pipeline._pr_head_worktree(): + pass + + assert len(calls) == 1 + + +def test_pr_head_worktree_ignores_cleanup_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: + if "remove" in cmd: + raise review_pr.subprocess.TimeoutExpired(cmd, timeout=30) + return _FakeCompleted(0) + + monkeypatch.setattr(review_pr.subprocess, "run", fake_run) + + pipeline = Pipeline(pr_number=7, repo="PostHog/posthog") + pipeline.pr = _fake_pr(head_sha="cafe123", base_ref="feat/parent-branch") + + with pipeline._pr_head_worktree() as explore_root: + assert explore_root is not None + + +def test_stacked_worktree_failure_returns_error(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + @review_pr.contextmanager + def unavailable_worktree(): + raise review_pr.WorktreeUnavailableError("checkout unavailable") + yield None + + pipeline = Pipeline(pr_number=7, repo="PostHog/posthog") + pipeline.pr = _fake_pr(head_sha="cafe123", base_ref="feat/parent-branch") + pipeline.gate_results = [] + monkeypatch.setattr(pipeline, "_pr_head_worktree", unavailable_worktree) + monkeypatch.setattr(pipeline, "_ensure_diff_path", lambda: tmp_path / "diff.patch") + monkeypatch.setattr(review_pr, "_POSTHOG_AVAILABLE", False) + + pipeline._llm_review("PENDING") + + assert pipeline.final_verdict == "ERROR" + assert pipeline.reviewer_output is not None + assert pipeline.reviewer_output["issues"] == ["checkout unavailable"] + + def test_pr_head_worktree_skipped_for_non_stacked(monkeypatch: pytest.MonkeyPatch) -> None: """A non-stacked PR (base is master) reviews from master — no worktree, so git is never invoked and the full-tree checkout cost is skipped.""" diff --git a/tools/pr-approval-agent/test_reviewer.py b/tools/pr-approval-agent/test_reviewer.py index 57d762e4b01a..7795fcb57ecb 100644 --- a/tools/pr-approval-agent/test_reviewer.py +++ b/tools/pr-approval-agent/test_reviewer.py @@ -193,6 +193,23 @@ def test_explore_root_defaults_to_repo_root() -> None: assert Reviewer(repo, explore_root=other).explore_root == other +def test_copy_diff_into_explore_root_cannot_follow_pr_symlink(tmp_path: Path) -> None: + source_diff = tmp_path / "source.patch" + source_diff.write_text("diff --git a/file b/file\n") + explore_root = tmp_path / "explore" + explore_root.mkdir() + outside_target = tmp_path / "outside" + outside_target.write_text("unchanged") + (explore_root / ".pr-review-diff.patch").symlink_to(outside_target) + + copied_diff = Reviewer(tmp_path, explore_root=explore_root)._copy_diff_into_explore_root(source_diff) + + assert copied_diff.parent == explore_root + assert copied_diff.name != ".pr-review-diff.patch" + assert copied_diff.read_text() == source_diff.read_text() + assert outside_target.read_text() == "unchanged" + + @pytest.mark.parametrize( "base_ref, expect_stack_note", [ From c39b46db5273d05bb7d9e723ca1b3f8f56e8e222 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Sun, 26 Jul 2026 19:01:48 +0000 Subject: [PATCH 005/289] feat(data-warehouse): implement apple_search_ads source --- .../temporal/data_imports/sources/SOURCES.md | 2 +- .../apple_search_ads/apple_search_ads.py | 578 ++++++++++++++++ .../canonical_descriptions.py | 150 ++++ .../sources/apple_search_ads/settings.py | 124 ++++ .../sources/apple_search_ads/source.py | 193 +++++- .../tests/test_apple_search_ads.py | 654 ++++++++++++++++++ .../tests/test_apple_search_ads_source.py | 201 ++++++ .../generated_configs/applesearchads.py | 7 +- 8 files changed, 1902 insertions(+), 7 deletions(-) create mode 100644 products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py create mode 100644 products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/canonical_descriptions.py create mode 100644 products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/settings.py create mode 100644 products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads.py create mode 100644 products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads_source.py diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md index 297a37161047..475dd539e514 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md @@ -64,6 +64,7 @@ the row lists both. | appdynamics | HTTP | requests | ✅ | | appfigures | HTTP | requests | ✅ | | appfollow | HTTP | requests | ✅ | +| apple_search_ads | HTTP | requests | ✅ | | appsflyer | HTTP (CSV reports) | requests | ✅ | | appsignal | HTTP (REST + GraphQL) | requests | ✅ | | appstack | HTTP | requests + `rest_source.RESTClient` | ✅ | @@ -722,7 +723,6 @@ doesn't conflict with concurrent PRs. - appcues - appdirect - appfolio -- apple_search_ads - apptivo - appwrite - arxiv diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py new file mode 100644 index 000000000000..6d607536ed50 --- /dev/null +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py @@ -0,0 +1,578 @@ +import time +import dataclasses +from collections.abc import Iterator +from datetime import UTC, date, datetime, timedelta +from typing import Any, Optional + +import jwt +import requests +import structlog +from structlog.types import FilteringBoundLogger +from urllib3.util.retry import Retry + +from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline.typings import SourceResponse +from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.settings import ( + APPLE_SEARCH_ADS_ENDPOINTS, + DEFAULT_INITIAL_LOOKBACK_DAYS, + PAGE_SIZE, + REPORT_WINDOW_DAYS, + AppleSearchAdsEndpointConfig, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session +from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager + +APPLE_SEARCH_ADS_HOST = "https://api.searchads.apple.com" +# Apple Search Ads authenticates through Apple ID's OAuth token endpoint, not a Search Ads host. +APPLE_OAUTH_TOKEN_URL = "https://appleid.apple.com/auth/oauth2/token" +APPLE_OAUTH_AUDIENCE = "https://appleid.apple.com" +APPLE_OAUTH_SCOPE = "searchadsorg" + +# Lifetime of the ES256 client-secret assertion we sign per token exchange. Apple allows up to +# 180 days; we mint a fresh short-lived one every time so no long-lived secret is stored. +CLIENT_SECRET_TTL_SECONDS = 30 * 60 +REQUEST_TIMEOUT_SECONDS = 120 + +# Cheap org-scoped probe for credential validation: it exercises the access token *and* the +# `X-AP-Context` org id, which `/acls` would not. +CREDENTIAL_PROBE_PATH = "/campaigns" + +# Apple's documented ceiling is 100 requests/minute per account, surfaced as 429 with +# `Retry-After`. The transport honors that header; POST is added to the retryable methods +# because every read path here except `/campaigns` and `/acls` is a side-effect-free POST +# (`/find`, `/reports/...`) that urllib3 would otherwise refuse to retry. +APPLE_SEARCH_ADS_RETRY = Retry( + total=5, + backoff_factor=1.0, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=frozenset(["GET", "HEAD", "OPTIONS", "POST"]), + respect_retry_after_header=True, + raise_on_status=False, +) + +logger = structlog.get_logger(__name__) + + +class AppleSearchAdsAuthError(Exception): + pass + + +@dataclasses.dataclass(frozen=True) +class AppleSearchAdsCredentials: + org_id: str + client_id: str + team_id: str + key_id: str + private_key: str + + +@dataclasses.dataclass +class AppleSearchAdsResumeConfig: + # Offset into the current page set. Entity endpoints only ever use this field. + offset: int = 0 + # ISO date of the reporting window in progress, and the campaign it was being fanned out + # to. Both are matched by value on resume, so a changed campaign list restarts the run's + # window range rather than silently skipping a campaign. + window_start: Optional[str] = None + campaign_id: Optional[int] = None + + +def _normalize_private_key(private_key: str) -> str: + """Accept a PEM pasted with literal ``\\n`` escapes as well as real newlines.""" + return private_key.replace("\\n", "\n").strip() + + +def build_client_secret(credentials: AppleSearchAdsCredentials, *, issued_at: Optional[int] = None) -> str: + """Sign the ES256 client-secret assertion Apple's token endpoint expects. + + Apple Search Ads has no static client secret: the caller signs a JWT with the private key + whose public half was uploaded in the Search Ads UI (`kid` = key id, `iss` = team id, + `sub` = client id) and presents that as `client_secret`. + """ + now = int(issued_at if issued_at is not None else time.time()) + try: + return jwt.encode( + { + "sub": credentials.client_id, + "aud": APPLE_OAUTH_AUDIENCE, + "iat": now, + "exp": now + CLIENT_SECRET_TTL_SECONDS, + "iss": credentials.team_id, + }, + _normalize_private_key(credentials.private_key), + algorithm="ES256", + headers={"alg": "ES256", "kid": credentials.key_id}, + ) + except (jwt.PyJWTError, ValueError, TypeError) as e: + raise AppleSearchAdsAuthError( + "Could not sign the Apple Search Ads client secret. The private key must be the " + f"unencrypted EC (P-256) PEM generated for your Search Ads API key: {e}" + ) from e + + +class AppleSearchAdsClient: + """Minimal Campaign Management API client: token minting plus JSON request helpers.""" + + def __init__( + self, + credentials: AppleSearchAdsCredentials, + api_version: str, + request_logger: Optional[FilteringBoundLogger] = None, + ) -> None: + self._credentials = credentials + self._base_url = f"{APPLE_SEARCH_ADS_HOST}/api/{api_version}" + self._logger: FilteringBoundLogger = request_logger or logger + self._access_token: Optional[str] = None + self._session = make_tracked_session( + retry=APPLE_SEARCH_ADS_RETRY, + redact_values=(credentials.private_key,), + ) + # The token exchange body carries the signed assertion and the response the bearer + # token, neither of which the name-based sample scrubbers would recognise. + self._token_session = make_tracked_session( + retry=APPLE_SEARCH_ADS_RETRY, + redact_values=(credentials.private_key,), + capture=False, + ) + + @property + def base_url(self) -> str: + return self._base_url + + def authenticate(self) -> str: + self._access_token = self._mint_access_token() + return self._access_token + + def _mint_access_token(self) -> str: + client_secret = build_client_secret(self._credentials) + response = self._token_session.post( + APPLE_OAUTH_TOKEN_URL, + data={ + "grant_type": "client_credentials", + "client_id": self._credentials.client_id, + "client_secret": client_secret, + "scope": APPLE_OAUTH_SCOPE, + }, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + timeout=REQUEST_TIMEOUT_SECONDS, + ) + if not response.ok: + self._logger.error( + f"Apple Search Ads token exchange failed: status={response.status_code}, url={APPLE_OAUTH_TOKEN_URL}" + ) + response.raise_for_status() + + access_token = response.json().get("access_token") + if not access_token: + raise AppleSearchAdsAuthError("Apple's token response did not contain an access token") + return str(access_token) + + def _headers(self, requires_org_context: bool) -> dict[str, str]: + headers = {"Authorization": f"Bearer {self._access_token}", "Accept": "application/json"} + if requires_org_context: + headers["X-AP-Context"] = f"orgId={self._credentials.org_id}" + return headers + + def _send( + self, + method: str, + url: str, + *, + params: Optional[dict[str, Any]], + body: Optional[dict[str, Any]], + requires_org_context: bool, + ) -> requests.Response: + headers = self._headers(requires_org_context) + if method == "POST": + return self._session.post(url, json=body or {}, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS) + return self._session.get(url, params=params, headers=headers, timeout=REQUEST_TIMEOUT_SECONDS) + + def _request( + self, + method: str, + path: str, + *, + params: Optional[dict[str, Any]] = None, + body: Optional[dict[str, Any]] = None, + requires_org_context: bool = True, + ) -> requests.Response: + if self._access_token is None: + self.authenticate() + + url = f"{self._base_url}{path}" + response = self._send(method, url, params=params, body=body, requires_org_context=requires_org_context) + # Access tokens live an hour, which a backfill routinely outlives — re-mint once and + # replay before treating a 401 as a credential problem. + if response.status_code == 401: + self.authenticate() + response = self._send(method, url, params=params, body=body, requires_org_context=requires_org_context) + return response + + def request_json( + self, + method: str, + path: str, + *, + params: Optional[dict[str, Any]] = None, + body: Optional[dict[str, Any]] = None, + requires_org_context: bool = True, + ) -> dict[str, Any]: + response = self._request(method, path, params=params, body=body, requires_org_context=requires_org_context) + if not response.ok: + self._logger.error( + f"Apple Search Ads API error: status={response.status_code}, " + f"body={response.text}, url={self._base_url}{path}" + ) + response.raise_for_status() + + payload = response.json() + return payload if isinstance(payload, dict) else {} + + def probe_status(self, path: str, *, params: Optional[dict[str, Any]] = None) -> int: + return self._request("GET", path, params=params).status_code + + +def validate_credentials( + credentials: AppleSearchAdsCredentials, + api_version: str, + schema_name: Optional[str] = None, +) -> tuple[bool, str | None]: + """Mint a token and probe one org-scoped endpoint. + + A 403 means the credentials are genuine but the role can't read this resource; accepted at + source-create (``schema_name is None``) so a user who only granted a subset of access can + still connect, and reported per-table otherwise. + """ + client = AppleSearchAdsClient(credentials, api_version) + try: + client.authenticate() + except AppleSearchAdsAuthError as e: + return False, str(e) + except requests.RequestException as e: + return False, f"Could not exchange the Apple Search Ads credentials for an access token: {e}" + + try: + status = client.probe_status(CREDENTIAL_PROBE_PATH, params={"limit": 1}) + except requests.RequestException as e: + return False, f"Could not reach the Apple Search Ads API: {e}" + + if status == 200: + return True, None + if status == 401: + return False, "Apple Search Ads rejected the access token. Check the client ID, team ID and key ID." + if status == 403: + if schema_name is None: + return True, None + return False, "These Apple Search Ads credentials do not have permission to read this table." + return False, f"Apple Search Ads returned an unexpected status code: {status}" + + +def _today() -> date: + return datetime.now(UTC).date() + + +def _to_date(value: Any) -> Optional[date]: + """Coerce an incremental cursor value (date/datetime/ISO string) to a plain date.""" + if value is None: + return None + if isinstance(value, datetime): + return value.astimezone(UTC).date() if value.tzinfo is not None else value.date() + if isinstance(value, date): + return value + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")).date() + except ValueError: + return None + + +def _report_start_date( + should_use_incremental_field: bool, + db_incremental_field_last_value: Any, + start_date: Optional[str], + today: date, +) -> date: + """First reporting day to request. + + The pipeline already shifts the stored watermark back by the schema's lookback, so it is + used verbatim. Without a watermark the run starts at the user's configured start date, or + a bounded default so a first sync can't walk the whole account history. + """ + if should_use_incremental_field: + watermark = _to_date(db_incremental_field_last_value) + if watermark is not None: + return watermark + + configured = _to_date(start_date) if start_date else None + if configured is not None: + return configured + return today - timedelta(days=DEFAULT_INITIAL_LOOKBACK_DAYS) + + +def _report_windows(start: date, end: date) -> list[tuple[date, date]]: + """Split ``start..end`` (inclusive) into ascending windows of at most one week.""" + windows: list[tuple[date, date]] = [] + window_start = start + while window_start <= end: + window_end = min(window_start + timedelta(days=REPORT_WINDOW_DAYS - 1), end) + windows.append((window_start, window_end)) + window_start = window_end + timedelta(days=1) + return windows + + +def _report_body(window_start: date, window_end: date, offset: int) -> dict[str, Any]: + return { + "startTime": window_start.isoformat(), + "endTime": window_end.isoformat(), + "granularity": "DAILY", + # Report in the organization's own time zone so the `date` column matches what the + # Search Ads UI shows for the same campaign. + "timeZone": "ORTZ", + "selector": { + "conditions": [], + # No `orderBy`: Apple's sortable-field enum differs per report level and rejects + # unknown fields, and rows are keyed by entity + date so merge order is irrelevant. + "pagination": {"offset": offset, "limit": PAGE_SIZE}, + }, + "returnRecordsWithNoMetrics": False, + "returnRowTotals": False, + "returnGrandTotals": False, + } + + +def _report_page_rows(payload: dict[str, Any]) -> list[dict[str, Any]]: + reporting_data = payload.get("data") or {} + if not isinstance(reporting_data, dict): + return [] + response = reporting_data.get("reportingDataResponse") or {} + if not isinstance(response, dict): + return [] + rows = response.get("row") or [] + return [row for row in rows if isinstance(row, dict)] + + +def flatten_report_rows(payload: dict[str, Any], campaign_id: Optional[int]) -> list[dict[str, Any]]: + """Explode one report page into a row per entity per day. + + Apple returns one row per entity carrying a `metadata` block plus a `granularity` array of + daily metric buckets; the warehouse wants those flattened so `date` is a real column. + """ + flattened: list[dict[str, Any]] = [] + for row in _report_page_rows(payload): + metadata = dict(row.get("metadata") or {}) + if campaign_id is not None: + # Ad-group and keyword reports are requested per campaign and their metadata does + # not repeat the campaign id the primary key needs. + metadata.setdefault("campaignId", campaign_id) + for daily in row.get("granularity") or []: + if isinstance(daily, dict): + flattened.append({**metadata, **daily}) + return flattened + + +def _entity_page( + client: AppleSearchAdsClient, config: AppleSearchAdsEndpointConfig, offset: int +) -> list[dict[str, Any]]: + if config.kind == "find": + payload = client.request_json( + "POST", + config.path, + body={"conditions": [], "pagination": {"offset": offset, "limit": PAGE_SIZE}}, + requires_org_context=config.requires_org_context, + ) + elif config.kind == "query_page": + payload = client.request_json( + "GET", + config.path, + params={"limit": PAGE_SIZE, "offset": offset}, + requires_org_context=config.requires_org_context, + ) + else: + payload = client.request_json("GET", config.path, requires_org_context=config.requires_org_context) + + rows = payload.get("data") + return [row for row in rows if isinstance(row, dict)] if isinstance(rows, list) else [] + + +def _iter_entity_rows( + client: AppleSearchAdsClient, + config: AppleSearchAdsEndpointConfig, + resumable_source_manager: ResumableSourceManager[AppleSearchAdsResumeConfig], + resume: Optional[AppleSearchAdsResumeConfig], +) -> Iterator[list[dict[str, Any]]]: + offset = resume.offset if resume is not None else 0 + + while True: + rows = _entity_page(client, config, offset) + if rows: + yield rows + + if config.kind == "single" or len(rows) < PAGE_SIZE: + break + + offset += len(rows) + resumable_source_manager.save_state(AppleSearchAdsResumeConfig(offset=offset)) + + +def _list_campaign_ids(client: AppleSearchAdsClient) -> list[int]: + """Campaign ids to fan the per-campaign report endpoints out over, in a stable order.""" + campaigns_config = APPLE_SEARCH_ADS_ENDPOINTS["campaigns"] + ids: set[int] = set() + offset = 0 + while True: + rows = _entity_page(client, campaigns_config, offset) + for row in rows: + campaign_id = row.get("id") + if campaign_id is not None: + ids.add(int(campaign_id)) + if len(rows) < PAGE_SIZE: + break + offset += len(rows) + return sorted(ids) + + +def _resume_index( + tasks: list[tuple[date, date, Optional[int]]], + resume: Optional[AppleSearchAdsResumeConfig], + request_logger: FilteringBoundLogger, +) -> tuple[int, int]: + """Locate a saved checkpoint in this run's task list, by value rather than position.""" + if resume is None or not resume.window_start: + return 0, 0 + + key = (resume.window_start, resume.campaign_id) + for index, (window_start, _window_end, campaign_id) in enumerate(tasks): + if (window_start.isoformat(), campaign_id) == key: + return index, resume.offset + + request_logger.debug( + f"Apple Search Ads: saved checkpoint {key} is not in this run's window range, starting from the beginning" + ) + return 0, 0 + + +def _iter_report_rows( + client: AppleSearchAdsClient, + config: AppleSearchAdsEndpointConfig, + resumable_source_manager: ResumableSourceManager[AppleSearchAdsResumeConfig], + resume: Optional[AppleSearchAdsResumeConfig], + request_logger: FilteringBoundLogger, + *, + should_use_incremental_field: bool, + db_incremental_field_last_value: Any, + start_date: Optional[str], +) -> Iterator[list[dict[str, Any]]]: + today = _today() + start = _report_start_date(should_use_incremental_field, db_incremental_field_last_value, start_date, today) + campaign_ids: list[Optional[int]] = [None] + if config.fan_out_over_campaigns: + campaign_ids = list(_list_campaign_ids(client)) + + tasks = [ + (window_start, window_end, campaign_id) + for window_start, window_end in _report_windows(start, today) + for campaign_id in campaign_ids + ] + start_index, start_offset = _resume_index(tasks, resume, request_logger) + + for index in range(start_index, len(tasks)): + window_start, window_end, campaign_id = tasks[index] + path = config.path.format(campaign_id=campaign_id) if campaign_id is not None else config.path + offset = start_offset if index == start_index else 0 + + while True: + payload = client.request_json( + "POST", + path, + body=_report_body(window_start, window_end, offset), + requires_org_context=config.requires_org_context, + ) + rows = flatten_report_rows(payload, campaign_id) + if rows: + yield rows + + page_size = len(_report_page_rows(payload)) + if page_size < PAGE_SIZE: + break + + offset += page_size + resumable_source_manager.save_state( + AppleSearchAdsResumeConfig( + offset=offset, window_start=window_start.isoformat(), campaign_id=campaign_id + ) + ) + + if index + 1 < len(tasks): + next_window_start, _next_window_end, next_campaign_id = tasks[index + 1] + resumable_source_manager.save_state( + AppleSearchAdsResumeConfig( + offset=0, window_start=next_window_start.isoformat(), campaign_id=next_campaign_id + ) + ) + + +def get_rows( + credentials: AppleSearchAdsCredentials, + endpoint: str, + api_version: str, + request_logger: FilteringBoundLogger, + resumable_source_manager: ResumableSourceManager[AppleSearchAdsResumeConfig], + should_use_incremental_field: bool = False, + db_incremental_field_last_value: Any = None, + start_date: Optional[str] = None, +) -> Iterator[list[dict[str, Any]]]: + config = APPLE_SEARCH_ADS_ENDPOINTS[endpoint] + client = AppleSearchAdsClient(credentials, api_version, request_logger) + resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None + + if config.kind == "report": + yield from _iter_report_rows( + client, + config, + resumable_source_manager, + resume, + request_logger, + should_use_incremental_field=should_use_incremental_field, + db_incremental_field_last_value=db_incremental_field_last_value, + start_date=start_date, + ) + else: + yield from _iter_entity_rows(client, config, resumable_source_manager, resume) + + # The stream ran to completion; leaving the last checkpoint would make a later attempt + # resume mid-range instead of restarting cleanly. + resumable_source_manager.clear_state() + + +def apple_search_ads_source( + credentials: AppleSearchAdsCredentials, + endpoint: str, + api_version: str, + request_logger: FilteringBoundLogger, + resumable_source_manager: ResumableSourceManager[AppleSearchAdsResumeConfig], + should_use_incremental_field: bool = False, + db_incremental_field_last_value: Any = None, + start_date: Optional[str] = None, +) -> SourceResponse: + config = APPLE_SEARCH_ADS_ENDPOINTS[endpoint] + + return SourceResponse( + name=endpoint, + items=lambda: get_rows( + credentials=credentials, + endpoint=endpoint, + api_version=api_version, + request_logger=request_logger, + resumable_source_manager=resumable_source_manager, + should_use_incremental_field=should_use_incremental_field, + db_incremental_field_last_value=db_incremental_field_last_value, + start_date=start_date, + ), + primary_keys=list(config.primary_keys), + # Reporting windows are walked oldest-first, so `date` only ever moves forward across + # batches by at most one window — which the schema's trailing lookback re-reads. + sort_mode="asc", + partition_count=1, + partition_size=1, + partition_mode="datetime" if config.partition_key else None, + partition_format="month" if config.partition_key else None, + partition_keys=[config.partition_key] if config.partition_key else None, + ) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/canonical_descriptions.py new file mode 100644 index 000000000000..77e1b4b22940 --- /dev/null +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/canonical_descriptions.py @@ -0,0 +1,150 @@ +"""Canonical, documentation-sourced descriptions for Apple Search Ads endpoints and columns. + +Sourced from Apple's Search Ads Campaign Management API v5 reference +(https://developer.apple.com/documentation/apple_search_ads). Keyed by the endpoint names in +`settings.py` `APPLE_SEARCH_ADS_ENDPOINTS`, which match the `ExternalDataSchema.name` of a synced +table. Columns absent here fall back to LLM enrichment. +""" + +from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import ( + CanonicalDescriptions, +) + +# Metrics shared by every reporting table, so the daily grain reads the same everywhere. +_REPORT_METRIC_COLUMNS: dict[str, str] = { + "date": "Calendar day the metrics cover, in the organization's time zone.", + "impressions": "Number of times the ad was shown on that day.", + "taps": "Number of taps on the ad.", + "installs": "Total conversions attributed to the ad, combining new downloads and redownloads.", + "newDownloads": "Conversions from users who had not previously downloaded the app.", + "redownloads": "Conversions from users who had previously downloaded the app.", + "latOnInstalls": "Conversions from devices with Limit Ad Tracking enabled.", + "latOffInstalls": "Conversions from devices with Limit Ad Tracking disabled.", + "ttr": "Tap-through rate: taps divided by impressions.", + "conversionRate": "Conversion rate: installs divided by taps.", + "localSpend": "Amount spent on that day, as an amount plus currency code.", + "avgCPA": "Average cost per acquisition, in the organization's currency.", + "avgCPT": "Average cost per tap, in the organization's currency.", + "avgCPM": "Average cost per thousand impressions, in the organization's currency.", +} + +CANONICAL_DESCRIPTIONS: CanonicalDescriptions = { + "acls": { + "description": "Organizations the API credentials can read, with the role granted to them.", + "docs_url": "https://developer.apple.com/documentation/apple_search_ads/get_user_acl", + "columns": { + "orgId": "Identifier of the organization, used as the `orgId` in the API context header.", + "orgName": "Display name of the organization.", + "currency": "Three-letter ISO currency code the organization is billed in.", + "timeZone": "Time zone the organization's reporting is expressed in.", + "paymentModel": "Billing model for the organization: LOC (line of credit), PAYG, or unset.", + "roleNames": "Roles the API user holds on the organization, such as API Read Only.", + }, + }, + "campaigns": { + "description": "Campaigns in the organization, each targeting one app in one or more storefronts.", + "docs_url": "https://developer.apple.com/documentation/apple_search_ads/campaign", + "columns": { + "id": "Unique identifier for the campaign.", + "orgId": "Identifier of the organization that owns the campaign.", + "name": "Name of the campaign.", + "adamId": "App Store identifier of the app the campaign promotes.", + "budgetAmount": "Total budget for the campaign, as an amount plus currency code.", + "dailyBudgetAmount": "Daily budget cap, as an amount plus currency code.", + "countriesOrRegions": "Storefronts the campaign runs in, as country or region codes.", + "adChannelType": "Channel the campaign advertises on, such as SEARCH or DISPLAY.", + "supplySources": "App Store placements the campaign serves in, such as APPSTORE_SEARCH_RESULTS.", + "billingEvent": "Event the campaign is billed on — TAPS for Search Ads campaigns.", + "paymentModel": "Billing model in effect for the campaign.", + "startTime": "When the campaign starts serving.", + "endTime": "When the campaign stops serving, if an end is set.", + "status": "Status the advertiser set: ENABLED or PAUSED.", + "servingStatus": "Whether the campaign is currently RUNNING or NOT_RUNNING.", + "servingStateReasons": "Reasons the campaign is not serving, if any.", + "displayStatus": "Combined status shown in the Search Ads UI.", + "modificationTime": "When the campaign was last changed.", + "deleted": "Whether the campaign has been deleted.", + }, + }, + "ad_groups": { + "description": "Ad groups across every campaign in the organization, holding bids and targeting.", + "docs_url": "https://developer.apple.com/documentation/apple_search_ads/adgroup", + "columns": { + "id": "Unique identifier for the ad group.", + "campaignId": "Identifier of the campaign the ad group belongs to.", + "orgId": "Identifier of the organization that owns the ad group.", + "name": "Name of the ad group.", + "defaultBidAmount": "Default cost-per-tap bid, as an amount plus currency code.", + "cpaGoal": "Optional cost-per-acquisition goal, as an amount plus currency code.", + "pricingModel": "Pricing model for the ad group, such as CPC.", + "automatedKeywordsOptIn": "Whether Apple may add matching keywords automatically.", + "targetingDimensions": "Audience, device, demographic and locality targeting for the ad group.", + "startTime": "When the ad group starts serving.", + "endTime": "When the ad group stops serving, if an end is set.", + "status": "Status the advertiser set: ENABLED or PAUSED.", + "servingStatus": "Whether the ad group is currently RUNNING or NOT_RUNNING.", + "servingStateReasons": "Reasons the ad group is not serving, if any.", + "displayStatus": "Combined status shown in the Search Ads UI.", + "modificationTime": "When the ad group was last changed.", + "deleted": "Whether the ad group has been deleted.", + }, + }, + "keywords": { + "description": "Targeting keywords across every ad group in the organization.", + "docs_url": "https://developer.apple.com/documentation/apple_search_ads/keyword", + "columns": { + "id": "Unique identifier for the keyword.", + "adGroupId": "Identifier of the ad group the keyword targets within.", + "campaignId": "Identifier of the campaign the keyword belongs to.", + "text": "The keyword text bid on.", + "matchType": "How the search term must match the keyword: EXACT or BROAD.", + "bidAmount": "Cost-per-tap bid for the keyword, as an amount plus currency code.", + "status": "Status the advertiser set: ACTIVE or PAUSED.", + "modificationTime": "When the keyword was last changed.", + "deleted": "Whether the keyword has been deleted.", + }, + }, + "campaign_report": { + "description": "Daily performance metrics per campaign, one row per campaign per day.", + "docs_url": "https://developer.apple.com/documentation/apple_search_ads/get_campaign-level_reports", + "columns": { + "campaignId": "Identifier of the campaign the metrics belong to.", + "campaignName": "Name of the campaign at the time the report was run.", + "campaignStatus": "Status of the campaign at the time the report was run.", + "app": "App the campaign promotes, with its App Store identifier and name.", + "countriesOrRegions": "Storefronts the campaign served in.", + "deleted": "Whether the campaign has since been deleted.", + **_REPORT_METRIC_COLUMNS, + }, + }, + "ad_group_report": { + "description": "Daily performance metrics per ad group, one row per ad group per day.", + "docs_url": "https://developer.apple.com/documentation/apple_search_ads/get_ad_group-level_reports", + "columns": { + "campaignId": "Identifier of the campaign the ad group belongs to.", + "adGroupId": "Identifier of the ad group the metrics belong to.", + "adGroupName": "Name of the ad group at the time the report was run.", + "adGroupStatus": "Status of the ad group at the time the report was run.", + "defaultBidAmount": "Default cost-per-tap bid in effect for the ad group.", + "deleted": "Whether the ad group has since been deleted.", + **_REPORT_METRIC_COLUMNS, + }, + }, + "keyword_report": { + "description": "Daily performance metrics per targeting keyword, one row per keyword per day.", + "docs_url": "https://developer.apple.com/documentation/apple_search_ads/get_keyword-level_reports", + "columns": { + "campaignId": "Identifier of the campaign the keyword belongs to.", + "keywordId": "Identifier of the keyword the metrics belong to.", + "keyword": "The keyword text bid on.", + "matchType": "How the search term matched the keyword: EXACT or BROAD.", + "adGroupId": "Identifier of the ad group the keyword targets within.", + "adGroupName": "Name of the ad group at the time the report was run.", + "bid": "Cost-per-tap bid in effect for the keyword.", + "keywordStatus": "Status of the keyword at the time the report was run.", + "keywordDisplayStatus": "Combined keyword status shown in the Search Ads UI.", + "deleted": "Whether the keyword has since been deleted.", + **_REPORT_METRIC_COLUMNS, + }, + }, +} diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/settings.py new file mode 100644 index 000000000000..4f1d2f1936fd --- /dev/null +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/settings.py @@ -0,0 +1,124 @@ +from dataclasses import dataclass, field +from typing import Literal, Optional + +from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field +from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType + +# How each endpoint is read: +# single — one GET, whole result set in `data` (no pagination params). +# query_page — GET with `limit`/`offset` query params. +# find — POST whose body *is* a Selector (`conditions`/`pagination`). +# report — POST a date-bounded report request; rows arrive nested under +# `data.reportingDataResponse.row`, one per entity with a daily +# `granularity` array. +EndpointKind = Literal["single", "query_page", "find", "report"] + +# Apple caps `limit` (entity endpoints) and `selector.pagination.limit` (find/report) at 1000. +PAGE_SIZE = 1000 + +# Reporting requests are bounded to a short window so that the per-batch incremental +# watermark can never advance further ahead of the data than the trailing lookback below +# re-reads. Within one window rows arrive grouped by entity rather than by date, so the +# window length is the ordering error budget — keep it <= the lookback. +REPORT_WINDOW_DAYS = 7 + +# Apple restates recent reporting rows (3-4h ingestion delay plus attribution), so every +# incremental run re-reads a trailing week rather than trusting the frozen watermark. +REPORT_LOOKBACK_SECONDS = REPORT_WINDOW_DAYS * 24 * 60 * 60 + +# How far back the first sync of a report table reaches when the user gives no start date. +DEFAULT_INITIAL_LOOKBACK_DAYS = 365 + + +@dataclass +class AppleSearchAdsEndpointConfig: + name: str + # Path under `https://api.searchads.apple.com/api/{version}`. + path: str + kind: EndpointKind + primary_keys: list[str] + # Every Campaign Management endpoint except `/acls` is scoped to one organization via + # the `X-AP-Context: orgId=...` header. + requires_org_context: bool = True + # Apple only exposes ad-group/keyword level reports per campaign, so those tables are + # built by fanning out over the org's campaign ids. + fan_out_over_campaigns: bool = False + incremental_fields: list[IncrementalField] = field(default_factory=list) + # Reporting date — set by Apple, never restated to a different day, so it is a stable + # partition key. + partition_key: Optional[str] = None + + +APPLE_SEARCH_ADS_ENDPOINTS: dict[str, AppleSearchAdsEndpointConfig] = { + "acls": AppleSearchAdsEndpointConfig( + name="acls", + path="/acls", + kind="single", + primary_keys=["orgId"], + requires_org_context=False, + ), + "campaigns": AppleSearchAdsEndpointConfig( + name="campaigns", + path="/campaigns", + kind="query_page", + primary_keys=["id"], + ), + "ad_groups": AppleSearchAdsEndpointConfig( + name="ad_groups", + path="/adgroups/find", + kind="find", + primary_keys=["id"], + ), + "keywords": AppleSearchAdsEndpointConfig( + name="keywords", + path="/targetingkeywords/find", + kind="find", + primary_keys=["id"], + ), + "campaign_report": AppleSearchAdsEndpointConfig( + name="campaign_report", + path="/reports/campaigns", + kind="report", + primary_keys=["campaignId", "date"], + partition_key="date", + incremental_fields=[incremental_field("date", IncrementalFieldType.Date)], + ), + "ad_group_report": AppleSearchAdsEndpointConfig( + name="ad_group_report", + path="/reports/campaigns/{campaign_id}/adgroups", + kind="report", + fan_out_over_campaigns=True, + primary_keys=["campaignId", "adGroupId", "date"], + partition_key="date", + incremental_fields=[incremental_field("date", IncrementalFieldType.Date)], + ), + "keyword_report": AppleSearchAdsEndpointConfig( + name="keyword_report", + path="/reports/campaigns/{campaign_id}/keywords", + kind="report", + fan_out_over_campaigns=True, + # Apple keyword ids are unique across ad groups, so the campaign the row was fanned + # out from plus the keyword and date identify a row table-wide. + primary_keys=["campaignId", "keywordId", "date"], + partition_key="date", + incremental_fields=[incremental_field("date", IncrementalFieldType.Date)], + ), +} + +ENDPOINTS = tuple(APPLE_SEARCH_ADS_ENDPOINTS.keys()) + +INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = { + name: config.incremental_fields for name, config in APPLE_SEARCH_ADS_ENDPOINTS.items() +} + +REPORT_ENDPOINTS = tuple(name for name, config in APPLE_SEARCH_ADS_ENDPOINTS.items() if config.kind == "report") + +ENDPOINT_DESCRIPTIONS: dict[str, str] = { + "acls": "Organizations the API credentials can read, with currency, time zone and role names.", + "campaigns": "Campaigns in the organization, with budget, serving status and countries or regions.", + "ad_groups": "Ad groups across every campaign in the organization, with default bid and targeting.", + "keywords": "Targeting keywords across every ad group in the organization, with match type and bid.", + "campaign_report": "Daily campaign performance: impressions, taps, installs, spend and derived rates.", + "ad_group_report": "Daily ad group performance for every campaign in the organization.", + "keyword_report": "Daily keyword performance for every campaign in the organization.", +} diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py index 08adf15b67c6..835ed5905595 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py @@ -1,13 +1,42 @@ -from typing import cast +from typing import Optional, cast from posthog.schema import ( DataWarehouseSourceCategory, ExternalDataSourceType as SchemaExternalDataSourceType, + ReleaseStatus, SourceConfig, + SourceFieldInputConfig, + SourceFieldInputConfigType, ) -from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource +from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline.typings import ( + SourceInputs, + SourceResponse, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.apple_search_ads import ( + AppleSearchAdsCredentials, + AppleSearchAdsResumeConfig, + apple_search_ads_source, + validate_credentials as validate_apple_search_ads_credentials, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.settings import ( + APPLE_SEARCH_ADS_ENDPOINTS, + ENDPOINT_DESCRIPTIONS, + ENDPOINTS, + INCREMENTAL_FIELDS, + REPORT_ENDPOINTS, + REPORT_LOOKBACK_SECONDS, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource +from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import ( + CanonicalDescriptions, +) from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry +from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager +from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import ( + SourceSchema, + build_endpoint_schemas, +) from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.applesearchads import ( AppleSearchAdsSourceConfig, ) @@ -15,18 +44,172 @@ @SourceRegistry.register -class AppleSearchAdsSource(SimpleSource[AppleSearchAdsSourceConfig]): +class AppleSearchAdsSource(ResumableSource[AppleSearchAdsSourceConfig, AppleSearchAdsResumeConfig]): + lists_tables_without_credentials = True # static endpoint catalog — safe for public docs + + supported_versions = ("v5",) + default_version = "v5" + api_docs_url = "https://developer.apple.com/documentation/apple_search_ads" + @property def source_type(self) -> ExternalDataSourceType: return ExternalDataSourceType.APPLESEARCHADS + def get_non_retryable_errors(self) -> dict[str, str | None]: + return { + "400 Client Error: Bad Request for url: https://appleid.apple.com/auth/oauth2/token": "Apple rejected the signed client secret. Check your client ID, team ID, key ID and private key.", + "401 Client Error: Unauthorized for url: https://appleid.apple.com/auth/oauth2/token": "Apple rejected the signed client secret. Check your client ID, team ID, key ID and private key.", + "401 Client Error: Unauthorized for url: https://api.searchads.apple.com": "Apple Search Ads rejected the access token. Your API key may have been revoked — generate a new one and reconnect.", + "403 Client Error: Forbidden for url: https://api.searchads.apple.com": "Apple Search Ads denied access to this organization. Check that the API user has at least read access to the organization ID you entered.", + "Could not sign the Apple Search Ads client secret": "The private key isn't a valid unencrypted EC (P-256) PEM. Paste the key you generated for your Search Ads API key and reconnect.", + } + @property def get_source_config(self) -> SourceConfig: return SourceConfig( name=SchemaExternalDataSourceType.APPLE_SEARCH_ADS, category=DataWarehouseSourceCategory.ADVERTISING, label="Apple Search Ads", + caption="""Connect your Apple Search Ads account to pull campaigns, ad groups, keywords and daily performance into the PostHog Data warehouse. + +In the Search Ads UI, create an API user with at least **Read only** access, generate an API key, and keep the private key it gives you. Then enter the organization ID, client ID, team ID and key ID from the API key page, plus the private key itself. PostHog signs a short-lived token with the key on every sync, so no long-lived secret is stored.""", iconPath="/static/services/apple_search_ads.png", - fields=cast(list[FieldType], []), - unreleasedSource=True, + docsUrl="https://posthog.com/docs/cdp/sources/apple-search-ads", + releaseStatus=ReleaseStatus.ALPHA, + keywords=["asa", "app store ads", "search ads"], + fields=cast( + list[FieldType], + [ + SourceFieldInputConfig( + name="org_id", + label="Organization ID", + type=SourceFieldInputConfigType.TEXT, + required=True, + placeholder="123456", + secret=False, + ), + SourceFieldInputConfig( + name="client_id", + label="Client ID", + type=SourceFieldInputConfigType.TEXT, + required=True, + placeholder="SEARCHADS.27478e17-...", + secret=False, + ), + SourceFieldInputConfig( + name="apple_team_id", + label="Team ID", + type=SourceFieldInputConfigType.TEXT, + required=True, + placeholder="SEARCHADS.27478e17-...", + secret=False, + ), + SourceFieldInputConfig( + name="key_id", + label="Key ID", + type=SourceFieldInputConfigType.TEXT, + required=True, + placeholder="a1b2c3d4-...", + secret=False, + ), + SourceFieldInputConfig( + name="private_key", + label="Private key", + type=SourceFieldInputConfigType.TEXTAREA, + required=True, + placeholder="-----BEGIN EC PRIVATE KEY-----", + secret=True, + ), + SourceFieldInputConfig( + name="start_date", + label="Report start date", + type=SourceFieldInputConfigType.TEXT, + required=False, + placeholder="2024-01-01", + secret=False, + ), + ], + ), + ) + + def get_canonical_descriptions(self) -> CanonicalDescriptions: + from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.canonical_descriptions import ( + CANONICAL_DESCRIPTIONS, + ) + + return CANONICAL_DESCRIPTIONS + + def get_schemas( + self, + config: AppleSearchAdsSourceConfig, + team_id: int, + with_counts: bool = False, + names: list[str] | None = None, + force_refresh: bool = False, + api_version: str | None = None, + ) -> list[SourceSchema]: + schemas = build_endpoint_schemas( + ENDPOINTS, + INCREMENTAL_FIELDS, + names, + descriptions=ENDPOINT_DESCRIPTIONS, + # Every incremental run re-reads a trailing window of already-imported days, so + # these tables have to merge on their primary key; appending would duplicate rows. + merge_only=REPORT_ENDPOINTS, + ) + + for schema in schemas: + # Apple keeps revising the last few days of reporting data (ingestion delay plus + # attribution), so an incremental run re-reads a trailing window instead of + # trusting the frozen watermark. + if APPLE_SEARCH_ADS_ENDPOINTS[schema.name].partition_key is not None: + schema.default_incremental_lookback_seconds = REPORT_LOOKBACK_SECONDS + + return schemas + + def validate_credentials( + self, + config: AppleSearchAdsSourceConfig, + team_id: int, + schema_name: Optional[str] = None, + api_version: str | None = None, + ) -> tuple[bool, str | None]: + return validate_apple_search_ads_credentials( + self._credentials(config), + self.resolve_api_version(api_version), + schema_name, + ) + + def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[AppleSearchAdsResumeConfig]: + # Entity and report endpoints store incompatible checkpoint shapes, so keep each + # endpoint's state in its own Redis slot. + return ResumableSourceManager[AppleSearchAdsResumeConfig](inputs, AppleSearchAdsResumeConfig).with_namespace( + inputs.schema_name + ) + + def source_for_pipeline( + self, + config: AppleSearchAdsSourceConfig, + resumable_source_manager: ResumableSourceManager[AppleSearchAdsResumeConfig], + inputs: SourceInputs, + ) -> SourceResponse: + return apple_search_ads_source( + credentials=self._credentials(config), + endpoint=inputs.schema_name, + api_version=self.resolve_api_version(inputs.api_version), + request_logger=inputs.logger, + resumable_source_manager=resumable_source_manager, + should_use_incremental_field=inputs.should_use_incremental_field, + db_incremental_field_last_value=inputs.db_incremental_field_last_value, + start_date=config.start_date, + ) + + @staticmethod + def _credentials(config: AppleSearchAdsSourceConfig) -> AppleSearchAdsCredentials: + return AppleSearchAdsCredentials( + org_id=config.org_id, + client_id=config.client_id, + team_id=config.apple_team_id, + key_id=config.key_id, + private_key=config.private_key, ) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads.py new file mode 100644 index 000000000000..d4cc223b5695 --- /dev/null +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads.py @@ -0,0 +1,654 @@ +import dataclasses +from collections.abc import Iterable +from datetime import date, datetime, timedelta +from typing import Any, Optional, cast + +import pytest +from unittest import mock + +import jwt +import structlog +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import ec +from parameterized import parameterized +from requests.exceptions import HTTPError + +from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.apple_search_ads import ( + APPLE_OAUTH_AUDIENCE, + APPLE_OAUTH_TOKEN_URL, + APPLE_SEARCH_ADS_HOST, + AppleSearchAdsAuthError, + AppleSearchAdsClient, + AppleSearchAdsCredentials, + AppleSearchAdsResumeConfig, + _report_start_date, + _report_windows, + apple_search_ads_source, + build_client_secret, + flatten_report_rows, + get_rows, + validate_credentials, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.settings import ( + APPLE_SEARCH_ADS_ENDPOINTS, + DEFAULT_INITIAL_LOOKBACK_DAYS, + ENDPOINTS, + PAGE_SIZE, + REPORT_WINDOW_DAYS, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager + +SESSION_PATCH = ( + "products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads." + "apple_search_ads.make_tracked_session" +) +TODAY_PATCH = ( + "products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.apple_search_ads._today" +) + +API_VERSION = "v5" +BASE_URL = f"{APPLE_SEARCH_ADS_HOST}/api/{API_VERSION}" + +_private_key = ec.generate_private_key(ec.SECP256R1()) +PRIVATE_KEY_PEM = _private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), +).decode() +PUBLIC_KEY_PEM = ( + _private_key.public_key() + .public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + .decode() +) + +CREDENTIALS = AppleSearchAdsCredentials( + org_id="555", + client_id="SEARCHADS.client", + team_id="SEARCHADS.team", + key_id="key-1", + private_key=PRIVATE_KEY_PEM, +) + +LOGGER = cast(Any, structlog.get_logger(__name__)) + + +def _with_key(private_key: str) -> AppleSearchAdsCredentials: + return dataclasses.replace(CREDENTIALS, private_key=private_key) + + +class _FakeResponse: + def __init__(self, status_code: int = 200, json_data: Optional[dict[str, Any]] = None, url: str = BASE_URL): + self.status_code = status_code + self._json_data = json_data if json_data is not None else {} + self.url = url + self.text = str(self._json_data) + + @property + def ok(self) -> bool: + return self.status_code < 400 + + def json(self) -> dict[str, Any]: + return self._json_data + + def raise_for_status(self) -> None: + if not self.ok: + kind = "Client Error" if self.status_code < 500 else "Server Error" + raise HTTPError(f"{self.status_code} {kind}: for url: {self.url}", response=cast(Any, None)) + + +def _token_response(token: str = "access-token") -> _FakeResponse: + return _FakeResponse(200, {"access_token": token, "expires_in": 3600}, url=APPLE_OAUTH_TOKEN_URL) + + +class _FakeSession: + """Replays queued API responses and records every request the client made.""" + + def __init__(self, api_responses: list[_FakeResponse], token_responses: Optional[list[_FakeResponse]] = None): + self._api_responses = list(api_responses) + self._token_responses = list(token_responses) if token_responses is not None else None + self.calls: list[dict[str, Any]] = [] + + @property + def api_calls(self) -> list[dict[str, Any]]: + return [call for call in self.calls if call["url"] != APPLE_OAUTH_TOKEN_URL] + + @property + def token_calls(self) -> list[dict[str, Any]]: + return [call for call in self.calls if call["url"] == APPLE_OAUTH_TOKEN_URL] + + def _next(self, url: str) -> _FakeResponse: + if url == APPLE_OAUTH_TOKEN_URL: + if self._token_responses is not None: + return self._token_responses.pop(0) + return _token_response() + if not self._api_responses: + raise AssertionError(f"unexpected extra request to {url}") + return self._api_responses.pop(0) + + def get( + self, + url: str, + params: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, + timeout: Optional[float] = None, + ) -> _FakeResponse: + self.calls.append({"method": "GET", "url": url, "params": params, "headers": headers or {}}) + return self._next(url) + + def post( + self, + url: str, + json: Optional[dict[str, Any]] = None, + data: Optional[dict[str, Any]] = None, + headers: Optional[dict[str, str]] = None, + timeout: Optional[float] = None, + ) -> _FakeResponse: + self.calls.append({"method": "POST", "url": url, "json": json, "data": data, "headers": headers or {}}) + return self._next(url) + + +class _FakeResumableManager(ResumableSourceManager[AppleSearchAdsResumeConfig]): + """In-memory stand-in for the Redis-backed manager (no `super().__init__`).""" + + def __init__(self, resume_state: Optional[AppleSearchAdsResumeConfig] = None): + self._resume_state = resume_state + self.saved_states: list[AppleSearchAdsResumeConfig] = [] + self.cleared = False + + def can_resume(self) -> bool: + return self._resume_state is not None + + def load_state(self) -> AppleSearchAdsResumeConfig | None: + return self._resume_state + + def save_state(self, data: AppleSearchAdsResumeConfig) -> None: + self.saved_states.append(data) + + def clear_state(self) -> None: + self.cleared = True + + +def _entity_page(rows: list[dict[str, Any]]) -> _FakeResponse: + return _FakeResponse(200, {"data": rows, "pagination": {"totalResults": len(rows)}}) + + +def _report_payload(rows: list[dict[str, Any]]) -> dict[str, Any]: + return {"data": {"reportingDataResponse": {"row": rows}}} + + +def _report_page(rows: list[dict[str, Any]]) -> _FakeResponse: + return _FakeResponse(200, _report_payload(rows)) + + +def _report_row(metadata: dict[str, Any], dates: list[str]) -> dict[str, Any]: + return { + "metadata": metadata, + "granularity": [{"date": day, "impressions": 10, "taps": 1} for day in dates], + } + + +def _run( + endpoint: str, + session: _FakeSession, + manager: _FakeResumableManager, + **kwargs: Any, +) -> list[list[dict[str, Any]]]: + with mock.patch(SESSION_PATCH, return_value=session): + return list( + get_rows( + credentials=CREDENTIALS, + endpoint=endpoint, + api_version=API_VERSION, + request_logger=LOGGER, + resumable_source_manager=manager, + **kwargs, + ) + ) + + +class TestAppleSearchAdsTransport: + def test_client_secret_is_a_signed_es256_assertion(self) -> None: + token = build_client_secret(CREDENTIALS, issued_at=1_700_000_000) + + header = jwt.get_unverified_header(token) + assert header["alg"] == "ES256" + assert header["kid"] == "key-1" + + claims = jwt.decode( + token, + PUBLIC_KEY_PEM, + algorithms=["ES256"], + audience=APPLE_OAUTH_AUDIENCE, + options={"verify_exp": False}, + ) + assert claims["sub"] == CREDENTIALS.client_id + assert claims["iss"] == CREDENTIALS.team_id + assert claims["iat"] == 1_700_000_000 + assert claims["exp"] > claims["iat"] + + def test_client_secret_accepts_a_pem_with_escaped_newlines(self) -> None: + escaped = CREDENTIALS.private_key.replace("\n", "\\n") + token = build_client_secret(_with_key(escaped)) + + assert jwt.decode(token, PUBLIC_KEY_PEM, algorithms=["ES256"], audience=APPLE_OAUTH_AUDIENCE) + + @parameterized.expand( + [ + ("garbage", "not-a-key"), + ("empty", ""), + ("truncated_pem", "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----"), + ] + ) + def test_client_secret_rejects_an_unusable_private_key(self, _name: str, private_key: str) -> None: + with pytest.raises(AppleSearchAdsAuthError): + build_client_secret(_with_key(private_key)) + + def test_requests_carry_the_bearer_token_and_org_context(self) -> None: + session = _FakeSession([_entity_page([{"id": 1}])]) + manager = _FakeResumableManager() + + _run("campaigns", session, manager) + + assert len(session.token_calls) == 1 + token_body = session.token_calls[0]["data"] + assert token_body["grant_type"] == "client_credentials" + assert token_body["client_id"] == CREDENTIALS.client_id + assert token_body["scope"] == "searchadsorg" + + headers = session.api_calls[0]["headers"] + assert headers["Authorization"] == "Bearer access-token" + assert headers["X-AP-Context"] == "orgId=555" + + def test_acls_is_a_single_page_without_org_context(self) -> None: + session = _FakeSession([_entity_page([{"orgId": 555}])]) + manager = _FakeResumableManager() + + batches = _run("acls", session, manager) + + assert batches == [[{"orgId": 555}]] + assert len(session.api_calls) == 1 + assert session.api_calls[0]["url"] == f"{BASE_URL}/acls" + assert "X-AP-Context" not in session.api_calls[0]["headers"] + + def test_find_endpoints_page_in_the_request_body(self) -> None: + session = _FakeSession([_entity_page([{"id": 7}])]) + manager = _FakeResumableManager() + + _run("ad_groups", session, manager) + + call = session.api_calls[0] + assert call["method"] == "POST" + assert call["url"] == f"{BASE_URL}/adgroups/find" + assert call["json"]["pagination"] == {"offset": 0, "limit": PAGE_SIZE} + + def test_expired_access_token_is_reminted_once_and_the_request_replayed(self) -> None: + session = _FakeSession( + [_FakeResponse(401, url=BASE_URL), _entity_page([{"id": 1}])], + token_responses=[_token_response("first"), _token_response("second")], + ) + manager = _FakeResumableManager() + + batches = _run("campaigns", session, manager) + + assert batches == [[{"id": 1}]] + assert len(session.token_calls) == 2 + assert session.api_calls[0]["headers"]["Authorization"] == "Bearer first" + assert session.api_calls[1]["headers"]["Authorization"] == "Bearer second" + + @parameterized.expand([("unauthorized", 401), ("forbidden", 403), ("server_error", 500)]) + def test_a_persistent_error_status_raises(self, _name: str, status: int) -> None: + # Two identical failures so the single 401 re-mint retry is exhausted too. + session = _FakeSession([_FakeResponse(status, url=BASE_URL), _FakeResponse(status, url=BASE_URL)]) + manager = _FakeResumableManager() + + with pytest.raises(HTTPError): + _run("campaigns", session, manager) + + def test_entity_pagination_advances_the_offset_and_checkpoints_between_pages(self) -> None: + first_page = [{"id": index} for index in range(PAGE_SIZE)] + session = _FakeSession([_entity_page(first_page), _entity_page([{"id": PAGE_SIZE}])]) + manager = _FakeResumableManager() + + batches = _run("campaigns", session, manager) + + assert [len(batch) for batch in batches] == [PAGE_SIZE, 1] + assert [call["params"]["offset"] for call in session.api_calls] == [0, PAGE_SIZE] + assert [state.offset for state in manager.saved_states] == [PAGE_SIZE] + assert manager.cleared is True + + def test_entity_pagination_resumes_from_the_saved_offset(self) -> None: + session = _FakeSession([_entity_page([{"id": 1}])]) + manager = _FakeResumableManager(AppleSearchAdsResumeConfig(offset=2000)) + + _run("campaigns", session, manager) + + assert session.api_calls[0]["params"]["offset"] == 2000 + + def test_empty_first_page_yields_nothing_and_terminates(self) -> None: + session = _FakeSession([_entity_page([])]) + manager = _FakeResumableManager() + + assert _run("keywords", session, manager) == [] + assert len(session.api_calls) == 1 + + +class TestReportWindows: + @parameterized.expand( + [ + ("single_day", date(2026, 1, 1), date(2026, 1, 1), [(date(2026, 1, 1), date(2026, 1, 1))]), + ( + "exactly_one_window", + date(2026, 1, 1), + date(2026, 1, 7), + [(date(2026, 1, 1), date(2026, 1, 7))], + ), + ( + "spills_into_a_second_window", + date(2026, 1, 1), + date(2026, 1, 9), + [(date(2026, 1, 1), date(2026, 1, 7)), (date(2026, 1, 8), date(2026, 1, 9))], + ), + ("end_before_start", date(2026, 1, 9), date(2026, 1, 1), []), + ] + ) + def test_windows_are_ascending_and_inclusive( + self, _name: str, start: date, end: date, expected: list[tuple[date, date]] + ) -> None: + assert _report_windows(start, end) == expected + + def test_windows_never_exceed_the_configured_length(self) -> None: + windows = _report_windows(date(2026, 1, 1), date(2026, 3, 1)) + + assert all((end - start).days + 1 <= REPORT_WINDOW_DAYS for start, end in windows) + # Contiguous with no gaps or overlaps. + assert all(later[0] == earlier[1] + timedelta(days=1) for earlier, later in zip(windows, windows[1:])) + + @parameterized.expand( + [ + ("watermark_wins", True, date(2026, 5, 1), "2020-01-01", date(2026, 5, 1)), + ("iso_string_watermark", True, "2026-05-01", None, date(2026, 5, 1)), + ("datetime_watermark", True, datetime(2026, 5, 1, 6, 30), None, date(2026, 5, 1)), + ("configured_start_date", False, None, "2026-02-03", date(2026, 2, 3)), + ("unparseable_start_date_falls_back", False, None, "not-a-date", None), + ("no_watermark_no_start_date", False, None, None, None), + ("incremental_without_watermark", True, None, None, None), + ] + ) + def test_report_start_date( + self, + _name: str, + should_use_incremental_field: bool, + watermark: Any, + start_date: Optional[str], + expected: Optional[date], + ) -> None: + today = date(2026, 6, 1) + resolved = _report_start_date(should_use_incremental_field, watermark, start_date, today) + + assert resolved == (expected or today - timedelta(days=DEFAULT_INITIAL_LOOKBACK_DAYS)) + + +class TestFlattenReportRows: + def test_granularity_buckets_become_one_row_per_day(self) -> None: + payload = _report_payload([_report_row({"campaignId": 1, "campaignName": "A"}, ["2026-01-01", "2026-01-02"])]) + + rows = flatten_report_rows(payload, None) + + assert rows == [ + {"campaignId": 1, "campaignName": "A", "date": "2026-01-01", "impressions": 10, "taps": 1}, + {"campaignId": 1, "campaignName": "A", "date": "2026-01-02", "impressions": 10, "taps": 1}, + ] + + def test_fan_out_injects_the_campaign_id_the_primary_key_needs(self) -> None: + payload = _report_payload([_report_row({"adGroupId": 9}, ["2026-01-01"])]) + + rows = flatten_report_rows(payload, 42) + + assert rows[0]["campaignId"] == 42 + assert rows[0]["adGroupId"] == 9 + + def test_fan_out_does_not_clobber_a_campaign_id_apple_supplied(self) -> None: + payload = _report_payload([_report_row({"campaignId": 7, "adGroupId": 9}, ["2026-01-01"])]) + + assert flatten_report_rows(payload, 42)[0]["campaignId"] == 7 + + @parameterized.expand( + [ + ("no_granularity", {"data": {"reportingDataResponse": {"row": [{"metadata": {"campaignId": 1}}]}}}), + ("empty_row_list", {"data": {"reportingDataResponse": {"row": []}}}), + ("null_reporting_data", {"data": None}), + ("missing_data_key", {}), + ] + ) + def test_pages_without_daily_metrics_flatten_to_nothing(self, _name: str, payload: dict[str, Any]) -> None: + assert flatten_report_rows(payload, None) == [] + + +class TestReportSync: + def test_campaign_report_requests_one_windowed_page_per_window(self) -> None: + session = _FakeSession( + [ + _report_page([_report_row({"campaignId": 1}, ["2026-01-01"])]), + _report_page([_report_row({"campaignId": 1}, ["2026-01-08"])]), + ] + ) + manager = _FakeResumableManager() + + with mock.patch(TODAY_PATCH, return_value=date(2026, 1, 9)): + batches = _run( + "campaign_report", + session, + manager, + should_use_incremental_field=True, + db_incremental_field_last_value=date(2026, 1, 1), + ) + + assert [row["date"] for batch in batches for row in batch] == ["2026-01-01", "2026-01-08"] + bodies = [call["json"] for call in session.api_calls] + assert [(body["startTime"], body["endTime"]) for body in bodies] == [ + ("2026-01-01", "2026-01-07"), + ("2026-01-08", "2026-01-09"), + ] + assert bodies[0]["granularity"] == "DAILY" + assert bodies[0]["selector"]["pagination"] == {"offset": 0, "limit": PAGE_SIZE} + # After the first window completes, the checkpoint points at the next window. + assert manager.saved_states[0] == AppleSearchAdsResumeConfig( + offset=0, window_start="2026-01-08", campaign_id=None + ) + + def test_fan_out_reports_walk_every_campaign_in_every_window(self) -> None: + session = _FakeSession( + [ + _entity_page([{"id": 20}, {"id": 10}]), + _report_page([_report_row({"adGroupId": 1}, ["2026-01-01"])]), + _report_page([_report_row({"adGroupId": 2}, ["2026-01-01"])]), + ] + ) + manager = _FakeResumableManager() + + with mock.patch(TODAY_PATCH, return_value=date(2026, 1, 3)): + batches = _run( + "ad_group_report", + session, + manager, + should_use_incremental_field=True, + db_incremental_field_last_value=date(2026, 1, 1), + ) + + report_calls = [call for call in session.api_calls if "/reports/" in call["url"]] + # Campaign ids are visited in a stable ascending order, not response order. + assert [call["url"] for call in report_calls] == [ + f"{BASE_URL}/reports/campaigns/10/adgroups", + f"{BASE_URL}/reports/campaigns/20/adgroups", + ] + assert [row["campaignId"] for batch in batches for row in batch] == [10, 20] + assert manager.saved_states[0] == AppleSearchAdsResumeConfig( + offset=0, window_start="2026-01-01", campaign_id=20 + ) + + def test_fan_out_reports_resume_at_the_checkpointed_campaign(self) -> None: + session = _FakeSession( + [ + _entity_page([{"id": 10}, {"id": 20}]), + _report_page([_report_row({"adGroupId": 2}, ["2026-01-01"])]), + ] + ) + manager = _FakeResumableManager(AppleSearchAdsResumeConfig(offset=0, window_start="2026-01-01", campaign_id=20)) + + with mock.patch(TODAY_PATCH, return_value=date(2026, 1, 3)): + _run( + "ad_group_report", + session, + manager, + should_use_incremental_field=True, + db_incremental_field_last_value=date(2026, 1, 1), + ) + + report_calls = [call for call in session.api_calls if "/reports/" in call["url"]] + assert [call["url"] for call in report_calls] == [f"{BASE_URL}/reports/campaigns/20/adgroups"] + + def test_a_checkpoint_outside_this_runs_windows_restarts_the_range(self) -> None: + session = _FakeSession([_report_page([_report_row({"campaignId": 1}, ["2026-01-01"])])]) + manager = _FakeResumableManager( + AppleSearchAdsResumeConfig(offset=500, window_start="2019-01-01", campaign_id=None) + ) + + with mock.patch(TODAY_PATCH, return_value=date(2026, 1, 3)): + _run( + "campaign_report", + session, + manager, + should_use_incremental_field=True, + db_incremental_field_last_value=date(2026, 1, 1), + ) + + body = session.api_calls[0]["json"] + assert body["startTime"] == "2026-01-01" + assert body["selector"]["pagination"]["offset"] == 0 + + def test_report_pagination_continues_while_a_page_is_full(self) -> None: + full_page = _report_page([_report_row({"campaignId": index}, ["2026-01-01"]) for index in range(PAGE_SIZE)]) + session = _FakeSession([full_page, _report_page([_report_row({"campaignId": 9999}, ["2026-01-01"])])]) + manager = _FakeResumableManager() + + with mock.patch(TODAY_PATCH, return_value=date(2026, 1, 2)): + _run( + "campaign_report", + session, + manager, + should_use_incremental_field=True, + db_incremental_field_last_value=date(2026, 1, 1), + ) + + offsets = [call["json"]["selector"]["pagination"]["offset"] for call in session.api_calls] + assert offsets == [0, PAGE_SIZE] + assert manager.saved_states[0] == AppleSearchAdsResumeConfig( + offset=PAGE_SIZE, window_start="2026-01-01", campaign_id=None + ) + + +class TestSourceResponse: + @parameterized.expand([(endpoint,) for endpoint in ENDPOINTS]) + def test_response_matches_the_endpoint_catalog(self, endpoint: str) -> None: + config = APPLE_SEARCH_ADS_ENDPOINTS[endpoint] + + response = apple_search_ads_source( + credentials=CREDENTIALS, + endpoint=endpoint, + api_version=API_VERSION, + request_logger=LOGGER, + resumable_source_manager=_FakeResumableManager(), + ) + + assert response.name == endpoint + assert response.primary_keys == config.primary_keys + # Windows are walked oldest-first, so the watermark only ever moves forward. + assert response.sort_mode == "asc" + if config.partition_key is None: + assert response.partition_mode is None + assert response.partition_keys is None + else: + assert response.partition_mode == "datetime" + assert response.partition_keys == [config.partition_key] + + def test_items_are_lazy(self) -> None: + response = apple_search_ads_source( + credentials=CREDENTIALS, + endpoint="campaigns", + api_version=API_VERSION, + request_logger=LOGGER, + resumable_source_manager=_FakeResumableManager(), + ) + + # No HTTP happens until the pipeline iterates, so nothing needed mocking above. + assert callable(response.items) + assert isinstance(cast("Iterable[Any]", response.items()), Iterable) + + +class TestValidateCredentials: + @parameterized.expand( + [ + ("ok", 200, None, True), + ("unauthorized", 401, None, False), + ("forbidden_at_source_create", 403, None, True), + ("forbidden_for_a_schema", 403, "campaigns", False), + ("unexpected_status", 500, None, False), + ] + ) + def test_probe_status_is_mapped(self, _name: str, status: int, schema_name: Optional[str], expected: bool) -> None: + session = _FakeSession([_FakeResponse(status, url=BASE_URL), _FakeResponse(status, url=BASE_URL)]) + + with mock.patch(SESSION_PATCH, return_value=session): + is_valid, message = validate_credentials(CREDENTIALS, API_VERSION, schema_name) + + assert is_valid is expected + assert (message is None) is expected + + def test_probe_targets_an_org_scoped_endpoint(self) -> None: + session = _FakeSession([_FakeResponse(200, {"data": []}, url=BASE_URL)]) + + with mock.patch(SESSION_PATCH, return_value=session): + assert validate_credentials(CREDENTIALS, API_VERSION) == (True, None) + + assert session.api_calls[0]["url"] == f"{BASE_URL}/campaigns" + assert session.api_calls[0]["headers"]["X-AP-Context"] == "orgId=555" + + def test_an_unusable_private_key_fails_before_any_request(self) -> None: + session = _FakeSession([]) + + with mock.patch(SESSION_PATCH, return_value=session): + is_valid, message = validate_credentials(_with_key("nope"), API_VERSION) + + assert is_valid is False + assert message is not None and "private key" in message + assert session.calls == [] + + def test_a_token_endpoint_rejection_is_reported(self) -> None: + session = _FakeSession([], token_responses=[_FakeResponse(400, url=APPLE_OAUTH_TOKEN_URL)]) + + with mock.patch(SESSION_PATCH, return_value=session): + is_valid, message = validate_credentials(CREDENTIALS, API_VERSION) + + assert is_valid is False + assert message is not None + + def test_a_token_response_without_an_access_token_is_reported(self) -> None: + session = _FakeSession([], token_responses=[_FakeResponse(200, {}, url=APPLE_OAUTH_TOKEN_URL)]) + + with mock.patch(SESSION_PATCH, return_value=session): + is_valid, message = validate_credentials(CREDENTIALS, API_VERSION) + + assert is_valid is False + assert message == "Apple's token response did not contain an access token" + + +class TestClientBaseUrl: + @parameterized.expand([("v5", "v5"), ("pinned_older", "v4")]) + def test_base_url_follows_the_resolved_api_version(self, _name: str, api_version: str) -> None: + with mock.patch(SESSION_PATCH, return_value=_FakeSession([])): + client = AppleSearchAdsClient(CREDENTIALS, api_version) + + assert client.base_url == f"{APPLE_SEARCH_ADS_HOST}/api/{api_version}" diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads_source.py new file mode 100644 index 000000000000..26c3bd6788ef --- /dev/null +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads_source.py @@ -0,0 +1,201 @@ +from typing import Any, cast + +from unittest import mock + +from parameterized import parameterized + +from posthog.schema import ( + DataWarehouseSourceCategory, + ReleaseStatus, + SourceFieldInputConfig, + SourceFieldInputConfigType, +) + +from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.apple_search_ads import ( + AppleSearchAdsCredentials, + AppleSearchAdsResumeConfig, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.canonical_descriptions import ( + CANONICAL_DESCRIPTIONS, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.settings import ( + APPLE_SEARCH_ADS_ENDPOINTS, + ENDPOINTS, + REPORT_LOOKBACK_SECONDS, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.source import ( + AppleSearchAdsSource, +) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager +from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.applesearchads import ( + AppleSearchAdsSourceConfig, +) +from products.warehouse_sources.backend.types import ExternalDataSourceType + +SOURCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.source" + +REPORT_ENDPOINTS = tuple(name for name, config in APPLE_SEARCH_ADS_ENDPOINTS.items() if config.partition_key) +ENTITY_ENDPOINTS = tuple(name for name, config in APPLE_SEARCH_ADS_ENDPOINTS.items() if not config.partition_key) + + +class TestAppleSearchAdsSource: + def setup_method(self) -> None: + self.source = AppleSearchAdsSource() + self.team_id = 123 + self.config = AppleSearchAdsSourceConfig( + org_id="555", + client_id="SEARCHADS.client", + apple_team_id="SEARCHADS.team", + key_id="key-1", + private_key="-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----", + start_date="2026-01-01", + ) + + def test_source_type(self) -> None: + assert self.source.source_type == ExternalDataSourceType.APPLESEARCHADS + + def test_get_source_config(self) -> None: + config = self.source.get_source_config + + assert config.name.value == "AppleSearchAds" + assert config.label == "Apple Search Ads" + assert config.category == DataWarehouseSourceCategory.ADVERTISING + assert config.releaseStatus == ReleaseStatus.ALPHA + assert not config.unreleasedSource + assert config.iconPath == "/static/services/apple_search_ads.png" + assert config.docsUrl == "https://posthog.com/docs/cdp/sources/apple-search-ads" + + @parameterized.expand( + [ + ("org_id", SourceFieldInputConfigType.TEXT, True, False), + ("client_id", SourceFieldInputConfigType.TEXT, True, False), + ("apple_team_id", SourceFieldInputConfigType.TEXT, True, False), + ("key_id", SourceFieldInputConfigType.TEXT, True, False), + ("private_key", SourceFieldInputConfigType.TEXTAREA, True, True), + ("start_date", SourceFieldInputConfigType.TEXT, False, False), + ] + ) + def test_source_fields( + self, name: str, field_type: SourceFieldInputConfigType, required: bool, secret: bool + ) -> None: + fields = { + field.name: field + for field in self.source.get_source_config.fields + if isinstance(field, SourceFieldInputConfig) + } + + assert set(fields) == {"org_id", "client_id", "apple_team_id", "key_id", "private_key", "start_date"} + field = fields[name] + assert field.type == field_type + assert field.required is required + assert field.secret is secret + + def test_api_version_metadata(self) -> None: + assert self.source.supported_versions == ("v5",) + assert self.source.default_version == "v5" + assert self.source.api_docs_url.startswith("https://") + + def test_lists_tables_without_credentials(self) -> None: + # `get_schemas` walks a static catalog, so the public docs can render the table list. + assert self.source.lists_tables_without_credentials is True + + def test_get_schemas_covers_the_endpoint_catalog(self) -> None: + schemas = self.source.get_schemas(self.config, self.team_id) + + assert {schema.name for schema in schemas} == set(ENDPOINTS) + assert all(schema.description for schema in schemas) + + def test_get_schemas_filters_by_name(self) -> None: + schemas = self.source.get_schemas(self.config, self.team_id, names=["campaigns", "campaign_report"]) + + assert {schema.name for schema in schemas} == {"campaigns", "campaign_report"} + + @parameterized.expand([(endpoint,) for endpoint in REPORT_ENDPOINTS]) + def test_report_tables_are_incremental_on_date_with_a_lookback(self, endpoint: str) -> None: + schema = next(s for s in self.source.get_schemas(self.config, self.team_id) if s.name == endpoint) + + assert schema.supports_incremental is True + assert [f["field"] for f in schema.incremental_fields] == ["date"] + assert schema.default_incremental_lookback_seconds == REPORT_LOOKBACK_SECONDS + # The lookback re-reads already-imported days, so appending would duplicate them. + assert schema.supports_append is False + + @parameterized.expand([(endpoint,) for endpoint in ENTITY_ENDPOINTS]) + def test_entity_tables_are_full_refresh_only(self, endpoint: str) -> None: + schema = next(s for s in self.source.get_schemas(self.config, self.team_id) if s.name == endpoint) + + # Apple's entity endpoints have no updated-since filter, so there is nothing to track. + assert schema.supports_incremental is False + assert schema.incremental_fields == [] + assert schema.default_incremental_lookback_seconds is None + + def test_canonical_descriptions_cover_every_endpoint(self) -> None: + descriptions = self.source.get_canonical_descriptions() + + assert descriptions is CANONICAL_DESCRIPTIONS + assert set(descriptions) == set(ENDPOINTS) + for endpoint, entry in descriptions.items(): + primary_keys = APPLE_SEARCH_ADS_ENDPOINTS[endpoint].primary_keys + assert set(primary_keys) <= set(entry.get("columns", {})), endpoint + + @parameterized.expand([("unauthorized", 401), ("forbidden", 403)]) + def test_non_retryable_errors_cover_auth_failures(self, _name: str, status: int) -> None: + errors = self.source.get_non_retryable_errors() + + assert any(str(status) in key and "searchads.apple.com" in key for key in errors) + assert all(message for message in errors.values()) + + def test_validate_credentials_maps_the_config_onto_apple_credentials(self) -> None: + with mock.patch(f"{SOURCE_MODULE}.validate_apple_search_ads_credentials") as mock_validate: + mock_validate.return_value = (True, None) + + assert self.source.validate_credentials(self.config, self.team_id) == (True, None) + + credentials, api_version, schema_name = mock_validate.call_args.args + assert credentials == AppleSearchAdsCredentials( + org_id="555", + client_id="SEARCHADS.client", + team_id="SEARCHADS.team", + key_id="key-1", + private_key=self.config.private_key, + ) + assert api_version == "v5" + assert schema_name is None + + def test_validate_credentials_honors_a_pinned_api_version(self) -> None: + with mock.patch(f"{SOURCE_MODULE}.validate_apple_search_ads_credentials") as mock_validate: + mock_validate.return_value = (True, None) + self.source.validate_credentials(self.config, self.team_id, api_version="v4") + + assert mock_validate.call_args.args[1] == "v4" + + def test_get_resumable_source_manager_is_namespaced_per_schema(self) -> None: + inputs = mock.MagicMock() + inputs.schema_name = "campaign_report" + + manager = self.source.get_resumable_source_manager(inputs) + + assert isinstance(manager, ResumableSourceManager) + assert manager._data_class is AppleSearchAdsResumeConfig + # Entity and report checkpoints have incompatible shapes, so they must not share a slot. + assert manager._namespace == "campaign_report" + + def test_source_for_pipeline_plumbs_arguments(self) -> None: + inputs = mock.MagicMock() + inputs.schema_name = "campaign_report" + inputs.should_use_incremental_field = True + inputs.db_incremental_field_last_value = "2026-05-01" + inputs.api_version = None + manager = mock.MagicMock() + + with mock.patch(f"{SOURCE_MODULE}.apple_search_ads_source") as mock_source: + self.source.source_for_pipeline(self.config, manager, inputs) + + kwargs = cast("dict[str, Any]", mock_source.call_args.kwargs) + assert kwargs["endpoint"] == "campaign_report" + assert kwargs["api_version"] == "v5" + assert kwargs["resumable_source_manager"] is manager + assert kwargs["should_use_incremental_field"] is True + assert kwargs["db_incremental_field_last_value"] == "2026-05-01" + assert kwargs["start_date"] == "2026-01-01" + assert kwargs["credentials"].org_id == "555" diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/applesearchads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/applesearchads.py index ba3110fef47d..a9daa4874c19 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/applesearchads.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/applesearchads.py @@ -6,4 +6,9 @@ @config.config class AppleSearchAdsSourceConfig(config.Config): - pass + org_id: str + client_id: str + apple_team_id: str + key_id: str + private_key: str + start_date: str | None = None From df05418d34a26a3a8e116bbda6624009fd656211 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Mon, 27 Jul 2026 13:43:05 +0100 Subject: [PATCH 006/289] fix(data-warehouse): bound apple search ads report start date Floor an implausibly old configured start date and iterate report window/campaign combinations lazily, so a start like 0001-01-01 on an account with many campaigns can't build millions of tuples or fan out over thousands of empty windows and exhaust an import worker. Generated-By: PostHog Code Task-Id: 7932bdfb-3788-467b-8035-53d3b3a5a162 --- .../apple_search_ads/apple_search_ads.py | 67 +++++++++++++------ .../sources/apple_search_ads/settings.py | 5 ++ .../tests/test_apple_search_ads.py | 9 ++- 3 files changed, 59 insertions(+), 22 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py index 6d607536ed50..fe36dad826ea 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py @@ -1,6 +1,7 @@ import time +import itertools import dataclasses -from collections.abc import Iterator +from collections.abc import Callable, Iterator from datetime import UTC, date, datetime, timedelta from typing import Any, Optional @@ -14,6 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.settings import ( APPLE_SEARCH_ADS_ENDPOINTS, DEFAULT_INITIAL_LOOKBACK_DAYS, + MAX_INITIAL_LOOKBACK_DAYS, PAGE_SIZE, REPORT_WINDOW_DAYS, AppleSearchAdsEndpointConfig, @@ -303,7 +305,9 @@ def _report_start_date( configured = _to_date(start_date) if start_date else None if configured is not None: - return configured + # Floor the configured date so an implausibly old start can't fan the report out over + # thousands of empty windows and exhaust the import worker. + return max(configured, today - timedelta(days=MAX_INITIAL_LOOKBACK_DAYS)) return today - timedelta(days=DEFAULT_INITIAL_LOOKBACK_DAYS) @@ -429,24 +433,44 @@ def _list_campaign_ids(client: AppleSearchAdsClient) -> list[int]: return sorted(ids) -def _resume_index( - tasks: list[tuple[date, date, Optional[int]]], +ReportTask = tuple[date, date, Optional[int]] + + +def _report_tasks(start: date, end: date, campaign_ids: list[Optional[int]]) -> Iterator[ReportTask]: + """Every (window, campaign) report request this run must make, lazily. + + Yielded rather than listed so a large window range never materialises as millions of + tuples up front. + """ + for window_start, window_end in _report_windows(start, end): + for campaign_id in campaign_ids: + yield window_start, window_end, campaign_id + + +def _advance_to_resume( + make_tasks: Callable[[], Iterator[ReportTask]], resume: Optional[AppleSearchAdsResumeConfig], request_logger: FilteringBoundLogger, -) -> tuple[int, int]: - """Locate a saved checkpoint in this run's task list, by value rather than position.""" +) -> tuple[Iterator[ReportTask], int]: + """Fast-forward the lazy task stream to a saved checkpoint, matched by value not position. + + A checkpoint from a different window range (e.g. the start date changed) is never found, so + the run restarts from the first task with a fresh stream. + """ if resume is None or not resume.window_start: - return 0, 0 + return make_tasks(), 0 key = (resume.window_start, resume.campaign_id) - for index, (window_start, _window_end, campaign_id) in enumerate(tasks): + tasks = make_tasks() + for task in tasks: + window_start, _window_end, campaign_id = task if (window_start.isoformat(), campaign_id) == key: - return index, resume.offset + return itertools.chain([task], tasks), resume.offset request_logger.debug( f"Apple Search Ads: saved checkpoint {key} is not in this run's window range, starting from the beginning" ) - return 0, 0 + return make_tasks(), 0 def _iter_report_rows( @@ -466,17 +490,17 @@ def _iter_report_rows( if config.fan_out_over_campaigns: campaign_ids = list(_list_campaign_ids(client)) - tasks = [ - (window_start, window_end, campaign_id) - for window_start, window_end in _report_windows(start, today) - for campaign_id in campaign_ids - ] - start_index, start_offset = _resume_index(tasks, resume, request_logger) + tasks, start_offset = _advance_to_resume(lambda: _report_tasks(start, today, campaign_ids), resume, request_logger) - for index in range(start_index, len(tasks)): - window_start, window_end, campaign_id = tasks[index] + current = next(tasks, None) + resume_offset = start_offset + while current is not None: + window_start, window_end, campaign_id = current + # Peek at the next task so a completed window can checkpoint where the run should pick up. + upcoming = next(tasks, None) path = config.path.format(campaign_id=campaign_id) if campaign_id is not None else config.path - offset = start_offset if index == start_index else 0 + offset = resume_offset + resume_offset = 0 while True: payload = client.request_json( @@ -500,13 +524,14 @@ def _iter_report_rows( ) ) - if index + 1 < len(tasks): - next_window_start, _next_window_end, next_campaign_id = tasks[index + 1] + if upcoming is not None: + next_window_start, _next_window_end, next_campaign_id = upcoming resumable_source_manager.save_state( AppleSearchAdsResumeConfig( offset=0, window_start=next_window_start.isoformat(), campaign_id=next_campaign_id ) ) + current = upcoming def get_rows( diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/settings.py index 4f1d2f1936fd..3564967d64d2 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/settings.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/settings.py @@ -29,6 +29,11 @@ # How far back the first sync of a report table reaches when the user gives no start date. DEFAULT_INITIAL_LOOKBACK_DAYS = 365 +# The earliest a configured start date may reach. Apple Search Ads has held no reporting data +# from before it launched, so a start date older than this is a typo — clamp it rather than fan +# a report out over thousands of empty windows (one report request per window per campaign). +MAX_INITIAL_LOOKBACK_DAYS = 11 * 365 + @dataclass class AppleSearchAdsEndpointConfig: diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads.py index d4cc223b5695..143724e3956e 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/tests/test_apple_search_ads.py @@ -33,6 +33,7 @@ APPLE_SEARCH_ADS_ENDPOINTS, DEFAULT_INITIAL_LOOKBACK_DAYS, ENDPOINTS, + MAX_INITIAL_LOOKBACK_DAYS, PAGE_SIZE, REPORT_WINDOW_DAYS, ) @@ -375,6 +376,9 @@ def test_windows_never_exceed_the_configured_length(self) -> None: ("unparseable_start_date_falls_back", False, None, "not-a-date", None), ("no_watermark_no_start_date", False, None, None, None), ("incremental_without_watermark", True, None, None, None), + # An implausibly old configured start is floored so it can't fan out over thousands + # of empty windows. + ("ancient_start_date_is_floored", False, None, "0001-01-01", None), ] ) def test_report_start_date( @@ -388,7 +392,10 @@ def test_report_start_date( today = date(2026, 6, 1) resolved = _report_start_date(should_use_incremental_field, watermark, start_date, today) - assert resolved == (expected or today - timedelta(days=DEFAULT_INITIAL_LOOKBACK_DAYS)) + if _name == "ancient_start_date_is_floored": + assert resolved == today - timedelta(days=MAX_INITIAL_LOOKBACK_DAYS) + else: + assert resolved == (expected or today - timedelta(days=DEFAULT_INITIAL_LOOKBACK_DAYS)) class TestFlattenReportRows: From 63d6a6213955dcfb8a3e41d22605482700bd6122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Tue, 4 Aug 2026 10:58:49 -0400 Subject: [PATCH 007/289] fix(slack): resolve repo from CI and PR links --- posthog/git.py | 25 +++++++++++--- .../ai/slack_app/activities/classifiers.py | 34 +++++++++++++++++++ .../tests/ai/test_classify_task_needs_repo.py | 13 +++++++ posthog/test/test_git.py | 19 +++++++++++ .../backend/tests/test_guess_repository.py | 2 +- 5 files changed, 87 insertions(+), 6 deletions(-) diff --git a/posthog/git.py b/posthog/git.py index 5dcba76caebb..3c423c96e937 100644 --- a/posthog/git.py +++ b/posthog/git.py @@ -49,13 +49,21 @@ def get_git_branch() -> Optional[str]: return None +# Lookbehind so `mygithub.com/a/b` doesn't match; a scheme's `//` still does. +_GITHUB_REPO_URL_PATTERN = re.compile(r"(? str | None: - """Return the first explicit `owner/repo` token in `text` that matches a connected repo. + """Return the first repo named in `text` that matches a connected repo. + + Two forms, in priority order: a bare `owner/repo` token, then any + `github.com/owner/repo…` URL. Both match case-insensitively against `all_repos`. Bare + tokens strip surrounding punctuation and handle Slack's `` link form. + `text` is assumed already cleaned of any platform-specific noise (e.g. bot mentions) + by the caller. - Tokenizes on whitespace and matches bare `owner/repo` tokens (no `@` prefix needed) - case-insensitively against `all_repos`. Strips surrounding punctuation and handles - Slack's `` link form. `text` is assumed already cleaned of any - platform-specific noise (e.g. bot mentions) by the caller. + Bare tokens win because typing one out is a stronger signal of intent than pasting a + link that happens to be in the message. Pure helper (no Django / heavy deps) so any product can import it downward from core. """ @@ -80,4 +88,11 @@ def extract_explicit_repo(text: str, all_repos: list[str]) -> str | None: if match: return match + # Scanned over the raw string, not per token: Slack wraps URLs in `<…|…>`, so the + # tokenizer above never sees one in isolation. + for owner, repo in _GITHUB_REPO_URL_PATTERN.findall(text): + match = normalized_repos.get(f"{owner}/{repo.removesuffix('.git')}".lower()) + if match: + return match + return None diff --git a/posthog/temporal/ai/slack_app/activities/classifiers.py b/posthog/temporal/ai/slack_app/activities/classifiers.py index 8949e11d47be..5eacee0faf87 100644 --- a/posthog/temporal/ai/slack_app/activities/classifiers.py +++ b/posthog/temporal/ai/slack_app/activities/classifiers.py @@ -15,6 +15,37 @@ CLASSIFIER_THREAD_HISTORY_MESSAGES = 10 +# Nothing else in a repo is called "flaky", and a workflow-run URL is only ever CI. +_CI_UNAMBIGUOUS_PATTERNS = ( + r"\bde-?flak", + r"\bflak(?:y|e|es|iness)\b", + r"\bmerge queue\b", + r"github\.com/[\w.-]+/[\w.-]+/actions\b", +) +# Both halves required, so "check the test dashboard" stays an analytics ask. +_CI_SUBJECT_PATTERNS = (r"\btests?\b", r"\bspecs?\b", r"\bsuites?\b", r"\bshards?\b", r"\bci\b", r"\bmaster\b") +_CI_FAILURE_PATTERNS = ( + r"\bfail(?:s|ed|ing|ure|ures)?\b", + r"\bred\b", + r"\bbroke(?:n)?\b", + r"\btimed?\s?out\b", + r"\berror(?:s|ing)?\b", +) + + +def _is_ci_failure_ask(normalized: str) -> bool: + """Whether the conversation is about a broken or flaky CI run. + + Checked first because CI work is code work wearing none of the usual tells — a flaky + test report rarely names a file — and because it often mentions a product noun ("the + experiment insight test is flaky") that would short-circuit the classifier to no-repo. + """ + if any(re.search(pattern, normalized) for pattern in _CI_UNAMBIGUOUS_PATTERNS): + return True + return any(re.search(pattern, normalized) for pattern in _CI_SUBJECT_PATTERNS) and any( + re.search(pattern, normalized) for pattern in _CI_FAILURE_PATTERNS + ) + def classify_task_needs_repo( event_text: str, @@ -33,6 +64,9 @@ def classify_task_needs_repo( conversation = "\n".join(f"{msg['user']}: {msg['text']}" for msg in thread_messages) normalized = f"{conversation}\nLatest message: {event_text}".lower() + if _is_ci_failure_ask(normalized): + return True + # Substring match: keep the shortest form that uniquely identifies the # concept without colliding with code-review vocabulary. Plurals are used # only when the singular substring-matches a common non-analytics word diff --git a/posthog/temporal/tests/ai/test_classify_task_needs_repo.py b/posthog/temporal/tests/ai/test_classify_task_needs_repo.py index bb8ed9b32c77..c8a54f3dbd91 100644 --- a/posthog/temporal/tests/ai/test_classify_task_needs_repo.py +++ b/posthog/temporal/tests/ai/test_classify_task_needs_repo.py @@ -37,6 +37,19 @@ class TestClassifyTaskNeedsRepo: ("analytics_hogql", "write a hogql query to count signups by country", False), ("flag_search", "find the feature flag for the new onboarding", False), ("replay_question", "show me session replays of failed checkouts", False), + # Real #flakey-tests asks. None names a file or says "PR"/"commit", and + # several carry a product noun that used to short-circuit them to no-repo. + ( + "ci_is_this_flaky", + "is this flaky? https://github.com/posthog/posthog/actions/runs/30560492835/job/90936416640", + True, + ), + ("ci_master_broken", "django tests failing on master, investigate", True), + ("ci_rust_in_pr", "why is rust CI failing in this PR?", True), + ("ci_product_noun_no_longer_short_circuits", "the experiment insight test is flaky", True), + ("ci_merge_queue", "seeing a lot of pending failures in the trunk merge queue", True), + # The failure word alone is not CI — this is still a product ask. + ("product_failure_without_ci_subject", "the dashboard fails to load for this user", False), ] ) def test_heuristic_classification(self, _name, text, expected): diff --git a/posthog/test/test_git.py b/posthog/test/test_git.py index 987b0314c2da..526fc17c85e4 100644 --- a/posthog/test/test_git.py +++ b/posthog/test/test_git.py @@ -20,6 +20,25 @@ class TestExtractExplicitRepo: ("no_repo_token", "the dashboards are slow", None), ("unconnected_repo", "fix acme/widgets please", None), ("bare_url_ignored", "https://posthog.com/posthog is down", None), + ( + "actions_run_url", + "is this flaky? https://github.com/posthog/posthog/actions/runs/30560492835/job/90936416640", + "posthog/posthog", + ), + ( + "slack_wrapped_actions_url_with_label", + "why did this fail? ", + "posthog/posthog-js", + ), + ("clone_url_suffix", "cloned from git@github.com/posthog/posthog.git", "posthog/posthog"), + ("unconnected_repo_url", "see https://github.com/acme/widgets/pull/1", None), + ("lookalike_host", "see https://mygithub.com/posthog/posthog/pull/1", None), + ( + "bare_token_beats_later_url", + "fix posthog/posthog-js — context: https://github.com/posthog/posthog/pull/1", + "posthog/posthog-js", + ), ] ) def test_extracts_matching_repo(self, _name: str, text: str, expected: str | None): diff --git a/products/slack_app/backend/tests/test_guess_repository.py b/products/slack_app/backend/tests/test_guess_repository.py index a13e216fdbc3..7eaf1a00eb16 100644 --- a/products/slack_app/backend/tests/test_guess_repository.py +++ b/products/slack_app/backend/tests/test_guess_repository.py @@ -429,7 +429,7 @@ class TestExtractExplicitRepo: ("simple", "fix posthog/posthog-js please", "posthog/posthog-js"), ("no_match", "hello world", None), ("case_insensitive", "check PostHog/PostHog", "posthog/posthog"), - ("url_false_positive", "see https://github.com/posthog/posthog/issues/1", None), + ("github_url", "see https://github.com/posthog/posthog/issues/1", "posthog/posthog"), ("backticks", "please fix `posthog/posthog-js`", "posthog/posthog-js"), ( "slack_link_label", From 568add6708974923c1a67cd3b2c2c443409332ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Tue, 4 Aug 2026 11:11:48 -0400 Subject: [PATCH 008/289] refactor(slack): parse repo urls, bail on ambiguity --- posthog/git.py | 74 ++++++++++++++++++++++++---------------- posthog/test/test_git.py | 13 ++++++- 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/posthog/git.py b/posthog/git.py index 3c423c96e937..1659295efabb 100644 --- a/posthog/git.py +++ b/posthog/git.py @@ -2,6 +2,7 @@ import subprocess from functools import cache from typing import Optional +from urllib.parse import urlsplit _git_commit_baked_in: Optional[str] = None try: @@ -49,21 +50,41 @@ def get_git_branch() -> Optional[str]: return None -# Lookbehind so `mygithub.com/a/b` doesn't match; a scheme's `//` still does. -_GITHUB_REPO_URL_PATTERN = re.compile(r"(?,.;:!?" +_GITHUB_HOSTS = frozenset({"github.com", "www.github.com"}) + + +def _repo_from_github_url(token: str) -> str | None: + """`owner/repo` from a GitHub URL token, or None if it isn't one.""" + candidate = token.replace("git@github.com:", "https://github.com/", 1) + if "//" not in candidate: + candidate = f"https://{candidate}" # urlsplit only populates netloc when a scheme is present + try: + parts = urlsplit(candidate) + except ValueError: + return None + # Exact host match, so `mygithub.com` and `github.com.evil.tld` can never resolve. + if parts.hostname not in _GITHUB_HOSTS: + return None + segments = [segment for segment in parts.path.split("/") if segment] + if len(segments) < 2: + return None + return f"{segments[0]}/{segments[1].removesuffix('.git')}" def extract_explicit_repo(text: str, all_repos: list[str]) -> str | None: - """Return the first repo named in `text` that matches a connected repo. + """Return the repo named in `text` that matches a connected repo, if exactly one is. - Two forms, in priority order: a bare `owner/repo` token, then any - `github.com/owner/repo…` URL. Both match case-insensitively against `all_repos`. Bare - tokens strip surrounding punctuation and handle Slack's `` link form. - `text` is assumed already cleaned of any platform-specific noise (e.g. bot mentions) - by the caller. + Two tiers of evidence, strongest first: a bare `owner/repo` token, then a + `github.com/owner/repo…` URL of any depth (a run, a pull request, a file permalink). + Typing a repo out is more deliberate than pasting a link that happens to be in the + message, so a bare token wins outright. `text` is assumed already cleaned of any + platform-specific noise (e.g. bot mentions) by the caller. - Bare tokens win because typing one out is a stronger signal of intent than pasting a - link that happens to be in the message. + Links only resolve when the message points at a single connected repo. Two different + linked repos is genuine ambiguity, and returning None lets the caller disambiguate + (the Slack cascade falls through to its discovery agent, then a repo picker) rather + than silently starting work in whichever was pasted first. Pure helper (no Django / heavy deps) so any product can import it downward from core. """ @@ -71,28 +92,21 @@ def extract_explicit_repo(text: str, all_repos: list[str]) -> str | None: return None normalized_repos = {repo.lower(): repo for repo in all_repos} + linked: set[str] = set() for token in text.split(): - candidate = token.strip("`'\"()[]{}<>,.;:!?") - - # Slack can format links as ; for repo tokens we want the label. - if "|" in candidate: - candidate = candidate.split("|", 1)[1].strip("`'\"()[]{}<>,.;:!?") - - if not candidate or "://" in candidate or candidate.startswith("http"): - continue - if not re.fullmatch(r"[\w.-]+/[\w.-]+", candidate): - continue + # Slack formats links as ; either side can carry the repo, so try both. + for candidate in (part.strip(_TOKEN_PUNCTUATION) for part in token.split("|")): + if not candidate: + continue - match = normalized_repos.get(candidate.lower()) - if match: - return match + if re.fullmatch(r"[\w.-]+/[\w.-]+", candidate): + match = normalized_repos.get(candidate.lower()) + if match: + return match - # Scanned over the raw string, not per token: Slack wraps URLs in `<…|…>`, so the - # tokenizer above never sees one in isolation. - for owner, repo in _GITHUB_REPO_URL_PATTERN.findall(text): - match = normalized_repos.get(f"{owner}/{repo.removesuffix('.git')}".lower()) - if match: - return match + from_url = _repo_from_github_url(candidate) + if from_url and (match := normalized_repos.get(from_url.lower())): + linked.add(match) - return None + return next(iter(linked)) if len(linked) == 1 else None diff --git a/posthog/test/test_git.py b/posthog/test/test_git.py index 526fc17c85e4..b3e6638689aa 100644 --- a/posthog/test/test_git.py +++ b/posthog/test/test_git.py @@ -31,14 +31,25 @@ class TestExtractExplicitRepo: "github.com/posthog/posthog-js/…/29764624536>", "posthog/posthog-js", ), - ("clone_url_suffix", "cloned from git@github.com/posthog/posthog.git", "posthog/posthog"), + ("clone_url_suffix", "cloned from git@github.com:posthog/posthog.git", "posthog/posthog"), ("unconnected_repo_url", "see https://github.com/acme/widgets/pull/1", None), ("lookalike_host", "see https://mygithub.com/posthog/posthog/pull/1", None), + ("host_prefix_spoof", "see https://github.com.evil.tld/posthog/posthog", None), ( "bare_token_beats_later_url", "fix posthog/posthog-js — context: https://github.com/posthog/posthog/pull/1", "posthog/posthog-js", ), + ( + "two_linked_repos_is_ambiguous", + "https://github.com/posthog/posthog/pull/1 broke https://github.com/posthog/posthog-js/actions/runs/2", + None, + ), + ( + "same_repo_linked_twice_is_not_ambiguous", + "https://github.com/posthog/posthog/pull/1 and https://github.com/posthog/posthog/actions/runs/2", + "posthog/posthog", + ), ] ) def test_extracts_matching_repo(self, _name: str, text: str, expected: str | None): From 1ec630bfcf290bd70eae18b2f54f4f2aaed40ddc Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Sun, 9 Aug 2026 21:23:00 +0100 Subject: [PATCH 009/289] fix(data-warehouse): point apple_search_ads at moved SourceResponse typings module `pipelines.pipeline.typings` was moved to `sources.common.typings` on master, which broke collection of the whole warehouse-sources suite. --- .../data_imports/sources/apple_search_ads/apple_search_ads.py | 2 +- .../temporal/data_imports/sources/apple_search_ads/source.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py index fe36dad826ea..0e8b4658e017 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py @@ -11,7 +11,7 @@ from structlog.types import FilteringBoundLogger from urllib3.util.retry import Retry -from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline.typings import SourceResponse +from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.settings import ( APPLE_SEARCH_ADS_ENDPOINTS, DEFAULT_INITIAL_LOOKBACK_DAYS, diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py index 835ed5905595..feb8f51b873f 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py @@ -9,7 +9,7 @@ SourceFieldInputConfigType, ) -from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline.typings import ( +from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ( SourceInputs, SourceResponse, ) From ffaa928702108fa43a288a34900a9105f5df2908 Mon Sep 17 00:00:00 2001 From: Tom Owers Date: Sun, 9 Aug 2026 21:53:07 +0100 Subject: [PATCH 010/289] chore(data-warehouse): restore isort ordering after typings import move --- .../sources/apple_search_ads/apple_search_ads.py | 2 +- .../temporal/data_imports/sources/apple_search_ads/source.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py index 0e8b4658e017..7f4f7ab1caf8 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/apple_search_ads.py @@ -11,7 +11,6 @@ from structlog.types import FilteringBoundLogger from urllib3.util.retry import Retry -from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.settings import ( APPLE_SEARCH_ADS_ENDPOINTS, DEFAULT_INITIAL_LOOKBACK_DAYS, @@ -22,6 +21,7 @@ ) from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager +from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse APPLE_SEARCH_ADS_HOST = "https://api.searchads.apple.com" # Apple Search Ads authenticates through Apple ID's OAuth token endpoint, not a Search Ads host. diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py index feb8f51b873f..9a212d2ca74a 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apple_search_ads/source.py @@ -9,10 +9,6 @@ SourceFieldInputConfigType, ) -from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import ( - SourceInputs, - SourceResponse, -) from products.warehouse_sources.backend.temporal.data_imports.sources.apple_search_ads.apple_search_ads import ( AppleSearchAdsCredentials, AppleSearchAdsResumeConfig, @@ -37,6 +33,7 @@ SourceSchema, build_endpoint_schemas, ) +from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.applesearchads import ( AppleSearchAdsSourceConfig, ) From 5047339848c72bea46846632bc2260c4a916a28f Mon Sep 17 00:00:00 2001 From: "tests-posthog[bot]" <250237707+tests-posthog[bot]@users.noreply.github.com> Date: Tue, 11 Aug 2026 06:30:58 +0000 Subject: [PATCH 011/289] test(backend): update query snapshots --- .../test_property_skip_indexes.ambr | 620 -- .../hogql/test/__snapshots__/test_query.ambr | 973 +-- .../test_events_predicate_pushdown.ambr | 362 - .../test/__snapshots__/test_in_cohort.ambr | 400 -- .../test/__snapshots__/test_lazy_tables.ambr | 255 - .../__snapshots__/test_property_types.ambr | 86 - .../test_state_aggregations.ambr | 532 -- .../test_event_taxonomy_query_runner.ambr | 125 - ...test_suggested_questions_query_runner.ambr | 23 - .../test_team_taxonomy_query_runner.ambr | 23 - .../test_trace_query_runner.ambr | 139 - .../test_traces_query_runner.ambr | 1169 +--- .../test_groups_query_runner.ambr | 83 - .../test/__snapshots__/test_funnel.ambr | 2416 ++----- ...test_funnel_breakdowns_by_current_url.ambr | 272 - .../test_funnel_correlation.ambr | 5878 ++--------------- 16 files changed, 928 insertions(+), 12428 deletions(-) diff --git a/posthog/hogql/test/__snapshots__/test_property_skip_indexes.ambr b/posthog/hogql/test/__snapshots__/test_property_skip_indexes.ambr index fc82245aaf04..aa1c1357f100 100644 --- a/posthog/hogql/test/__snapshots__/test_property_skip_indexes.ambr +++ b/posthog/hogql/test/__snapshots__/test_property_skip_indexes.ambr @@ -7,14 +7,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_dmat_string_no_skip_indexes[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_0_eq ''' SELECT count() AS `count()` @@ -23,14 +15,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_0_eq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '0'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_1_neq ''' SELECT count() AS `count()` @@ -39,14 +23,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_1_neq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(notEquals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '0'), 1)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_2_lt ''' SELECT count() AS `count()` @@ -55,14 +31,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_2_lt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_3_gt ''' SELECT count() AS `count()` @@ -71,14 +39,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_3_gt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(greater(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_4_icontains ''' SELECT count() AS `count()` @@ -87,14 +47,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_4_icontains[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), '%0%'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_5_is_set ''' SELECT count() AS `count()` @@ -103,14 +55,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_5_is_set[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), isNotNull(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_6_is_not_set ''' SELECT count() AS `count()` @@ -119,14 +63,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_6_is_not_set[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), isNull(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_7_in_multi ''' SELECT count() AS `count()` @@ -135,14 +71,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_json_only__no_skip_indexes_used_7_in_multi[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), tuple('0', '1'))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_lt_typed_datetime_property ''' SELECT count() AS `count()` @@ -151,14 +79,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_lt_typed_datetime_property[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(parseDateTime64BestEffortOrNull(if(notEquals(toJSONString(events.properties.^lt_datetime_prop), '{}'), toJSONString(events.properties.^lt_datetime_prop), if(isNull(events.properties.lt_datetime_prop), NULL, if(startsWith(dynamicType(events.properties.lt_datetime_prop), 'DateTime'), replaceOne(toString(events.properties.lt_datetime_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.lt_datetime_prop), 'Array'), startsWith(dynamicType(events.properties.lt_datetime_prop), 'Map'), startsWith(dynamicType(events.properties.lt_datetime_prop), 'Tuple')), toJSONString(events.properties.lt_datetime_prop), toString(events.properties.lt_datetime_prop))))), 6, 'UTC'), toDateTime64('2024-01-15 10:30:00.000000', 6, 'UTC')), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_lt_typed_numeric_property ''' SELECT count() AS `count()` @@ -167,14 +87,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_lt_typed_numeric_property[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), 'Float64'), 5), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_bloom_filter_0_eq ''' SELECT count() AS `count()` @@ -183,14 +95,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_bloom_filter_0_eq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_bloom_filter_1_in_multi ''' SELECT count() AS `count()` @@ -199,14 +103,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_bloom_filter_1_in_multi[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), tuple('2', '5'))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_bloom_filter_2_in_with_sentinel ''' SELECT count() AS `count()` @@ -215,14 +111,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_bloom_filter_2_in_with_sentinel[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), tuple('5', ''))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_bloom_filter_3_lt ''' SELECT count() AS `count()` @@ -231,14 +119,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_bloom_filter_3_lt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_0_eq ''' SELECT count() AS `count()` @@ -247,14 +127,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_0_eq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_1_lt ''' SELECT count() AS `count()` @@ -263,14 +135,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_1_lt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_2_gt ''' SELECT count() AS `count()` @@ -279,14 +143,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_2_gt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(greater(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_3_in_multi ''' SELECT count() AS `count()` @@ -295,14 +151,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_3_in_multi[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), tuple('2', '5'))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_4_icontains_no_isnotnull ''' SELECT count() AS `count()` @@ -311,14 +159,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_4_icontains_no_isnotnull[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), '%5%'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_5_regex ''' SELECT count() AS `count()` @@ -327,14 +167,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_minmax_5_regex[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(match(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), '[0-9]'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_ngrambf_lower_0_icontains_long ''' SELECT count() AS `count()` @@ -343,14 +175,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_ngrambf_lower_0_icontains_long[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), '%value_that_is_long_enough%'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_ngrambf_lower_1_icontains_null_sentinel ''' SELECT count() AS `count()` @@ -359,14 +183,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_non_nullable_ngrambf_lower_1_icontains_null_sentinel[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), '%null%'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_0_eq ''' SELECT count() AS `count()` @@ -375,14 +191,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_0_eq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_1_in_multi ''' SELECT count() AS `count()` @@ -391,14 +199,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_1_in_multi[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), tuple('2', '5'))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_2_neq ''' SELECT count() AS `count()` @@ -407,14 +207,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_2_neq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(notEquals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 1)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_3_not_in ''' SELECT count() AS `count()` @@ -423,14 +215,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_3_not_in[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(notIn(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), tuple('2', '5')), 1)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_4_lt ''' SELECT count() AS `count()` @@ -439,14 +223,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_4_lt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_5_gt ''' SELECT count() AS `count()` @@ -455,14 +231,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_5_gt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(greater(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_6_icontains ''' SELECT count() AS `count()` @@ -471,14 +239,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_6_icontains[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), '%5%'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_7_is_set ''' SELECT count() AS `count()` @@ -487,14 +247,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_bloom_filter_7_is_set[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), isNotNull(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_0_eq ''' SELECT count() AS `count()` @@ -503,14 +255,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_0_eq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_1_neq ''' SELECT count() AS `count()` @@ -519,14 +263,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_1_neq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(notEquals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 1)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_2_lt ''' SELECT count() AS `count()` @@ -535,14 +271,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_2_lt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_3_gt ''' SELECT count() AS `count()` @@ -551,14 +279,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_3_gt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(greater(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_4_lte ''' SELECT count() AS `count()` @@ -567,14 +287,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_4_lte[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(lessOrEquals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_5_gte ''' SELECT count() AS `count()` @@ -583,14 +295,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_5_gte[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(greaterOrEquals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_6_in_multi ''' SELECT count() AS `count()` @@ -599,14 +303,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_6_in_multi[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), tuple('2', '5'))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_7_icontains_with_isnotnull ''' SELECT count() AS `count()` @@ -615,14 +311,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_7_icontains_with_isnotnull[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), '%5%'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_8_regex ''' SELECT count() AS `count()` @@ -631,14 +319,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_8_regex[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(match(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), '[0-9]'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_non_string_constant_against_string_column_errors_0_int ''' SELECT count() AS `count()` @@ -647,14 +327,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_non_string_constant_against_string_column_errors_0_int[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), 5), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_non_string_constant_against_string_column_errors_1_float ''' SELECT count() AS `count()` @@ -663,14 +335,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_non_string_constant_against_string_column_errors_1_float[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), 5.5), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_non_string_constant_against_string_column_errors_2_datetime ''' SELECT count() AS `count()` @@ -679,14 +343,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_non_string_constant_against_string_column_errors_2_datetime[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), toDateTime64('2024-01-15 10:30:00.000000', 6, 'UTC')), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_numeric_looking_strings_compare_lexically ''' SELECT events.distinct_id AS distinct_id @@ -707,26 +363,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_numeric_looking_strings_compare_lexically[new_events_schema] - ''' - SELECT events.distinct_id AS distinct_id - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '500'), 0)) - ORDER BY events.distinct_id ASC - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_string_constant_flavors_0_string_pure_alpha ''' SELECT count() AS `count()` @@ -735,14 +371,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_string_constant_flavors_0_string_pure_alpha[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), 'apple'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_string_constant_flavors_1_string_numeric_looking ''' SELECT count() AS `count()` @@ -751,14 +379,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_string_constant_flavors_1_string_numeric_looking[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_string_constant_flavors_2_string_date_looking ''' SELECT count() AS `count()` @@ -767,14 +387,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_string_constant_flavors_2_string_date_looking[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '2024-01-15'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_string_constant_flavors_3_string_iso_datetime_looking ''' SELECT count() AS `count()` @@ -783,14 +395,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_minmax_lt_string_constant_flavors_3_string_iso_datetime_looking[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '2024-01-15T10:30:00Z'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_0_icontains_long ''' SELECT count() AS `count()` @@ -799,14 +403,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_0_icontains_long[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), '%value_that_is_long_enough%'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_1_not_icontains ''' SELECT count() AS `count()` @@ -815,14 +411,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_1_not_icontains[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(notILike(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), '%abc%'), 1)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_2_eq ''' SELECT count() AS `count()` @@ -831,14 +419,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_2_eq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_3_lt ''' SELECT count() AS `count()` @@ -847,14 +427,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_3_lt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_4_regex ''' SELECT count() AS `count()` @@ -863,14 +435,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_4_regex[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(match(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), 'value_that_is_long_enough'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_property_group_optimized_0_eq_string ''' SELECT count() AS `count()` @@ -879,14 +443,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_property_group_optimized_0_eq_string[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_property_group_optimized_1_is_set ''' SELECT count() AS `count()` @@ -895,14 +451,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_property_group_optimized_1_is_set[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), isNotNull(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_property_group_optimized_2_is_not_set ''' SELECT count() AS `count()` @@ -911,14 +459,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_property_group_optimized_2_is_not_set[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), isNull(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_property_group_optimized_3_in_multi ''' SELECT count() AS `count()` @@ -927,14 +467,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_property_group_optimized_3_in_multi[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), tuple('2', '5'))) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_property_group_optimized_4_lt ''' SELECT count() AS `count()` @@ -943,14 +475,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_property_group_optimized_4_lt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_property_group_optimized_5_icontains ''' SELECT count() AS `count()` @@ -959,14 +483,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_property_group_optimized_5_icontains[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(if(notEquals(toJSONString(events.properties.^test_prop), '{}'), toJSONString(events.properties.^test_prop), if(isNull(events.properties.test_prop), NULL, if(startsWith(dynamicType(events.properties.test_prop), 'DateTime'), replaceOne(toString(events.properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.test_prop), 'Array'), startsWith(dynamicType(events.properties.test_prop), 'Map'), startsWith(dynamicType(events.properties.test_prop), 'Tuple')), toJSONString(events.properties.test_prop), toString(events.properties.test_prop)))))), '%5%'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_typed_datetime_mat_col_uses_minmax_index ''' SELECT count() AS `count()` @@ -975,14 +491,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_typed_datetime_mat_col_uses_minmax_index[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(parseDateTime64BestEffortOrNull(if(notEquals(toJSONString(events.properties.^typed_datetime_prop), '{}'), toJSONString(events.properties.^typed_datetime_prop), if(isNull(events.properties.typed_datetime_prop), NULL, if(startsWith(dynamicType(events.properties.typed_datetime_prop), 'DateTime'), replaceOne(toString(events.properties.typed_datetime_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.typed_datetime_prop), 'Array'), startsWith(dynamicType(events.properties.typed_datetime_prop), 'Map'), startsWith(dynamicType(events.properties.typed_datetime_prop), 'Tuple')), toJSONString(events.properties.typed_datetime_prop), toString(events.properties.typed_datetime_prop))))), 6, 'UTC'), '2024-01-05 00:00:00'), 0)) - LIMIT 50000 - ''' -# --- # name: TestEventPropertySkipIndexes.test_typed_numeric_mat_col_uses_minmax_index ''' SELECT count() AS `count()` @@ -991,14 +499,6 @@ LIMIT 50000 ''' # --- -# name: TestEventPropertySkipIndexes.test_typed_numeric_mat_col_uses_minmax_index[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^typed_numeric_prop), '{}'), toJSONString(events.properties.^typed_numeric_prop), if(isNull(events.properties.typed_numeric_prop), NULL, if(startsWith(dynamicType(events.properties.typed_numeric_prop), 'DateTime'), replaceOne(toString(events.properties.typed_numeric_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.typed_numeric_prop), 'Array'), startsWith(dynamicType(events.properties.typed_numeric_prop), 'Map'), startsWith(dynamicType(events.properties.typed_numeric_prop), 'Tuple')), toJSONString(events.properties.typed_numeric_prop), toString(events.properties.typed_numeric_prop))))), 'Float64'), 5), 0)) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_bloom_filter_0_eq ''' SELECT count() AS `count()` @@ -1007,14 +507,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_bloom_filter_0_eq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_bloom_filter_1_in_multi ''' SELECT count() AS `count()` @@ -1023,14 +515,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_bloom_filter_1_in_multi[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), tuple('2', '5'))) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_bloom_filter_2_not_in ''' SELECT count() AS `count()` @@ -1039,14 +523,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_bloom_filter_2_not_in[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(notIn(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), tuple('2', '5')), 1)) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_bloom_filter_3_lt ''' SELECT count() AS `count()` @@ -1055,14 +531,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_bloom_filter_3_lt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_minmax_0_eq ''' SELECT count() AS `count()` @@ -1071,14 +539,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_minmax_0_eq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_minmax_1_lt ''' SELECT count() AS `count()` @@ -1087,14 +547,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_minmax_1_lt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_minmax_2_gt ''' SELECT count() AS `count()` @@ -1103,14 +555,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_minmax_2_gt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(greater(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_minmax_3_in_multi ''' SELECT count() AS `count()` @@ -1119,14 +563,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_minmax_3_in_multi[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), tuple('2', '5'))) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_minmax_4_icontains_with_isnotnull ''' SELECT count() AS `count()` @@ -1135,14 +571,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_minmax_4_icontains_with_isnotnull[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop)))))), '%5%'), 0)) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_0_icontains_long ''' SELECT count() AS `count()` @@ -1151,14 +579,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_0_icontains_long[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(ilike(toString(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop)))))), '%value_that_is_long_enough%'), 0)) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_1_eq ''' SELECT count() AS `count()` @@ -1167,14 +587,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_mat_col_nullable_ngrambf_lower_1_eq[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_property_group_optimized_0_eq_string ''' SELECT count() AS `count()` @@ -1183,14 +595,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_property_group_optimized_0_eq_string[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_property_group_optimized_1_is_set ''' SELECT count() AS `count()` @@ -1199,14 +603,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_property_group_optimized_1_is_set[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), isNotNull(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))))) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_property_group_optimized_2_in_multi ''' SELECT count() AS `count()` @@ -1215,14 +611,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_property_group_optimized_2_in_multi[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), tuple('2', '5'))) - LIMIT 50000 - ''' -# --- # name: TestPersonOnEventsPropertySkipIndexes.test_property_group_optimized_3_lt ''' SELECT count() AS `count()` @@ -1231,14 +619,6 @@ LIMIT 50000 ''' # --- -# name: TestPersonOnEventsPropertySkipIndexes.test_property_group_optimized_3_lt[new_events_schema] - ''' - SELECT count() AS `count()` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(less(if(notEquals(toJSONString(events.person_properties.^test_prop), '{}'), toJSONString(events.person_properties.^test_prop), if(isNull(events.person_properties.test_prop), NULL, if(startsWith(dynamicType(events.person_properties.test_prop), 'DateTime'), replaceOne(toString(events.person_properties.test_prop), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.test_prop), 'Array'), startsWith(dynamicType(events.person_properties.test_prop), 'Map'), startsWith(dynamicType(events.person_properties.test_prop), 'Tuple')), toJSONString(events.person_properties.test_prop), toString(events.person_properties.test_prop))))), '5'), 0)) - LIMIT 50000 - ''' -# --- # name: TestPersonPropertySkipIndexes.test_mat_col_nullable_bloom_filter_0_eq ''' SELECT count() AS `count()` diff --git a/posthog/hogql/test/__snapshots__/test_query.ambr b/posthog/hogql/test/__snapshots__/test_query.ambr index e7ea49c6ee71..4457bd4e3208 100644 --- a/posthog/hogql/test/__snapshots__/test_query.ambr +++ b/posthog/hogql/test/__snapshots__/test_query.ambr @@ -27,34 +27,6 @@ OFFSET 0 ''' # --- -# name: TestQuery.test_clickhouse_timestamp_handling[new_events_schema] - ''' - -- ClickHouse - SELECT events__fingerprint_issue_state.issue_id AS id, count(DISTINCT events.uuid) AS occurrences, count(DISTINCT nullIf(events.`$session_id`, %(hogql_val_0)s)) AS sessions, count(DISTINCT events.distinct_id) AS users, max(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS last_seen, min(toTimeZone(events.timestamp, %(hogql_val_2)s)) AS first_seen, reverse(arrayMap(x -> countEqual(groupArray(dateDiff(%(hogql_val_3)s, toStartOfHour(toTimeZone(events.timestamp, %(hogql_val_4)s)), toStartOfHour(now64(6, %(hogql_val_5)s)))), x), range(24))) AS volumeDay, reverse(arrayMap(x -> countEqual(groupArray(dateDiff(%(hogql_val_6)s, toStartOfDay(toTimeZone(events.timestamp, %(hogql_val_7)s)), toStartOfDay(now64(6, %(hogql_val_8)s)))), x), range(31))) AS volumeMonth, reverse(arrayMap(x -> countEqual(groupArray(dateDiff(%(hogql_val_9)s, toStartOfHour(toTimeZone(events.timestamp, %(hogql_val_10)s)), toStartOfHour(now64(6, %(hogql_val_11)s)))), x), range(168))) AS customVolume - FROM events_json AS events LEFT OUTER JOIN ( - SELECT cityHash64(error_tracking_fingerprint_issue_state.fingerprint) AS fp_hash, toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_id), error_tracking_fingerprint_issue_state.version), 1)) AS issue_id - FROM error_tracking_fingerprint_issue_state - WHERE equals(error_tracking_fingerprint_issue_state.team_id, 99999) - GROUP BY fp_hash - HAVING equals(argMax(error_tracking_fingerprint_issue_state.is_deleted, error_tracking_fingerprint_issue_state.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__fingerprint_issue_state ON equals(cityHash64(events.properties.`$exception_fingerprint`), events__fingerprint_issue_state.fp_hash) - WHERE and(equals(events.team_id, 99999), and(equals(events.event, %(hogql_val_12)s), isNotNull(events__fingerprint_issue_state.issue_id), or(and(greater(toTimeZone(events.timestamp, %(hogql_val_13)s), parseDateTime64BestEffort(%(hogql_val_14)s, 6, %(hogql_val_15)s)), less(toTimeZone(events.timestamp, %(hogql_val_16)s), parseDateTime64BestEffort(%(hogql_val_17)s, 6, %(hogql_val_18)s))), and(greater(toTimeZone(events.timestamp, %(hogql_val_19)s), toDateTime(%(hogql_val_20)s, %(hogql_val_21)s)), less(toTimeZone(events.timestamp, %(hogql_val_22)s), toDateTime64(%(hogql_val_23)s, 6, %(hogql_val_24)s)))))) - GROUP BY events__fingerprint_issue_state.issue_id - ORDER BY occurrences DESC - LIMIT 51 - OFFSET 0 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT issue_id AS id, count(DISTINCT uuid) AS occurrences, count(DISTINCT nullIf($session_id, '')) AS sessions, count(DISTINCT distinct_id) AS users, max(timestamp) AS last_seen, min(timestamp) AS first_seen, reverse(arrayMap(x -> countEqual(groupArray(dateDiff('hour', toStartOfHour(timestamp), toStartOfHour(now()))), x), range(24))) AS volumeDay, reverse(arrayMap(x -> countEqual(groupArray(dateDiff('day', toStartOfDay(timestamp), toStartOfDay(now()))), x), range(31))) AS volumeMonth, reverse(arrayMap(x -> countEqual(groupArray(dateDiff('hour', toStartOfHour(timestamp), toStartOfHour(now()))), x), range(168))) AS customVolume - FROM events - WHERE and(equals(event, '$exception'), isNotNull(issue_id), or(and(greater(timestamp, toDateTime('2025-02-10 23:53:03.175952+02:30')), less(timestamp, toDateTime('2025-02-11 23:53'))), and(greater(timestamp, toDateTime('2025-02-12 23:53:03')), less(timestamp, toDateTime('2025-02-13 23:53:03.175952'))))) - GROUP BY issue_id - ORDER BY occurrences DESC - LIMIT 51 - OFFSET 0 - ''' -# --- # name: TestQuery.test_hogql_arrays ''' -- ClickHouse @@ -87,26 +59,6 @@ LIMIT 1 ''' # --- -# name: TestQuery.test_hogql_groupby_unnecessary_ifnull[new_events_schema] - ''' - -- ClickHouse - SELECT toDate(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS timestamp, count() AS cnt - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), greaterOrEquals(timestamp, addDays(today(), -10))) - GROUP BY timestamp - HAVING greater(cnt, 10) - LIMIT 1 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT toDate(timestamp) AS timestamp, count() AS cnt - FROM events - WHERE greaterOrEquals(timestamp, addDays(today(), -10)) - GROUP BY timestamp - HAVING greater(cnt, 10) - LIMIT 1 - ''' -# --- # name: TestQuery.test_hogql_lambdas ''' -- ClickHouse @@ -153,40 +105,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_hogql_proper_ifnull[new_events_schema] - ''' - -- ClickHouse - WITH latest_events AS ( - SELECT events.distinct_id AS distinct_id, argMax(events.properties.`$os_version`, toTimeZone(events.timestamp, %(hogql_val_0)s)) AS latest_os_version - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(equals(events.properties.`$os`, %(hogql_val_1)s), isNotNull(events.properties.`$os`)), greaterOrEquals(events.timestamp, minus(now64(6, %(hogql_val_2)s), toIntervalDay(30)))) - GROUP BY events.distinct_id), major_versions AS ( - SELECT latest_events.distinct_id AS distinct_id, latest_events.latest_os_version AS latest_os_version, splitByChar(%(hogql_val_3)s, ifNull(latest_events.latest_os_version, %(hogql_val_4)s))[1] AS major_version - FROM latest_events) - SELECT major_versions.major_version AS major_version, count() AS user_count, round(divide(multiply(100, count()), sum(count()) OVER ()), 2) AS percentage - FROM major_versions - WHERE in(major_versions.major_version, tuple(%(hogql_val_5)s, %(hogql_val_6)s, %(hogql_val_7)s)) - GROUP BY major_versions.major_version - ORDER BY major_versions.major_version ASC - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - WITH latest_events AS ( - SELECT distinct_id, argMax(properties.$os_version, timestamp) AS latest_os_version - FROM events - WHERE and(equals(properties.$os, 'iOS'), greaterOrEquals(timestamp, minus(now(), toIntervalDay(30)))) - GROUP BY distinct_id), major_versions AS ( - SELECT distinct_id, latest_os_version, splitByChar('.', ifNull(latest_os_version, ''))[1] AS major_version - FROM latest_events) - SELECT major_version, count() AS user_count, round(divide(multiply(100, count()), sum(count()) OVER ()), 2) AS percentage - FROM major_versions - WHERE in(major_version, tuple('17', '18', '26')) - GROUP BY major_version - ORDER BY major_version ASC - LIMIT 100 - ''' -# --- # name: TestQuery.test_hogql_query_filters ''' -- ClickHouse @@ -237,56 +155,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_hogql_query_filters[new_events_schema.1] - ''' - - SELECT event, distinct_id - FROM events - WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), equals(properties.index, '4')) - LIMIT 100 - ''' -# --- -# name: TestQuery.test_hogql_query_filters[new_events_schema.2] - ''' - -- ClickHouse - SELECT events.event AS event, events.distinct_id AS distinct_id - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), equals(events.distinct_id, %(hogql_val_0)s), and(ifNull(equals(if(notEquals(toJSONString(events.properties.^index), '{}'), toJSONString(events.properties.^index), if(isNull(events.properties.index), NULL, if(startsWith(dynamicType(events.properties.index), 'DateTime'), replaceOne(toString(events.properties.index), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.index), 'Array'), startsWith(dynamicType(events.properties.index), 'Map'), startsWith(dynamicType(events.properties.index), 'Tuple')), toJSONString(events.properties.index), toString(events.properties.index))))), %(hogql_val_1)s), 0), lessOrEquals(events.timestamp, toDateTime64('2020-01-02 23:59:59.999999', 6, 'UTC')), greaterOrEquals(events.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')))) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, distinct_id - FROM events - WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), and(equals(properties.index, '4'), lessOrEquals(timestamp, toDateTime('2020-01-02 23:59:59.999999')), greaterOrEquals(timestamp, toDateTime('2020-01-01 00:00:00.000000')))) - LIMIT 100 - ''' -# --- -# name: TestQuery.test_hogql_query_filters[new_events_schema.3] - ''' - - SELECT event, distinct_id - FROM events - WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), and(equals(properties.index, '4'), lessOrEquals(timestamp, toDateTime('2020-01-02 23:59:59.999999')), greaterOrEquals(timestamp, toDateTime('2020-01-01 00:00:00.000000')))) - LIMIT 100 - ''' -# --- -# name: TestQuery.test_hogql_query_filters[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event, events.distinct_id AS distinct_id - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), equals(events.distinct_id, %(hogql_val_0)s), ifNull(equals(if(notEquals(toJSONString(events.properties.^index), '{}'), toJSONString(events.properties.^index), if(isNull(events.properties.index), NULL, if(startsWith(dynamicType(events.properties.index), 'DateTime'), replaceOne(toString(events.properties.index), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.index), 'Array'), startsWith(dynamicType(events.properties.index), 'Map'), startsWith(dynamicType(events.properties.index), 'Tuple')), toJSONString(events.properties.index), toString(events.properties.index))))), %(hogql_val_1)s), 0)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, distinct_id - FROM events - WHERE and(equals(distinct_id, 'RANDOM_TEST_ID::UUID'), equals(properties.index, '4')) - LIMIT 100 - ''' -# --- # name: TestQuery.test_hogql_query_filters_alias ''' -- ClickHouse @@ -303,22 +171,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_hogql_query_filters_alias[new_events_schema] - ''' - -- ClickHouse - SELECT e.event AS event, e.distinct_id AS distinct_id - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(e.properties.^random_uuid), '{}'), toJSONString(e.properties.^random_uuid), if(isNull(e.properties.random_uuid), NULL, if(startsWith(dynamicType(e.properties.random_uuid), 'DateTime'), replaceOne(toString(e.properties.random_uuid), ' ', 'T'), if(or(startsWith(dynamicType(e.properties.random_uuid), 'Array'), startsWith(dynamicType(e.properties.random_uuid), 'Map'), startsWith(dynamicType(e.properties.random_uuid), 'Tuple')), toJSONString(e.properties.random_uuid), toString(e.properties.random_uuid))))), %(hogql_val_0)s), 0)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, distinct_id - FROM events AS e - WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') - LIMIT 100 - ''' -# --- # name: TestQuery.test_hogql_query_filters_session_date_range ''' -- ClickHouse @@ -399,28 +251,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_hogql_union_all_limits[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events - WHERE equals(events.team_id, 99999) - LIMIT 100 UNION ALL - SELECT events.event AS event - FROM events_json AS events - WHERE equals(events.team_id, 99999) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events - LIMIT 100 UNION ALL - SELECT event - FROM events - LIMIT 100 - ''' -# --- # name: TestQuery.test_hogql_unnecessary_ifnull ''' -- ClickHouse @@ -437,22 +267,6 @@ LIMIT 1 ''' # --- -# name: TestQuery.test_hogql_unnecessary_ifnull[new_events_schema] - ''' - -- ClickHouse - SELECT toDate(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS timestamp, JSONExtractInt(ifNull(if(notEquals(toJSONString(events.properties.^field), '{}'), toJSONString(events.properties.^field), if(isNull(events.properties.field), NULL, if(startsWith(dynamicType(events.properties.field), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(events.properties.field), ' ', 'T')), ''), '"'), toJSONString(events.properties.field)))), '')) AS json_int - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), greaterOrEquals(timestamp, addDays(today(), -10)), equals(json_int, 17)) - LIMIT 1 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT toDate(timestamp) AS timestamp, JSONExtractInt(properties, 'field') AS json_int - FROM events - WHERE and(greaterOrEquals(timestamp, addDays(today(), -10)), equals(json_int, 17)) - LIMIT 1 - ''' -# --- # name: TestQuery.test_join_with_property_materialized_session_id ''' -- ClickHouse @@ -485,38 +299,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_join_with_property_materialized_session_id[new_events_schema.1] - ''' - - SELECT e.event, s.session_id - FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, e.properties.$session_id) - WHERE notEquals(e.properties.$session_id, NULL) - LIMIT 10 - ''' -# --- -# name: TestQuery.test_join_with_property_materialized_session_id[new_events_schema] - ''' - -- ClickHouse - SELECT e.event AS event, s.session_id AS session_id - FROM ( - SELECT e.event AS event, e.properties.`$session_id` AS `properties__$session_id` - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), isNotNull(e.properties.`$session_id`)) - LIMIT 10) AS e LEFT JOIN ( - SELECT session_replay_events.session_id AS session_id - FROM session_replay_events - WHERE equals(session_replay_events.team_id, 99999) - GROUP BY session_replay_events.session_id) AS s ON equals(s.session_id, e.`properties__$session_id`) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT e.event, s.session_id - FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, e.properties.$session_id) - WHERE notEquals(e.properties.$session_id, NULL) - LIMIT 10 - ''' -# --- # name: TestQuery.test_join_with_property_not_materialized ''' -- ClickHouse @@ -549,38 +331,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_join_with_property_not_materialized[new_events_schema.1] - ''' - - SELECT e.event, s.session_id - FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, e.properties.$$$session_id) - WHERE notEquals(e.properties.$$$session_id, NULL) - LIMIT 10 - ''' -# --- -# name: TestQuery.test_join_with_property_not_materialized[new_events_schema] - ''' - -- ClickHouse - SELECT e.event AS event, s.session_id AS session_id - FROM ( - SELECT e.event AS event, if(notEquals(toJSONString(e.properties.^`$$$session_id`), '{}'), toJSONString(e.properties.^`$$$session_id`), if(isNull(e.properties.`$$$session_id`), NULL, if(startsWith(dynamicType(e.properties.`$$$session_id`), 'DateTime'), replaceOne(toString(e.properties.`$$$session_id`), ' ', 'T'), if(or(startsWith(dynamicType(e.properties.`$$$session_id`), 'Array'), startsWith(dynamicType(e.properties.`$$$session_id`), 'Map'), startsWith(dynamicType(e.properties.`$$$session_id`), 'Tuple')), toJSONString(e.properties.`$$$session_id`), toString(e.properties.`$$$session_id`))))) AS `properties__$$$session_id` - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), isNotNull(if(notEquals(toJSONString(e.properties.^`$$$session_id`), '{}'), toJSONString(e.properties.^`$$$session_id`), if(isNull(e.properties.`$$$session_id`), NULL, if(startsWith(dynamicType(e.properties.`$$$session_id`), 'DateTime'), replaceOne(toString(e.properties.`$$$session_id`), ' ', 'T'), if(or(startsWith(dynamicType(e.properties.`$$$session_id`), 'Array'), startsWith(dynamicType(e.properties.`$$$session_id`), 'Map'), startsWith(dynamicType(e.properties.`$$$session_id`), 'Tuple')), toJSONString(e.properties.`$$$session_id`), toString(e.properties.`$$$session_id`))))))) - LIMIT 10) AS e LEFT JOIN ( - SELECT session_replay_events.session_id AS session_id - FROM session_replay_events - WHERE equals(session_replay_events.team_id, 99999) - GROUP BY session_replay_events.session_id) AS s ON equals(s.session_id, e.`properties__$$$session_id`) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT e.event, s.session_id - FROM events AS e LEFT JOIN session_replay_events AS s ON equals(s.session_id, e.properties.$$$session_id) - WHERE notEquals(e.properties.$$$session_id, NULL) - LIMIT 10 - ''' -# --- # name: TestQuery.test_prop_cohort_basic ''' -- ClickHouse @@ -643,39 +393,11 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_prop_cohort_basic[new_events_schema.1] - ''' - -- ClickHouse - SELECT events.event AS event, count(*) AS `count(*)` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.person_id, ( - SELECT cohortpeople.person_id AS person_id - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 99999), equals(cohortpeople.cohort_id, XX)) - GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version - HAVING greater(sum(cohortpeople.sign), 0)))) - GROUP BY events.event - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, count(*) - FROM events - WHERE in(person_id, ( - SELECT person_id - FROM raw_cohort_people - WHERE equals(cohort_id, XX) - GROUP BY person_id, cohort_id, version - HAVING greater(sum(sign), 0))) - GROUP BY event - LIMIT 100 - ''' -# --- -# name: TestQuery.test_prop_cohort_basic[new_events_schema] +# name: TestQuery.test_prop_cohort_static ''' -- ClickHouse SELECT events.event AS event, count() AS `count()` - FROM events_json AS events LEFT OUTER JOIN ( + FROM events LEFT OUTER JOIN ( SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id FROM person_distinct_id_overrides WHERE equals(person_distinct_id_overrides.team_id, 99999) @@ -683,11 +405,9 @@ HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) WHERE and(equals(events.team_id, 99999), in(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), ( - SELECT cohortpeople.person_id AS person_id - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 99999), equals(cohortpeople.cohort_id, XX)) - GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version - HAVING greater(sum(cohortpeople.sign), 0)))) + SELECT person_static_cohort.person_id AS person_id + FROM person_static_cohort + WHERE and(equals(person_static_cohort.team_id, 99999), equals(person_static_cohort.cohort_id, XX))))) GROUP BY events.event LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 @@ -697,45 +417,13 @@ FROM events WHERE in(person_id, ( SELECT person_id - FROM raw_cohort_people - WHERE equals(cohort_id, XX) - GROUP BY person_id, cohort_id, version - HAVING greater(sum(sign), 0))) + FROM static_cohort_people + WHERE equals(cohort_id, XX))) GROUP BY event LIMIT 100 ''' # --- -# name: TestQuery.test_prop_cohort_static - ''' - -- ClickHouse - SELECT events.event AS event, count() AS `count()` - FROM events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), in(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), ( - SELECT person_static_cohort.person_id AS person_id - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), equals(person_static_cohort.cohort_id, XX))))) - GROUP BY events.event - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, count() - FROM events - WHERE in(person_id, ( - SELECT person_id - FROM static_cohort_people - WHERE equals(cohort_id, XX))) - GROUP BY event - LIMIT 100 - ''' -# --- -# name: TestQuery.test_prop_cohort_static.1 +# name: TestQuery.test_prop_cohort_static.1 ''' -- ClickHouse SELECT events.event AS event, count(*) AS `count(*)` @@ -759,60 +447,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_prop_cohort_static[new_events_schema.1] - ''' - -- ClickHouse - SELECT events.event AS event, count(*) AS `count(*)` - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.person_id, ( - SELECT person_static_cohort.person_id AS person_id - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), equals(person_static_cohort.cohort_id, XX))))) - GROUP BY events.event - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, count(*) - FROM events - WHERE in(person_id, ( - SELECT person_id - FROM static_cohort_people - WHERE equals(cohort_id, XX))) - GROUP BY event - LIMIT 100 - ''' -# --- -# name: TestQuery.test_prop_cohort_static[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event, count() AS `count()` - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), in(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), ( - SELECT person_static_cohort.person_id AS person_id - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), equals(person_static_cohort.cohort_id, XX))))) - GROUP BY events.event - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, count() - FROM events - WHERE in(person_id, ( - SELECT person_id - FROM static_cohort_people - WHERE equals(cohort_id, XX))) - GROUP BY event - LIMIT 100 - ''' -# --- # name: TestQuery.test_query ''' -- ClickHouse @@ -831,24 +465,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_query[new_events_schema] - ''' - -- ClickHouse - SELECT count() AS `count()`, events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.properties.^random_uuid), '{}'), toJSONString(events.properties.^random_uuid), if(isNull(events.properties.random_uuid), NULL, if(startsWith(dynamicType(events.properties.random_uuid), 'DateTime'), replaceOne(toString(events.properties.random_uuid), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.random_uuid), 'Array'), startsWith(dynamicType(events.properties.random_uuid), 'Map'), startsWith(dynamicType(events.properties.random_uuid), 'Tuple')), toJSONString(events.properties.random_uuid), toString(events.properties.random_uuid))))), %(hogql_val_0)s), 0)) - GROUP BY events.event - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT count(), event - FROM events - WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') - GROUP BY event - LIMIT 100 - ''' -# --- # name: TestQuery.test_query_distinct ''' -- ClickHouse @@ -894,27 +510,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_query_joins_events_e_pdi[new_events_schema] - ''' - -- ClickHouse - SELECT e.event AS event, toTimeZone(e.timestamp, %(hogql_val_0)s) AS timestamp, e__pdi.distinct_id AS distinct_id, e__pdi.person_id AS person_id - FROM events_json AS e INNER JOIN ( - SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 99999) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) - WHERE equals(e.team_id, 99999) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, e.timestamp, e.pdi.distinct_id, pdi.person_id - FROM events AS e - LIMIT 10 - ''' -# --- # name: TestQuery.test_query_joins_events_first_to_persons ''' -- ClickHouse @@ -946,37 +541,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_query_joins_events_first_to_persons[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event, persons.properties___email AS email - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) JOIN ( - SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___email - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), ( - SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_1)s), person.version), plus(now64(6, %(hogql_val_2)s), toIntervalDay(1))))))) - SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), persons.id) - WHERE and(equals(events.team_id, 99999), equals(events.event, %(hogql_val_3)s)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT events.event, persons.properties.email - FROM events JOIN persons ON equals(events.person_id, persons.id) - WHERE equals(events.event, 'pageview') - LIMIT 100 - ''' -# --- # name: TestQuery.test_query_joins_events_pdi ''' -- ClickHouse @@ -998,27 +562,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_query_joins_events_pdi[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event, toTimeZone(events.timestamp, %(hogql_val_0)s) AS timestamp, events__pdi.distinct_id AS distinct_id, events__pdi.person_id AS person_id - FROM events_json AS events INNER JOIN ( - SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 99999) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) - WHERE equals(events.team_id, 99999) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, timestamp, pdi.distinct_id, pdi.person_id - FROM events - LIMIT 10 - ''' -# --- # name: TestQuery.test_query_joins_events_pdi_e_person_properties ''' -- ClickHouse @@ -1049,36 +592,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_query_joins_events_pdi_e_person_properties[new_events_schema] - ''' - -- ClickHouse - SELECT e.event AS event, toTimeZone(e.timestamp, %(hogql_val_3)s) AS timestamp, e__pdi.distinct_id AS distinct_id, e__pdi__person.properties___sneaky_mail AS sneaky_mail - FROM events_json AS e INNER JOIN ( - SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 99999) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) LEFT JOIN ( - SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), ( - SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_1)s), person.version), plus(now64(6, %(hogql_val_2)s), toIntervalDay(1))))))) - SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) - WHERE equals(e.team_id, 99999) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, e.timestamp, pdi.distinct_id, e.pdi.person.properties.sneaky_mail - FROM events AS e - LIMIT 10 - ''' -# --- # name: TestQuery.test_query_joins_events_pdi_person ''' -- ClickHouse @@ -1106,33 +619,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_query_joins_events_pdi_person[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event, toTimeZone(events.timestamp, %(hogql_val_2)s) AS timestamp, events__pdi.distinct_id AS distinct_id, events__pdi__person.id AS id - FROM events_json AS events INNER JOIN ( - SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS events__pdi___person_id, argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 99999) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) LEFT JOIN ( - SELECT person.id AS id - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_0)s), person.version), plus(now64(6, %(hogql_val_1)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS events__pdi__person ON equals(events__pdi.events__pdi___person_id, events__pdi__person.id) - WHERE equals(events.team_id, 99999) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, timestamp, pdi.distinct_id, pdi.person.id - FROM events - LIMIT 10 - ''' -# --- # name: TestQuery.test_query_joins_events_pdi_person_properties ''' -- ClickHouse @@ -1163,36 +649,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_query_joins_events_pdi_person_properties[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event, toTimeZone(events.timestamp, %(hogql_val_3)s) AS timestamp, events__pdi.distinct_id AS distinct_id, events__pdi__person.properties___sneaky_mail AS sneaky_mail - FROM events_json AS events INNER JOIN ( - SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS events__pdi___person_id, argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 99999) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) LEFT JOIN ( - SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), ( - SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_1)s), person.version), plus(now64(6, %(hogql_val_2)s), toIntervalDay(1))))))) - SETTINGS optimize_aggregation_in_order=1) AS events__pdi__person ON equals(events__pdi.events__pdi___person_id, events__pdi__person.id) - WHERE equals(events.team_id, 99999) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, timestamp, pdi.distinct_id, pdi.person.properties.sneaky_mail - FROM events - LIMIT 10 - ''' -# --- # name: TestQuery.test_query_joins_events_person_properties ''' -- ClickHouse @@ -1223,36 +679,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_query_joins_events_person_properties[new_events_schema] - ''' - -- ClickHouse - SELECT e.event AS event, toTimeZone(e.timestamp, %(hogql_val_3)s) AS timestamp, e__pdi__person.properties___sneaky_mail AS sneaky_mail - FROM events_json AS e INNER JOIN ( - SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS e__pdi___person_id, argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 99999) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS e__pdi ON equals(e.distinct_id, e__pdi.distinct_id) LEFT JOIN ( - SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), ( - SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_1)s), person.version), plus(now64(6, %(hogql_val_2)s), toIntervalDay(1))))))) - SETTINGS optimize_aggregation_in_order=1) AS e__pdi__person ON equals(e__pdi.e__pdi___person_id, e__pdi__person.id) - WHERE equals(e.team_id, 99999) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, e.timestamp, e.pdi.person.properties.sneaky_mail - FROM events AS e - LIMIT 10 - ''' -# --- # name: TestQuery.test_query_joins_events_person_properties_in_aggregration ''' -- ClickHouse @@ -1285,38 +711,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_query_joins_events_person_properties_in_aggregration[new_events_schema] - ''' - -- ClickHouse - SELECT s__pdi__person.properties___sneaky_mail AS sneaky_mail, count() AS `count()` - FROM events_json AS s INNER JOIN ( - SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS s__pdi___person_id, argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 99999) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS s__pdi ON equals(s.distinct_id, s__pdi.distinct_id) LEFT JOIN ( - SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), ( - SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_1)s), person.version), plus(now64(6, %(hogql_val_2)s), toIntervalDay(1))))))) - SETTINGS optimize_aggregation_in_order=1) AS s__pdi__person ON equals(s__pdi.s__pdi___person_id, s__pdi__person.id) - WHERE equals(s.team_id, 99999) - GROUP BY s__pdi__person.properties___sneaky_mail - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT s.pdi.person.properties.sneaky_mail, count() - FROM events AS s - GROUP BY s.pdi.person.properties.sneaky_mail - LIMIT 10 - ''' -# --- # name: TestQuery.test_query_joins_lazy_on_both_sides ''' -- ClickHouse @@ -1338,76 +732,21 @@ SETTINGS optimize_aggregation_in_order=1) AS e2__override ON equals(e2.distinct_id, e2__override.distinct_id) WHERE equals(e2.team_id, 99999)) AS e2 ON equals(if(not(empty(e1__override.distinct_id)), e1__override.person_id, e1.person_id), e2.person_id) WHERE and(equals(e1.team_id, 99999), equals(e1.event, %(hogql_val_0)s), equals(e2.event, %(hogql_val_1)s)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT e1.event, e2.event - FROM events AS e1 JOIN events AS e2 ON equals(e1.person_id, e2.person_id) - WHERE and(equals(e1.event, 'pageview'), equals(e2.event, 'click')) - LIMIT 100 - ''' -# --- -# name: TestQuery.test_query_joins_lazy_on_both_sides[new_events_schema] - ''' - -- ClickHouse - SELECT e1.event, e2.event AS event - FROM events_json AS e1 LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS e1__override ON equals(e1.distinct_id, e1__override.distinct_id) JOIN ( - SELECT e2.distinct_id AS distinct_id, e2.event AS event, e2.person_id AS event_person_id, if(not(empty(e2__override.distinct_id)), e2__override.person_id, e2.person_id) AS person_id - FROM events_json AS e2 LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS e2__override ON equals(e2.distinct_id, e2__override.distinct_id) - WHERE equals(e2.team_id, 99999)) AS e2 ON equals(if(not(empty(e1__override.distinct_id)), e1__override.person_id, e1.person_id), e2.person_id) - WHERE and(equals(e1.team_id, 99999), equals(e1.event, %(hogql_val_0)s), equals(e2.event, %(hogql_val_1)s)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT e1.event, e2.event - FROM events AS e1 JOIN events AS e2 ON equals(e1.person_id, e2.person_id) - WHERE and(equals(e1.event, 'pageview'), equals(e2.event, 'click')) - LIMIT 100 - ''' -# --- -# name: TestQuery.test_query_joins_pdi - ''' - -- ClickHouse - SELECT e.event AS event, toTimeZone(e.timestamp, %(hogql_val_0)s) AS timestamp, pdi.person_id AS person_id - FROM events AS e INNER JOIN ( - SELECT person_distinct_id2.distinct_id AS distinct_id, argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 99999) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS pdi ON equals(e.distinct_id, pdi.distinct_id) - WHERE equals(e.team_id, 99999) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, timestamp, pdi.person_id - FROM events AS e INNER JOIN ( - SELECT distinct_id, argMax(person_id, version) AS person_id - FROM raw_person_distinct_ids - GROUP BY distinct_id - HAVING equals(argMax(is_deleted, version), 0)) AS pdi ON equals(e.distinct_id, pdi.distinct_id) + LIMIT 100 + SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 + + -- HogQL + SELECT e1.event, e2.event + FROM events AS e1 JOIN events AS e2 ON equals(e1.person_id, e2.person_id) + WHERE and(equals(e1.event, 'pageview'), equals(e2.event, 'click')) LIMIT 100 ''' # --- -# name: TestQuery.test_query_joins_pdi[new_events_schema] +# name: TestQuery.test_query_joins_pdi ''' -- ClickHouse SELECT e.event AS event, toTimeZone(e.timestamp, %(hogql_val_0)s) AS timestamp, pdi.person_id AS person_id - FROM events_json AS e INNER JOIN ( + FROM events AS e INNER JOIN ( SELECT person_distinct_id2.distinct_id AS distinct_id, argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id FROM person_distinct_id2 WHERE equals(person_distinct_id2.team_id, 99999) @@ -1511,35 +850,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_query_joins_persons_to_events[new_events_schema] - ''' - -- ClickHouse - SELECT persons.id AS id, events.event AS event - FROM ( - SELECT person.id AS id - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_0)s), person.version), plus(now64(6, %(hogql_val_1)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS persons JOIN ( - SELECT events.distinct_id AS distinct_id, events.event AS event, events.person_id AS event_person_id, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE equals(events.team_id, 99999)) AS events ON equals(persons.id, events.person_id) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT persons.id, events.event - FROM persons JOIN events ON equals(persons.id, events.person_id) - LIMIT 10 - ''' -# --- # name: TestQuery.test_query_joins_simple ''' -- ClickHouse @@ -1570,36 +880,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_query_joins_simple[new_events_schema] - ''' - -- ClickHouse - SELECT e.event AS event, toTimeZone(e.timestamp, %(hogql_val_3)s) AS timestamp, pdi.distinct_id AS distinct_id, p.id AS id, p.properties___sneaky_mail AS sneaky_mail - FROM events_json AS e LEFT JOIN ( - SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 99999) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS pdi ON equals(pdi.distinct_id, e.distinct_id) LEFT JOIN ( - SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), ( - SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_1)s), person.version), plus(now64(6, %(hogql_val_2)s), toIntervalDay(1))))))) - SETTINGS optimize_aggregation_in_order=1) AS p ON equals(p.id, pdi.person_id) - WHERE equals(e.team_id, 99999) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, timestamp, pdi.distinct_id, p.id, p.properties.sneaky_mail - FROM events AS e LEFT JOIN person_distinct_ids AS pdi ON equals(pdi.distinct_id, e.distinct_id) LEFT JOIN persons AS p ON equals(p.id, pdi.person_id) - LIMIT 100 - ''' -# --- # name: TestQuery.test_query_person_distinct_ids ''' -- ClickHouse @@ -1650,36 +930,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_query_select_person_with_joins_without_poe[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event, toTimeZone(events.timestamp, %(hogql_val_3)s) AS timestamp, events__person.id AS id, events__person.properties___sneaky_mail AS sneaky_mail - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) LEFT JOIN ( - SELECT person.id AS id, replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___sneaky_mail - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), ( - SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_1)s), person.version), plus(now64(6, %(hogql_val_2)s), toIntervalDay(1))))))) - SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) - WHERE equals(events.team_id, 99999) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, timestamp, person.id, person.properties.sneaky_mail - FROM events - LIMIT 10 - ''' -# --- # name: TestQuery.test_query_select_person_with_poe_without_joins ''' -- ClickHouse @@ -1695,21 +945,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_query_select_person_with_poe_without_joins[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event, toTimeZone(events.timestamp, %(hogql_val_0)s) AS timestamp, events.person_id AS id, if(notEquals(toJSONString(events.person_properties.^sneaky_mail), '{}'), toJSONString(events.person_properties.^sneaky_mail), if(isNull(events.person_properties.sneaky_mail), NULL, if(startsWith(dynamicType(events.person_properties.sneaky_mail), 'DateTime'), replaceOne(toString(events.person_properties.sneaky_mail), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.sneaky_mail), 'Array'), startsWith(dynamicType(events.person_properties.sneaky_mail), 'Map'), startsWith(dynamicType(events.person_properties.sneaky_mail), 'Tuple')), toJSONString(events.person_properties.sneaky_mail), toString(events.person_properties.sneaky_mail))))) AS sneaky_mail - FROM events_json AS events - WHERE equals(events.team_id, 99999) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event, timestamp, person.id, person.properties.sneaky_mail - FROM events - LIMIT 10 - ''' -# --- # name: TestQuery.test_select_person_on_events ''' -- ClickHouse @@ -1727,23 +962,6 @@ LIMIT 10 ''' # --- -# name: TestQuery.test_select_person_on_events[new_events_schema] - ''' - -- ClickHouse - SELECT if(notEquals(toJSONString(s.person_properties.^sneaky_mail), '{}'), toJSONString(s.person_properties.^sneaky_mail), if(isNull(s.person_properties.sneaky_mail), NULL, if(startsWith(dynamicType(s.person_properties.sneaky_mail), 'DateTime'), replaceOne(toString(s.person_properties.sneaky_mail), ' ', 'T'), if(or(startsWith(dynamicType(s.person_properties.sneaky_mail), 'Array'), startsWith(dynamicType(s.person_properties.sneaky_mail), 'Map'), startsWith(dynamicType(s.person_properties.sneaky_mail), 'Tuple')), toJSONString(s.person_properties.sneaky_mail), toString(s.person_properties.sneaky_mail))))) AS sneaky_mail, count() AS `count()` - FROM events_json AS s - WHERE equals(s.team_id, 99999) - GROUP BY if(notEquals(toJSONString(s.person_properties.^sneaky_mail), '{}'), toJSONString(s.person_properties.^sneaky_mail), if(isNull(s.person_properties.sneaky_mail), NULL, if(startsWith(dynamicType(s.person_properties.sneaky_mail), 'DateTime'), replaceOne(toString(s.person_properties.sneaky_mail), ' ', 'T'), if(or(startsWith(dynamicType(s.person_properties.sneaky_mail), 'Array'), startsWith(dynamicType(s.person_properties.sneaky_mail), 'Map'), startsWith(dynamicType(s.person_properties.sneaky_mail), 'Tuple')), toJSONString(s.person_properties.sneaky_mail), toString(s.person_properties.sneaky_mail))))) - LIMIT 10 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT poe.properties.sneaky_mail, count() - FROM events AS s - GROUP BY poe.properties.sneaky_mail - LIMIT 10 - ''' -# --- # name: TestQuery.test_subquery ''' -- ClickHouse @@ -1768,30 +986,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_subquery[new_events_schema] - ''' - -- ClickHouse - SELECT cnt AS cnt, event AS event - FROM ( - SELECT count() AS cnt, events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.properties.^random_uuid), '{}'), toJSONString(events.properties.^random_uuid), if(isNull(events.properties.random_uuid), NULL, if(startsWith(dynamicType(events.properties.random_uuid), 'DateTime'), replaceOne(toString(events.properties.random_uuid), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.random_uuid), 'Array'), startsWith(dynamicType(events.properties.random_uuid), 'Map'), startsWith(dynamicType(events.properties.random_uuid), 'Tuple')), toJSONString(events.properties.random_uuid), toString(events.properties.random_uuid))))), %(hogql_val_0)s), 0)) - GROUP BY events.event) - GROUP BY cnt, event - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT cnt, event - FROM ( - SELECT count() AS cnt, event - FROM events - WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') - GROUP BY event) - GROUP BY cnt, event - LIMIT 100 - ''' -# --- # name: TestQuery.test_subquery_alias ''' -- ClickHouse @@ -1816,30 +1010,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_subquery_alias[new_events_schema] - ''' - -- ClickHouse - SELECT c.cnt AS cnt, c.event AS event - FROM ( - SELECT count(*) AS cnt, events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), ifNull(equals(if(notEquals(toJSONString(events.properties.^random_uuid), '{}'), toJSONString(events.properties.^random_uuid), if(isNull(events.properties.random_uuid), NULL, if(startsWith(dynamicType(events.properties.random_uuid), 'DateTime'), replaceOne(toString(events.properties.random_uuid), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.random_uuid), 'Array'), startsWith(dynamicType(events.properties.random_uuid), 'Map'), startsWith(dynamicType(events.properties.random_uuid), 'Tuple')), toJSONString(events.properties.random_uuid), toString(events.properties.random_uuid))))), %(hogql_val_0)s), 0)) - GROUP BY events.event) AS c - GROUP BY c.cnt, c.event - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT cnt, event - FROM ( - SELECT count(*) AS cnt, event - FROM events - WHERE equals(properties.random_uuid, 'RANDOM_TEST_ID::UUID') - GROUP BY event) AS c - GROUP BY cnt, event - LIMIT 100 - ''' -# --- # name: TestQuery.test_tuple_access ''' -- ClickHouse @@ -1871,37 +1041,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_tuple_access[new_events_schema] - ''' - -- ClickHouse - SELECT col_a AS col_a, arrayZip((sumMap((g).1, (g).2) AS x).1, x.2) AS r - FROM ( - SELECT col_a AS col_a, groupArray(tuple(col_b, col_c)) AS g - FROM ( - SELECT if(notEquals(toJSONString(events.properties.^index), '{}'), toJSONString(events.properties.^index), if(isNull(events.properties.index), NULL, if(startsWith(dynamicType(events.properties.index), 'DateTime'), replaceOne(toString(events.properties.index), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.index), 'Array'), startsWith(dynamicType(events.properties.index), 'Map'), startsWith(dynamicType(events.properties.index), 'Tuple')), toJSONString(events.properties.index), toString(events.properties.index))))) AS col_a, events.event AS col_b, count() AS col_c - FROM events_json AS events - WHERE equals(events.team_id, 99999) - GROUP BY if(notEquals(toJSONString(events.properties.^index), '{}'), toJSONString(events.properties.^index), if(isNull(events.properties.index), NULL, if(startsWith(dynamicType(events.properties.index), 'DateTime'), replaceOne(toString(events.properties.index), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.index), 'Array'), startsWith(dynamicType(events.properties.index), 'Map'), startsWith(dynamicType(events.properties.index), 'Tuple')), toJSONString(events.properties.index), toString(events.properties.index))))), events.event) - GROUP BY col_a) - GROUP BY col_a - ORDER BY col_a ASC - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT col_a, arrayZip((sumMap((g).1, (g).2) AS x).1, x.2) AS r - FROM ( - SELECT col_a, groupArray(tuple(col_b, col_c)) AS g - FROM ( - SELECT properties.index AS col_a, event AS col_b, count() AS col_c - FROM events - GROUP BY properties.index, event) - GROUP BY col_a) - GROUP BY col_a - ORDER BY col_a ASC - LIMIT 100 - ''' -# --- # name: TestQuery.test_with_pivot_table_1_level ''' -- ClickHouse @@ -1939,43 +1078,6 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_with_pivot_table_1_level[new_events_schema] - ''' - -- ClickHouse - WITH PIVOT_TABLE_COL_ABC AS ( - SELECT if(notEquals(toJSONString(events.properties.^index), '{}'), toJSONString(events.properties.^index), if(isNull(events.properties.index), NULL, if(startsWith(dynamicType(events.properties.index), 'DateTime'), replaceOne(toString(events.properties.index), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.index), 'Array'), startsWith(dynamicType(events.properties.index), 'Map'), startsWith(dynamicType(events.properties.index), 'Tuple')), toJSONString(events.properties.index), toString(events.properties.index))))) AS col_a, events.event AS col_b, count() AS col_c - FROM events_json AS events - WHERE equals(events.team_id, 99999) - GROUP BY if(notEquals(toJSONString(events.properties.^index), '{}'), toJSONString(events.properties.^index), if(isNull(events.properties.index), NULL, if(startsWith(dynamicType(events.properties.index), 'DateTime'), replaceOne(toString(events.properties.index), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.index), 'Array'), startsWith(dynamicType(events.properties.index), 'Map'), startsWith(dynamicType(events.properties.index), 'Tuple')), toJSONString(events.properties.index), toString(events.properties.index))))), events.event), PIVOT_FUNCTION_1 AS ( - SELECT PIVOT_TABLE_COL_ABC.col_a AS col_a, groupArray(tuple(PIVOT_TABLE_COL_ABC.col_b, PIVOT_TABLE_COL_ABC.col_c)) AS g - FROM PIVOT_TABLE_COL_ABC - GROUP BY PIVOT_TABLE_COL_ABC.col_a), PIVOT_FUNCTION_2 AS ( - SELECT PIVOT_FUNCTION_1.col_a AS col_a, arrayZip((sumMap((PIVOT_FUNCTION_1.g).1, (PIVOT_FUNCTION_1.g).2) AS x).1, x.2) AS r - FROM PIVOT_FUNCTION_1 - GROUP BY PIVOT_FUNCTION_1.col_a) - SELECT PIVOT_FUNCTION_2.col_a AS col_a, PIVOT_FUNCTION_2.r AS r - FROM PIVOT_FUNCTION_2 - ORDER BY PIVOT_FUNCTION_2.col_a ASC - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - WITH PIVOT_TABLE_COL_ABC AS ( - SELECT properties.index AS col_a, event AS col_b, count() AS col_c - FROM events - GROUP BY properties.index, event), PIVOT_FUNCTION_1 AS ( - SELECT col_a, groupArray(tuple(col_b, col_c)) AS g - FROM PIVOT_TABLE_COL_ABC - GROUP BY col_a), PIVOT_FUNCTION_2 AS ( - SELECT col_a, arrayZip((sumMap((g).1, (g).2) AS x).1, x.2) AS r - FROM PIVOT_FUNCTION_1 - GROUP BY col_a) - SELECT col_a, r - FROM PIVOT_FUNCTION_2 - ORDER BY col_a ASC - LIMIT 100 - ''' -# --- # name: TestQuery.test_with_pivot_table_2_levels ''' -- ClickHouse @@ -2017,44 +1119,3 @@ LIMIT 100 ''' # --- -# name: TestQuery.test_with_pivot_table_2_levels[new_events_schema] - ''' - -- ClickHouse - WITH PIVOT_TABLE_COL_ABC AS ( - SELECT if(notEquals(toJSONString(events.properties.^index), '{}'), toJSONString(events.properties.^index), if(isNull(events.properties.index), NULL, if(startsWith(dynamicType(events.properties.index), 'DateTime'), replaceOne(toString(events.properties.index), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.index), 'Array'), startsWith(dynamicType(events.properties.index), 'Map'), startsWith(dynamicType(events.properties.index), 'Tuple')), toJSONString(events.properties.index), toString(events.properties.index))))) AS col_a, events.event AS col_b, count() AS col_c - FROM events_json AS events - WHERE equals(events.team_id, 99999) - GROUP BY if(notEquals(toJSONString(events.properties.^index), '{}'), toJSONString(events.properties.^index), if(isNull(events.properties.index), NULL, if(startsWith(dynamicType(events.properties.index), 'DateTime'), replaceOne(toString(events.properties.index), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.index), 'Array'), startsWith(dynamicType(events.properties.index), 'Map'), startsWith(dynamicType(events.properties.index), 'Tuple')), toJSONString(events.properties.index), toString(events.properties.index))))), events.event), PIVOT_FUNCTION_1 AS ( - SELECT PIVOT_TABLE_COL_ABC.col_a AS col_a, groupArray(tuple(PIVOT_TABLE_COL_ABC.col_b, PIVOT_TABLE_COL_ABC.col_c)) AS g - FROM PIVOT_TABLE_COL_ABC - GROUP BY PIVOT_TABLE_COL_ABC.col_a), PIVOT_FUNCTION_2 AS ( - SELECT PIVOT_FUNCTION_1.col_a AS col_a, arrayZip((sumMap((PIVOT_FUNCTION_1.g).1, (PIVOT_FUNCTION_1.g).2) AS x).1, x.2) AS r - FROM PIVOT_FUNCTION_1 - GROUP BY PIVOT_FUNCTION_1.col_a), final AS ( - SELECT PIVOT_FUNCTION_2.col_a AS col_a, PIVOT_FUNCTION_2.r AS r - FROM PIVOT_FUNCTION_2) - SELECT final.col_a AS col_a, final.r AS r - FROM final - ORDER BY final.col_a ASC - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - WITH PIVOT_TABLE_COL_ABC AS ( - SELECT properties.index AS col_a, event AS col_b, count() AS col_c - FROM events - GROUP BY properties.index, event), PIVOT_FUNCTION_1 AS ( - SELECT col_a, groupArray(tuple(col_b, col_c)) AS g - FROM PIVOT_TABLE_COL_ABC - GROUP BY col_a), PIVOT_FUNCTION_2 AS ( - SELECT col_a, arrayZip((sumMap((g).1, (g).2) AS x).1, x.2) AS r - FROM PIVOT_FUNCTION_1 - GROUP BY col_a), final AS ( - SELECT col_a, r - FROM PIVOT_FUNCTION_2) - SELECT col_a, r - FROM final - ORDER BY col_a ASC - LIMIT 100 - ''' -# --- diff --git a/posthog/hogql/transforms/test/__snapshots__/test_events_predicate_pushdown.ambr b/posthog/hogql/transforms/test/__snapshots__/test_events_predicate_pushdown.ambr index 1af17f61b208..d066a7c7791d 100644 --- a/posthog/hogql/transforms/test/__snapshots__/test_events_predicate_pushdown.ambr +++ b/posthog/hogql/transforms/test/__snapshots__/test_events_predicate_pushdown.ambr @@ -15,22 +15,6 @@ LIMIT 50 ''' # --- -# name: TestEventsPredicatePushdownExecution.test_exec_aliased_events_timestamp_filter_with_session_join[new_events_schema] - ''' - - SELECT e.event AS event, e__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toUInt128(accurateCastOrNull(e.`$session_id`, %(hogql_val_0)s)) AS __pd_expr_0, e.event AS event - FROM events_json AS e - WHERE and(equals(e.team_id, 420), greaterOrEquals(e.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s)), less(e.timestamp, toDateTime64(%(hogql_val_3)s, 6, %(hogql_val_4)s))) - LIMIT 50) AS e LEFT JOIN ( - SELECT dateDiff(%(hogql_val_5)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_8)s, toIntervalDay(3))), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), plus(%(hogql_val_9)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS e__session ON equals(e.__pd_expr_0, e__session.session_id_v7) - LIMIT 50 - ''' -# --- # name: TestEventsPredicatePushdownExecution.test_exec_timestamp_filter_with_session_join ''' @@ -47,22 +31,6 @@ LIMIT 50 ''' # --- -# name: TestEventsPredicatePushdownExecution.test_exec_timestamp_filter_with_session_join[new_events_schema] - ''' - - SELECT events.event AS event, events__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_0)s)) AS __pd_expr_0, events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s)), less(events.timestamp, toDateTime64(%(hogql_val_3)s, 6, %(hogql_val_4)s))) - LIMIT 50) AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_5)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_8)s, toIntervalDay(3))), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), plus(%(hogql_val_9)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(events.__pd_expr_0, events__session.session_id_v7) - LIMIT 50 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_bare_timestamp_with_select_alias_pushes_down ''' @@ -79,22 +47,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_bare_timestamp_with_select_alias_pushes_down[new_events_schema] - ''' - - SELECT events.event AS event, events.__pd_expr_0 AS timestamp, events__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toTimeZone(toTimeZone(events.timestamp, %(hogql_val_0)s), %(hogql_val_1)s) AS __pd_expr_0, toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_2)s)) AS __pd_expr_1, events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(toTimeZone(toTimeZone(events.timestamp, %(hogql_val_3)s), %(hogql_val_4)s), %(hogql_val_5)s), lessOrEquals(toTimeZone(toTimeZone(events.timestamp, %(hogql_val_6)s), %(hogql_val_7)s), today())) - LIMIT 50000) AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_8)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_9)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_10)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_11)s, toIntervalDay(3))), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), plus(today(), toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(events.__pd_expr_1, events__session.session_id_v7) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_events_with_alias_and_session_join ''' @@ -111,22 +63,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_events_with_alias_and_session_join[new_events_schema] - ''' - - SELECT e.event AS event, e__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toUInt128(accurateCastOrNull(e.`$session_id`, %(hogql_val_0)s)) AS __pd_expr_0, e.event AS event - FROM events_json AS e - WHERE and(equals(e.team_id, 420), greaterOrEquals(e.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000) AS e LEFT JOIN ( - SELECT dateDiff(%(hogql_val_3)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_4)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_5)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_6)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS e__session ON equals(e.__pd_expr_0, e__session.session_id_v7) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_events_with_session_join_and_timestamp_filter ''' @@ -143,22 +79,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_events_with_session_join_and_timestamp_filter[new_events_schema] - ''' - - SELECT events.event AS event, events__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_0)s)) AS __pd_expr_0, events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000) AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_3)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_4)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_5)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_6)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(events.__pd_expr_0, events__session.session_id_v7) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_events_without_join_no_pushdown ''' @@ -168,15 +88,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_events_without_join_no_pushdown[new_events_schema] - ''' - - SELECT events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s))) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_events_without_where_no_pushdown ''' @@ -190,19 +101,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_events_without_where_no_pushdown[new_events_schema] - ''' - - SELECT events.event AS event, events__session.`$session_duration` AS `$session_duration` - FROM events_json AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_0)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_1)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_2)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE equals(raw_sessions.team_id, 420) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_3)s)), events__session.session_id_v7) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_multiple_poe_fields_with_session_join ''' @@ -219,22 +117,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_multiple_poe_fields_with_session_join[new_events_schema] - ''' - - SELECT events.event AS event, events.person_id AS id, events.person_properties AS properties, events.__pd_expr_0 AS created_at, events__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toTimeZone(events.person_created_at, %(hogql_val_0)s) AS __pd_expr_0, toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_1)s)) AS __pd_expr_1, events.event AS event, events.person_id AS person_id, concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null', JSONExtractKeysAndValuesRaw(toJSONString(events.person_properties)))), ','), '}') AS person_properties - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_2)s, 6, %(hogql_val_3)s))) - LIMIT 50000) AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_4)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_5)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_6)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_7)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(events.__pd_expr_1, events__session.session_id_v7) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_multiple_pushable_predicates ''' @@ -251,22 +133,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_multiple_pushable_predicates[new_events_schema] - ''' - - SELECT events.event AS event, events__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_0)s)) AS __pd_expr_0, events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s)), equals(events.event, %(hogql_val_3)s)) - LIMIT 50000) AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_4)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_5)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_6)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_7)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(events.__pd_expr_0, events__session.session_id_v7) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_0_explicit_join ''' @@ -280,19 +146,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_0_explicit_join[new_events_schema] - ''' - - SELECT sessions.session_id AS session_id, events.uuid AS uuid - FROM events_json AS events JOIN ( - SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_0)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS sessions ON equals(events.`$session_id`, sessions.session_id) - WHERE and(equals(events.team_id, 420), greater(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_1_inner_join ''' @@ -306,19 +159,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_1_inner_join[new_events_schema] - ''' - - SELECT sessions.session_id AS session_id, events.uuid AS uuid - FROM events_json AS events INNER JOIN ( - SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_0)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS sessions ON equals(events.`$session_id`, sessions.session_id) - WHERE and(equals(events.team_id, 420), greater(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_2_cross_join ''' @@ -332,19 +172,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_2_cross_join[new_events_schema] - ''' - - SELECT sessions.session_id AS session_id, events.uuid AS uuid - FROM events_json AS events CROSS JOIN ( - SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_0)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS sessions - WHERE and(equals(events.team_id, 420), greater(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_3_right_join ''' @@ -358,19 +185,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_3_right_join[new_events_schema] - ''' - - SELECT sessions.session_id AS session_id, events.uuid AS uuid - FROM events_json AS events RIGHT JOIN ( - SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_0)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS sessions ON equals(events.`$session_id`, sessions.session_id) - WHERE and(equals(events.team_id, 420), greater(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_4_right_outer_join ''' @@ -384,19 +198,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_4_right_outer_join[new_events_schema] - ''' - - SELECT sessions.session_id AS session_id, events.uuid AS uuid - FROM events_json AS events RIGHT OUTER JOIN ( - SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_0)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS sessions ON equals(events.`$session_id`, sessions.session_id) - WHERE and(equals(events.team_id, 420), greater(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_5_full_outer_join ''' @@ -410,19 +211,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_5_full_outer_join[new_events_schema] - ''' - - SELECT sessions.session_id AS session_id, events.uuid AS uuid - FROM events_json AS events FULL OUTER JOIN ( - SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_0)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS sessions ON equals(events.`$session_id`, sessions.session_id) - WHERE and(equals(events.team_id, 420), greater(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_6_full_join ''' @@ -436,19 +224,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_non_row_preserving_join_skips_pushdown_6_full_join[new_events_schema] - ''' - - SELECT sessions.session_id AS session_id, events.uuid AS uuid - FROM events_json AS events FULL JOIN ( - SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_0)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS sessions ON equals(events.`$session_id`, sessions.session_id) - WHERE and(equals(events.team_id, 420), greater(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_poe_created_at_with_session_join ''' @@ -465,22 +240,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_poe_created_at_with_session_join[new_events_schema] - ''' - - SELECT events.event AS event, events.__pd_expr_0 AS created_at, events__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toTimeZone(events.person_created_at, %(hogql_val_0)s) AS __pd_expr_0, toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_1)s)) AS __pd_expr_1, events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_2)s, 6, %(hogql_val_3)s))) - LIMIT 50000) AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_4)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_5)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_6)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_7)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(events.__pd_expr_1, events__session.session_id_v7) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_poe_id_with_session_join ''' @@ -497,22 +256,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_poe_id_with_session_join[new_events_schema] - ''' - - SELECT events.person_id AS id, events__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_0)s)) AS __pd_expr_0, events.person_id AS person_id - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000) AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_3)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_4)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_5)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_6)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(events.__pd_expr_0, events__session.session_id_v7) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_poe_properties_with_session_join ''' @@ -529,22 +272,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_poe_properties_with_session_join[new_events_schema] - ''' - - SELECT events.event AS event, events.person_properties AS properties, events__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_0)s)) AS __pd_expr_0, events.event AS event, concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null', JSONExtractKeysAndValuesRaw(toJSONString(events.person_properties)))), ','), '}') AS person_properties - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000) AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_3)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_4)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_5)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_6)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(events.__pd_expr_0, events__session.session_id_v7) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_row_preserving_join_pushes_timestamp_down_0_left_join ''' @@ -561,22 +288,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_row_preserving_join_pushes_timestamp_down_0_left_join[new_events_schema] - ''' - - SELECT sessions.session_id AS session_id, events.uuid AS uuid - FROM ( - SELECT events.`$session_id` AS `$session_id`, events.uuid AS uuid - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greater(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s))) - LIMIT 50000) AS events LEFT JOIN ( - SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_2)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS sessions ON equals(events.`$session_id`, sessions.session_id) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_row_preserving_join_pushes_timestamp_down_1_left_outer_join ''' @@ -593,22 +304,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_row_preserving_join_pushes_timestamp_down_1_left_outer_join[new_events_schema] - ''' - - SELECT sessions.session_id AS session_id, events.uuid AS uuid - FROM ( - SELECT events.`$session_id` AS `$session_id`, events.uuid AS uuid - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greater(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s))) - LIMIT 50000) AS events LEFT OUTER JOIN ( - SELECT toString(reinterpretAsUUID(bitOr(bitShiftLeft(raw_sessions.session_id_v7, 64), bitShiftRight(raw_sessions.session_id_v7, 64)))) AS session_id, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_2)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS sessions ON equals(events.`$session_id`, sessions.session_id) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_session_duration_filter_declines ''' @@ -622,19 +317,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_session_duration_filter_declines[new_events_schema] - ''' - - SELECT events.event AS event, events__session.`$session_duration` AS `$session_duration` - FROM events_json AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_0)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_1)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_2)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_3)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_4)s)), events__session.session_id_v7) - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_5)s, 6, %(hogql_val_6)s)), greater(events__session.`$session_duration`, 0)) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_simple_events_with_person_join ''' @@ -659,30 +341,6 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_simple_events_with_person_join[new_events_schema] - ''' - - SELECT events.event AS event, events__person.id AS id - FROM ( - SELECT events.distinct_id AS distinct_id, events.event AS event, events.person_id AS person_id - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greater(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s))) - LIMIT 50000) AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 420) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) LEFT JOIN ( - SELECT person.id AS id - FROM person - WHERE equals(person.team_id, 420) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_2)s), person.version), plus(now64(6, %(hogql_val_3)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) - LIMIT 50000 - ''' -# --- # name: TestEventsPredicatePushdownTransform.test_subquery_with_pushdown ''' @@ -703,23 +361,3 @@ LIMIT 50000 ''' # --- -# name: TestEventsPredicatePushdownTransform.test_subquery_with_pushdown[new_events_schema] - ''' - - SELECT event AS event, avg(`$session_duration`) AS `avg($session_duration)` - FROM ( - SELECT events.event AS event, events__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_0)s)) AS __pd_expr_0, events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s)), or(equals(events.event, %(hogql_val_3)s), equals(events.event, %(hogql_val_4)s))) - LIMIT 100) AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_5)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_8)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(events.__pd_expr_0, events__session.session_id_v7) - LIMIT 100) - GROUP BY event - LIMIT 50000 - ''' -# --- diff --git a/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr index 89fdff689fe9..5d3a2ce1793c 100644 --- a/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr +++ b/posthog/hogql/transforms/test/__snapshots__/test_in_cohort.ambr @@ -21,28 +21,6 @@ LIMIT 100 ''' # --- -# name: TestInCohort.test_in_cohort_conjoined_dynamic[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT cohortpeople.person_id AS cohort_person_id, 1 AS matched, cohortpeople.cohort_id AS cohort_id - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 99999), equals(cohortpeople.cohort_id, XX), equals(cohortpeople.version, 0))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(events.event, %(hogql_val_0)s), equals(__in_cohort.matched, 1)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id - FROM raw_cohort_people - WHERE and(equals(cohort_id, XX), equals(version, 0))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) - WHERE and(and(equals(1, 1), equals(event, 'RANDOM_TEST_ID::UUID')), equals(__in_cohort.matched, 1)) - LIMIT 100 - ''' -# --- # name: TestInCohort.test_in_cohort_conjoined_int ''' -- ClickHouse @@ -65,28 +43,6 @@ LIMIT 100 ''' # --- -# name: TestInCohort.test_in_cohort_conjoined_int[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT person_static_cohort.person_id AS cohort_person_id, 1 AS matched, person_static_cohort.cohort_id AS cohort_id - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), in(person_static_cohort.cohort_id, [1, 2, 3, 4, 5 /* ... */]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(__in_cohort.matched, 1)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id - FROM static_cohort_people - WHERE in(cohort_id, [1, 2, 3, 4, 5 /* ... */])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) - WHERE and(equals(1, 1), equals(__in_cohort.matched, 1)) - LIMIT 100 - ''' -# --- # name: TestInCohort.test_in_cohort_conjoined_string ''' -- ClickHouse @@ -109,28 +65,6 @@ LIMIT 100 ''' # --- -# name: TestInCohort.test_in_cohort_conjoined_string[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT person_static_cohort.person_id AS cohort_person_id, 1 AS matched, person_static_cohort.cohort_id AS cohort_id - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), in(person_static_cohort.cohort_id, [1, 2, 3, 4, 5 /* ... */]))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(__in_cohort.matched, 1)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id - FROM static_cohort_people - WHERE in(cohort_id, [1, 2, 3, 4, 5 /* ... */])) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) - WHERE and(equals(1, 1), equals(__in_cohort.matched, 1)) - LIMIT 100 - ''' -# --- # name: TestInCohort.test_in_cohort_deleted ''' -- ClickHouse @@ -153,28 +87,6 @@ LIMIT 100 ''' # --- -# name: TestInCohort.test_in_cohort_deleted[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT person_static_cohort.person_id AS person_id, 1 AS matched - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), equals(person_static_cohort.cohort_id, XX))) AS in_cohort__XX ON equals(in_cohort__XX.person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(in_cohort__XX.matched, 1)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id, 1 AS matched - FROM static_cohort_people - WHERE equals(cohort_id, XX)) AS in_cohort__XX ON equals(in_cohort__XX.person_id, person_id) - WHERE equals(in_cohort__XX.matched, 1) - LIMIT 100 - ''' -# --- # name: TestInCohort.test_in_cohort_dynamic ''' -- ClickHouse @@ -201,32 +113,6 @@ LIMIT 100 ''' # --- -# name: TestInCohort.test_in_cohort_dynamic[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT cohortpeople.person_id AS person_id, 1 AS matched - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 99999), equals(cohortpeople.cohort_id, XX)) - GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version - HAVING greater(sum(cohortpeople.sign), 0)) AS in_cohort__XX ON equals(in_cohort__XX.person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(in_cohort__XX.matched, 1), equals(events.event, %(hogql_val_0)s)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id, 1 AS matched - FROM raw_cohort_people - WHERE equals(cohort_id, XX) - GROUP BY person_id, cohort_id, version - HAVING greater(sum(sign), 0)) AS in_cohort__XX ON equals(in_cohort__XX.person_id, person_id) - WHERE and(equals(in_cohort__XX.matched, 1), equals(event, 'RANDOM_TEST_ID::UUID')) - LIMIT 100 - ''' -# --- # name: TestInCohort.test_in_cohort_same_cohort_referenced_twice ''' -- ClickHouse @@ -253,32 +139,6 @@ LIMIT 100 ''' # --- -# name: TestInCohort.test_in_cohort_same_cohort_referenced_twice[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT cohortpeople.person_id AS person_id, 1 AS matched - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 99999), equals(cohortpeople.cohort_id, XX)) - GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version - HAVING greater(sum(cohortpeople.sign), 0)) AS in_cohort__XX ON equals(in_cohort__XX.person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(in_cohort__XX.matched, 1), equals(events.event, %(hogql_val_0)s), equals(in_cohort__XX.matched, 1)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id, 1 AS matched - FROM raw_cohort_people - WHERE equals(cohort_id, XX) - GROUP BY person_id, cohort_id, version - HAVING greater(sum(sign), 0)) AS in_cohort__XX ON equals(in_cohort__XX.person_id, person_id) - WHERE and(equals(in_cohort__XX.matched, 1), equals(event, 'RANDOM_TEST_ID::UUID'), equals(in_cohort__XX.matched, 1)) - LIMIT 100 - ''' -# --- # name: TestInCohort.test_in_cohort_static ''' -- ClickHouse @@ -301,28 +161,6 @@ LIMIT 100 ''' # --- -# name: TestInCohort.test_in_cohort_static[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT person_static_cohort.person_id AS person_id, 1 AS matched - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), equals(person_static_cohort.cohort_id, XX))) AS in_cohort__XX ON equals(in_cohort__XX.person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(in_cohort__XX.matched, 1)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id, 1 AS matched - FROM static_cohort_people - WHERE equals(cohort_id, XX)) AS in_cohort__XX ON equals(in_cohort__XX.person_id, person_id) - WHERE equals(in_cohort__XX.matched, 1) - LIMIT 100 - ''' -# --- # name: TestInCohort.test_in_cohort_strings ''' -- ClickHouse @@ -345,28 +183,6 @@ LIMIT 100 ''' # --- -# name: TestInCohort.test_in_cohort_strings[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT person_static_cohort.person_id AS person_id, 1 AS matched - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), equals(person_static_cohort.cohort_id, XX))) AS in_cohort__XX ON equals(in_cohort__XX.person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(in_cohort__XX.matched, 1)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id, 1 AS matched - FROM static_cohort_people - WHERE equals(cohort_id, XX)) AS in_cohort__XX ON equals(in_cohort__XX.person_id, person_id) - WHERE equals(in_cohort__XX.matched, 1) - LIMIT 100 - ''' -# --- # name: TestInlineCohortLeftjoin.test_inline_conjoined_mixed_static_and_dynamic ''' -- ClickHouse @@ -410,49 +226,6 @@ LIMIT 100 ''' # --- -# name: TestInlineCohortLeftjoin.test_inline_conjoined_mixed_static_and_dynamic[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT person_static_cohort.person_id AS cohort_person_id, 1 AS matched, person_static_cohort.cohort_id AS cohort_id - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), in(person_static_cohort.cohort_id, [1, 2, 3, 4, 5 /* ... */])) UNION ALL - SELECT id AS cohort_person_id, 1 AS matched, 99999 AS cohort_id - FROM (( - SELECT persons.id AS id - FROM ( - SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '')), person.version), 1) AS properties___test_prop, person.id AS id - FROM person - WHERE and(equals(person.team_id, 99999), in(id, ( - SELECT where_optimization.id AS id - FROM person AS where_optimization - WHERE and(equals(where_optimization.team_id, 99999), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(where_optimization.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), %(hogql_val_2)s), 0))))) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_3)s), person.version), plus(now64(6, %(hogql_val_4)s), toIntervalDay(1))))) AS persons - WHERE ifNull(equals(persons.properties___test_prop, %(hogql_val_5)s), 0) - ORDER BY persons.id ASC - SETTINGS optimize_aggregation_in_order=1, join_algorithm='auto'))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(events.event, %(hogql_val_6)s), equals(__in_cohort.matched, 1)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id - FROM static_cohort_people - WHERE in(cohort_id, [1, 2, 3, 4, 5 /* ... */]) UNION ALL - SELECT id AS cohort_person_id, 1 AS matched, 99999 AS cohort_id - FROM (( - SELECT id - FROM persons - WHERE equals(properties.test_prop, 'RANDOM_TEST_ID::UUID') - ORDER BY id ASC))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) - WHERE and(and(or(equals(1, 1), equals(1, 1)), equals(event, 'RANDOM_TEST_ID::UUID')), equals(__in_cohort.matched, 1)) - LIMIT 100 - ''' -# --- # name: TestInlineCohortLeftjoin.test_inline_conjoined_off_vs_always ''' -- ClickHouse @@ -533,86 +306,6 @@ LIMIT 100 ''' # --- -# name: TestInlineCohortLeftjoin.test_inline_conjoined_off_vs_always[new_events_schema.1] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT id AS cohort_person_id, 1 AS matched, 99999 AS cohort_id - FROM (( - SELECT persons.id AS id - FROM ( - SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '')), person.version), 1) AS properties___test_prop, person.id AS id - FROM person - WHERE and(equals(person.team_id, 99999), in(id, ( - SELECT where_optimization.id AS id - FROM person AS where_optimization - WHERE and(equals(where_optimization.team_id, 99999), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(where_optimization.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), %(hogql_val_2)s), 0))))) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_3)s), person.version), plus(now64(6, %(hogql_val_4)s), toIntervalDay(1))))) AS persons - WHERE ifNull(equals(persons.properties___test_prop, %(hogql_val_5)s), 0) - ORDER BY persons.id ASC - SETTINGS optimize_aggregation_in_order=1, join_algorithm='auto')) UNION ALL - SELECT id AS cohort_person_id, 1 AS matched, 99999 AS cohort_id - FROM (( - SELECT persons.id AS id - FROM ( - SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_6)s), ''), 'null'), '^"|"$', '')), person.version), 1) AS properties___test_prop, person.id AS id - FROM person - WHERE and(equals(person.team_id, 99999), in(id, ( - SELECT where_optimization.id AS id - FROM person AS where_optimization - WHERE and(equals(where_optimization.team_id, 99999), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(where_optimization.properties, %(hogql_val_7)s), ''), 'null'), '^"|"$', ''), %(hogql_val_8)s), 0))))) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_9)s), person.version), plus(now64(6, %(hogql_val_10)s), toIntervalDay(1))))) AS persons - WHERE ifNull(equals(persons.properties___test_prop, %(hogql_val_11)s), 0) - ORDER BY persons.id ASC - SETTINGS optimize_aggregation_in_order=1, join_algorithm='auto'))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(events.event, %(hogql_val_12)s), equals(__in_cohort.matched, 1)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT id AS cohort_person_id, 1 AS matched, 99999 AS cohort_id - FROM (( - SELECT id - FROM persons - WHERE equals(properties.test_prop, 'RANDOM_TEST_ID::UUID') - ORDER BY id ASC)) UNION ALL - SELECT id AS cohort_person_id, 1 AS matched, 99999 AS cohort_id - FROM (( - SELECT id - FROM persons - WHERE equals(properties.test_prop, 'RANDOM_TEST_ID::UUID') - ORDER BY id ASC))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) - WHERE and(and(or(equals(1, 1), equals(1, 1)), equals(event, 'RANDOM_TEST_ID::UUID')), equals(__in_cohort.matched, 1)) - LIMIT 100 - ''' -# --- -# name: TestInlineCohortLeftjoin.test_inline_conjoined_off_vs_always[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT cohortpeople.person_id AS cohort_person_id, 1 AS matched, cohortpeople.cohort_id AS cohort_id - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 99999), or(and(equals(cohortpeople.cohort_id, XX), equals(cohortpeople.version, 0)), and(equals(cohortpeople.cohort_id, XX), equals(cohortpeople.version, 0))))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(events.event, %(hogql_val_0)s), equals(__in_cohort.matched, 1)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id AS cohort_person_id, 1 AS matched, cohort_id - FROM raw_cohort_people - WHERE or(and(equals(cohort_id, XX), equals(version, 0)), and(equals(cohort_id, XX), equals(version, 0)))) AS __in_cohort ON equals(__in_cohort.cohort_person_id, person_id) - WHERE and(and(or(equals(1, 1), equals(1, 1)), equals(event, 'RANDOM_TEST_ID::UUID')), equals(__in_cohort.matched, 1)) - LIMIT 100 - ''' -# --- # name: TestInlineCohortLeftjoin.test_inline_leftjoin_off_vs_always ''' -- ClickHouse @@ -684,77 +377,6 @@ LIMIT 100 ''' # --- -# name: TestInlineCohortLeftjoin.test_inline_leftjoin_off_vs_always[new_events_schema.1] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT id AS person_id, 1 AS matched - FROM (( - SELECT persons.id AS id - FROM ( - SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '')), person.version), 1) AS properties___test_prop, person.id AS id - FROM person - WHERE and(equals(person.team_id, 99999), in(id, ( - SELECT where_optimization.id AS id - FROM person AS where_optimization - WHERE and(equals(where_optimization.team_id, 99999), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(where_optimization.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', ''), %(hogql_val_2)s), 0))))) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_3)s), person.version), plus(now64(6, %(hogql_val_4)s), toIntervalDay(1))))) AS persons - WHERE ifNull(equals(persons.properties___test_prop, %(hogql_val_5)s), 0) - ORDER BY persons.id ASC - SETTINGS optimize_aggregation_in_order=1, join_algorithm='auto'))) AS in_cohort__XX ON equals(in_cohort__XX.person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(in_cohort__XX.matched, 1), equals(events.event, %(hogql_val_6)s)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT id AS person_id, 1 AS matched - FROM (( - SELECT id - FROM ( - SELECT tupleElement(argMax(tuple(raw_persons.properties.test_prop), raw_persons.version), 1) AS properties___test_prop, raw_persons.id AS id - FROM raw_persons - WHERE in(id, ( - SELECT id - FROM raw_persons AS where_optimization - WHERE equals(properties.test_prop, 'RANDOM_TEST_ID::UUID'))) - GROUP BY raw_persons.id - HAVING and(equals(tupleElement(argMax(tuple(raw_persons.is_deleted), raw_persons.version), 1), 0), less(tupleElement(argMax(tuple(raw_persons.created_at), raw_persons.version), 1), plus(now(), toIntervalDay(1))))) AS persons - WHERE equals(properties.test_prop, 'RANDOM_TEST_ID::UUID') - ORDER BY id ASC))) AS in_cohort__XX ON equals(in_cohort__XX.person_id, person_id) - WHERE and(equals(in_cohort__XX.matched, 1), equals(event, 'RANDOM_TEST_ID::UUID')) - LIMIT 100 - ''' -# --- -# name: TestInlineCohortLeftjoin.test_inline_leftjoin_off_vs_always[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT cohortpeople.person_id AS person_id, 1 AS matched - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 99999), equals(cohortpeople.cohort_id, XX)) - GROUP BY cohortpeople.person_id, cohortpeople.cohort_id, cohortpeople.version - HAVING greater(sum(cohortpeople.sign), 0)) AS in_cohort__XX ON equals(in_cohort__XX.person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(in_cohort__XX.matched, 1), equals(events.event, %(hogql_val_0)s)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id, 1 AS matched - FROM raw_cohort_people - WHERE equals(cohort_id, XX) - GROUP BY person_id, cohort_id, version - HAVING greater(sum(sign), 0)) AS in_cohort__XX ON equals(in_cohort__XX.person_id, person_id) - WHERE and(equals(in_cohort__XX.matched, 1), equals(event, 'RANDOM_TEST_ID::UUID')) - LIMIT 100 - ''' -# --- # name: TestInlineCohortLeftjoin.test_inline_static_always_uses_cohortpeople ''' -- ClickHouse @@ -777,25 +399,3 @@ LIMIT 100 ''' # --- -# name: TestInlineCohortLeftjoin.test_inline_static_always_uses_cohortpeople[new_events_schema] - ''' - -- ClickHouse - SELECT events.event AS event - FROM events_json AS events LEFT JOIN ( - SELECT person_static_cohort.person_id AS person_id, 1 AS matched - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), equals(person_static_cohort.cohort_id, XX))) AS in_cohort__XX ON equals(in_cohort__XX.person_id, events.person_id) - WHERE and(equals(events.team_id, 99999), equals(in_cohort__XX.matched, 1), equals(events.event, %(hogql_val_0)s)) - LIMIT 100 - SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 - - -- HogQL - SELECT event - FROM events LEFT JOIN ( - SELECT person_id, 1 AS matched - FROM static_cohort_people - WHERE equals(cohort_id, XX)) AS in_cohort__XX ON equals(in_cohort__XX.person_id, person_id) - WHERE and(equals(in_cohort__XX.matched, 1), equals(event, 'RANDOM_TEST_ID::UUID')) - LIMIT 100 - ''' -# --- diff --git a/posthog/hogql/transforms/test/__snapshots__/test_lazy_tables.ambr b/posthog/hogql/transforms/test/__snapshots__/test_lazy_tables.ambr index ec527e7dbf8c..9dc6b40382cd 100644 --- a/posthog/hogql/transforms/test/__snapshots__/test_lazy_tables.ambr +++ b/posthog/hogql/transforms/test/__snapshots__/test_lazy_tables.ambr @@ -15,22 +15,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_events_session_join_with_multiple_predicates[new_events_schema] - ''' - - SELECT events.event AS event, events__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_0)s)) AS __pd_expr_0, events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s)), less(events.timestamp, toDateTime64(%(hogql_val_3)s, 6, %(hogql_val_4)s)), equals(events.event, %(hogql_val_5)s), isNotNull(events.`$session_id`)) - LIMIT 50000) AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_6)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_7)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_8)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_9)s, toIntervalDay(3))), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), plus(%(hogql_val_10)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(events.__pd_expr_0, events__session.session_id_v7) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_events_session_join_with_session_duration_filter ''' @@ -44,19 +28,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_events_session_join_with_session_duration_filter[new_events_schema] - ''' - - SELECT events.event AS event, events__session.`$session_duration` AS `$session_duration` - FROM events_json AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_0)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_1)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_2)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_3)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_4)s)), events__session.session_id_v7) - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_5)s, 6, %(hogql_val_6)s)), greater(events__session.`$session_duration`, 0)) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_events_session_join_with_timestamp_filter ''' @@ -73,22 +44,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_events_session_join_with_timestamp_filter[new_events_schema] - ''' - - SELECT events.event AS event, events__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toUInt128(accurateCastOrNull(events.`$session_id`, %(hogql_val_0)s)) AS __pd_expr_0, events.event AS event - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s)), less(events.timestamp, toDateTime64(%(hogql_val_3)s, 6, %(hogql_val_4)s))) - LIMIT 50000) AS events LEFT JOIN ( - SELECT dateDiff(%(hogql_val_5)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_6)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_7)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_8)s, toIntervalDay(3))), lessOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), plus(%(hogql_val_9)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS events__session ON equals(events.__pd_expr_0, events__session.session_id_v7) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_events_sessions_join_with_alias ''' @@ -105,22 +60,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_events_sessions_join_with_alias[new_events_schema] - ''' - - SELECT e.event AS event, e__session.`$session_duration` AS `$session_duration` - FROM ( - SELECT toUInt128(accurateCastOrNull(e.`$session_id`, %(hogql_val_0)s)) AS __pd_expr_0, e.event AS event - FROM events_json AS e - WHERE and(equals(e.team_id, 420), greaterOrEquals(e.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s)), equals(e.event, %(hogql_val_3)s)) - LIMIT 50000) AS e LEFT JOIN ( - SELECT dateDiff(%(hogql_val_4)s, min(toTimeZone(raw_sessions.min_timestamp, %(hogql_val_5)s)), max(toTimeZone(raw_sessions.max_timestamp, %(hogql_val_6)s))) AS `$session_duration`, raw_sessions.session_id_v7 AS session_id_v7 - FROM raw_sessions - WHERE and(equals(raw_sessions.team_id, 420), greaterOrEquals(fromUnixTimestamp(intDiv(toUInt64(bitShiftRight(raw_sessions.session_id_v7, 80)), 1000)), minus(%(hogql_val_7)s, toIntervalDay(3)))) - GROUP BY raw_sessions.session_id_v7) AS e__session ON equals(e.__pd_expr_0, e__session.session_id_v7) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_lazy_join_on_lazy_table ''' @@ -157,23 +96,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_lazy_join_on_lazy_table_with_person_properties[new_events_schema] - ''' - - SELECT persons__events.event AS event - FROM ( - SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '')), person.version), 1) AS persons___properties___email, person.id AS id - FROM person - WHERE equals(person.team_id, 420) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_1)s), person.version), plus(now64(6, %(hogql_val_2)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS persons LEFT JOIN ( - SELECT events.event AS event, event AS persons__events___event - FROM events_json AS events - WHERE equals(events.team_id, 420)) AS persons__events ON equals(persons.persons___properties___email, persons__events.persons__events___event) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_lazy_join_on_lazy_table_with_properties ''' @@ -228,27 +150,6 @@ LIMIT 10 ''' # --- -# name: TestLazyJoins.test_resolve_lazy_table_as_table_in_join[new_events_schema] - ''' - - SELECT events.event AS event, events.distinct_id AS distinct_id, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, persons.properties___email AS email - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 420) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) LEFT JOIN ( - SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '')), person.version), 1) AS properties___email, person.id AS id - FROM person - WHERE equals(person.team_id, 420) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_1)s), person.version), plus(now64(6, %(hogql_val_2)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS persons ON equals(persons.id, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id)) - WHERE equals(events.team_id, 420) - LIMIT 10 - ''' -# --- # name: TestLazyJoins.test_resolve_lazy_table_indirect_duplicate_references ''' @@ -270,27 +171,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_resolve_lazy_table_indirect_duplicate_references[new_events_schema] - ''' - - SELECT if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, events__person.properties AS properties - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 420) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) LEFT JOIN ( - SELECT argMax(person.properties, person.version) AS properties, person.id AS id - FROM person - WHERE equals(person.team_id, 420) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_0)s), person.version), plus(now64(6, %(hogql_val_1)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_resolve_lazy_table_indirectly_referenced ''' @@ -312,27 +192,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_resolve_lazy_table_indirectly_referenced[new_events_schema] - ''' - - SELECT events__person.id AS id - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 420) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) LEFT JOIN ( - SELECT person.id AS id - FROM person - WHERE equals(person.team_id, 420) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_0)s), person.version), plus(now64(6, %(hogql_val_1)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_resolve_lazy_tables ''' @@ -348,21 +207,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_resolve_lazy_tables[new_events_schema] - ''' - - SELECT events.event AS event, events__pdi.person_id AS person_id - FROM events_json AS events INNER JOIN ( - SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 420) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_resolve_lazy_tables_one_level_properties ''' @@ -418,21 +262,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_resolve_lazy_tables_traversed_fields[new_events_schema] - ''' - - SELECT events.event AS event, if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 420) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_resolve_lazy_tables_two_levels ''' @@ -454,27 +283,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_resolve_lazy_tables_two_levels[new_events_schema] - ''' - - SELECT events.event AS event, events__pdi__person.id AS id - FROM events_json AS events INNER JOIN ( - SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS events__pdi___person_id, argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 420) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) LEFT JOIN ( - SELECT person.id AS id - FROM person - WHERE equals(person.team_id, 420) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_0)s), person.version), plus(now64(6, %(hogql_val_1)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS events__pdi__person ON equals(events__pdi.events__pdi___person_id, events__pdi__person.id) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_resolve_lazy_tables_two_levels_properties ''' @@ -496,27 +304,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_resolve_lazy_tables_two_levels_properties[new_events_schema] - ''' - - SELECT events.event AS event, events__pdi__person.`properties___$browser` AS `$browser` - FROM events_json AS events INNER JOIN ( - SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS events__pdi___person_id, argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id - FROM person_distinct_id2 - WHERE equals(person_distinct_id2.team_id, 420) - GROUP BY person_distinct_id2.distinct_id - HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__pdi ON equals(events.distinct_id, events__pdi.distinct_id) LEFT JOIN ( - SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '')), person.version), 1) AS `properties___$browser`, person.id AS id - FROM person - WHERE equals(person.team_id, 420) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_1)s), person.version), plus(now64(6, %(hogql_val_2)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS events__pdi__person ON equals(events__pdi.events__pdi___person_id, events__pdi__person.id) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_resolve_lazy_tables_two_levels_properties_duplicate ''' @@ -538,27 +325,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_resolve_lazy_tables_two_levels_properties_duplicate[new_events_schema] - ''' - - SELECT events.event AS event, events__person.properties AS properties, events__person.properties___name AS name - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 420) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) LEFT JOIN ( - SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '')), person.version), 1) AS properties___name, argMax(person.properties, person.version) AS properties, person.id AS id - FROM person - WHERE equals(person.team_id, 420) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_1)s), person.version), plus(now64(6, %(hogql_val_2)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_resolve_lazy_tables_two_levels_traversed ''' @@ -580,27 +346,6 @@ LIMIT 50000 ''' # --- -# name: TestLazyJoins.test_resolve_lazy_tables_two_levels_traversed[new_events_schema] - ''' - - SELECT events.event AS event, events__person.id AS id - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 420) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) LEFT JOIN ( - SELECT person.id AS id - FROM person - WHERE equals(person.team_id, 420) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_0)s), person.version), plus(now64(6, %(hogql_val_1)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestLazyJoins.test_select_count_from_lazy_table ''' diff --git a/posthog/hogql/transforms/test/__snapshots__/test_property_types.ambr b/posthog/hogql/transforms/test/__snapshots__/test_property_types.ambr index 89fda61384ec..b248041e0e8c 100644 --- a/posthog/hogql/transforms/test/__snapshots__/test_property_types.ambr +++ b/posthog/hogql/transforms/test/__snapshots__/test_property_types.ambr @@ -29,19 +29,6 @@ LIMIT 50000 ''' # --- -# name: TestPropertyTypes.test_group_boolean_property_types[new_events_schema] - ''' - - SELECT ifNull(equals(accurateCastOrNull(transform(toString(events__group_0.properties___group_boolean), %(hogql_val_2)s, %(hogql_val_3)s, NULL), %(hogql_val_4)s), 1), 0), ifNull(equals(accurateCastOrNull(transform(toString(events__group_0.properties___group_boolean), %(hogql_val_5)s, %(hogql_val_6)s, NULL), %(hogql_val_7)s), 0), 0), isNull(accurateCastOrNull(transform(toString(events__group_0.properties___group_boolean), %(hogql_val_8)s, %(hogql_val_9)s, NULL), %(hogql_val_10)s)) - FROM events_json AS events LEFT JOIN ( - SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, %(hogql_val_1)s)), 1) AS properties___group_boolean, groups.group_type_index AS index, groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 420), equals(index, 0)) - GROUP BY groups.group_type_index, groups.group_key) AS events__group_0 ON equals(events.`$group_0`, events__group_0.key) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestPropertyTypes.test_group_property_types ''' @@ -55,19 +42,6 @@ LIMIT 50000 ''' # --- -# name: TestPropertyTypes.test_group_property_types[new_events_schema] - ''' - - SELECT accurateCastOrNull(events__group_0.properties___inty, %(hogql_val_2)s) AS inty - FROM events_json AS events LEFT JOIN ( - SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, %(hogql_val_1)s)), 1) AS properties___inty, groups.group_type_index AS index, groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 420), equals(index, 0)) - GROUP BY groups.group_type_index, groups.group_key) AS events__group_0 ON equals(events.`$group_0`, events__group_0.key) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestPropertyTypes.test_resolve_property_types_combined ''' @@ -89,27 +63,6 @@ LIMIT 50000 ''' # --- -# name: TestPropertyTypes.test_resolve_property_types_combined[new_events_schema] - ''' - - SELECT multiply(accurateCastOrNull(events.properties.`$screen_width`, %(hogql_val_4)s), events__person.properties___tickets) - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 420) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) LEFT JOIN ( - SELECT tupleElement(argMax(tuple(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), %(hogql_val_1)s)), person.version), 1) AS properties___tickets, person.id AS id - FROM person - WHERE equals(person.team_id, 420) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_2)s), person.version), plus(now64(6, %(hogql_val_3)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestPropertyTypes.test_resolve_property_types_event ''' @@ -119,15 +72,6 @@ LIMIT 50000 ''' # --- -# name: TestPropertyTypes.test_resolve_property_types_event[new_events_schema] - ''' - - SELECT multiply(accurateCastOrNull(events.properties.`$screen_width`, %(hogql_val_0)s), accurateCastOrNull(events.properties.`$screen_height`, %(hogql_val_1)s)), accurateCastOrNull(transform(toString(if(notEquals(toJSONString(events.properties.^bool), '{}'), toJSONString(events.properties.^bool), if(isNull(events.properties.bool), NULL, if(startsWith(dynamicType(events.properties.bool), 'DateTime'), replaceOne(toString(events.properties.bool), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.bool), 'Array'), startsWith(dynamicType(events.properties.bool), 'Map'), startsWith(dynamicType(events.properties.bool), 'Tuple')), toJSONString(events.properties.bool), toString(events.properties.bool)))))), %(hogql_val_2)s, %(hogql_val_3)s, NULL), %(hogql_val_4)s) AS bool - FROM events_json AS events - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestPropertyTypes.test_resolve_property_types_event_person_poe_off ''' @@ -149,27 +93,6 @@ LIMIT 50000 ''' # --- -# name: TestPropertyTypes.test_resolve_property_types_event_person_poe_off[new_events_schema] - ''' - - SELECT events__person.properties___provided_timestamp AS provided_timestamp - FROM events_json AS events LEFT OUTER JOIN ( - SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 420) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) - SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) LEFT JOIN ( - SELECT tupleElement(argMax(tuple(parseDateTime64BestEffortOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', ''), 6, %(hogql_val_1)s)), person.version), 1) AS properties___provided_timestamp, person.id AS id - FROM person - WHERE equals(person.team_id, 420) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_2)s), person.version), plus(now64(6, %(hogql_val_3)s), toIntervalDay(1)))) - SETTINGS optimize_aggregation_in_order=1) AS events__person ON equals(if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id), events__person.id) - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestPropertyTypes.test_resolve_property_types_event_person_poe_on ''' @@ -179,15 +102,6 @@ LIMIT 50000 ''' # --- -# name: TestPropertyTypes.test_resolve_property_types_event_person_poe_on[new_events_schema] - ''' - - SELECT parseDateTime64BestEffortOrNull(if(notEquals(toJSONString(events.person_properties.^provided_timestamp), '{}'), toJSONString(events.person_properties.^provided_timestamp), if(isNull(events.person_properties.provided_timestamp), NULL, if(startsWith(dynamicType(events.person_properties.provided_timestamp), 'DateTime'), replaceOne(toString(events.person_properties.provided_timestamp), ' ', 'T'), if(or(startsWith(dynamicType(events.person_properties.provided_timestamp), 'Array'), startsWith(dynamicType(events.person_properties.provided_timestamp), 'Map'), startsWith(dynamicType(events.person_properties.provided_timestamp), 'Tuple')), toJSONString(events.person_properties.provided_timestamp), toString(events.person_properties.provided_timestamp))))), 6, %(hogql_val_0)s) AS provided_timestamp - FROM events_json AS events - WHERE equals(events.team_id, 420) - LIMIT 50000 - ''' -# --- # name: TestPropertyTypes.test_resolve_property_types_person ''' diff --git a/posthog/hogql/transforms/test/__snapshots__/test_state_aggregations.ambr b/posthog/hogql/transforms/test/__snapshots__/test_state_aggregations.ambr index cbea0bbfcbf7..47592059b475 100644 --- a/posthog/hogql/transforms/test/__snapshots__/test_state_aggregations.ambr +++ b/posthog/hogql/transforms/test/__snapshots__/test_state_aggregations.ambr @@ -22,29 +22,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_combine_cohort_analysis_time_windows[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(cohort_metrics, 1)), countMerge(tupleElement(cohort_metrics, 2)), sumMerge(tupleElement(cohort_metrics, 3))) AS cohort_metrics, %(hogql_val_7)s AS cohort_period - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(equals(events.event, %(hogql_val_0)s)), sumStateIf(1, equals(events.event, %(hogql_val_1)s))) AS cohort_metrics, %(hogql_val_2)s AS cohort_period - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_3)s, 6, %(hogql_val_4)s)), less(events.timestamp, toDateTime64(%(hogql_val_5)s, 6, %(hogql_val_6)s)))) - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(cohort_metrics, 1)), countMerge(tupleElement(cohort_metrics, 2)), sumMerge(tupleElement(cohort_metrics, 3))) AS cohort_metrics, %(hogql_val_15)s AS cohort_period - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(equals(events.event, %(hogql_val_8)s)), sumStateIf(1, equals(events.event, %(hogql_val_9)s))) AS cohort_metrics, %(hogql_val_10)s AS cohort_period - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_11)s, 6, %(hogql_val_12)s)), less(events.timestamp, toDateTime64(%(hogql_val_13)s, 6, %(hogql_val_14)s)))) - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(cohort_metrics, 1)), countMerge(tupleElement(cohort_metrics, 2)), sumMerge(tupleElement(cohort_metrics, 3))) AS cohort_metrics, %(hogql_val_23)s AS cohort_period - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(equals(events.event, %(hogql_val_16)s)), sumStateIf(1, equals(events.event, %(hogql_val_17)s))) AS cohort_metrics, %(hogql_val_18)s AS cohort_period - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_19)s, 6, %(hogql_val_20)s)), less(events.timestamp, toDateTime64(%(hogql_val_21)s, 6, %(hogql_val_22)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_combine_complex_nested_aggregation_patterns ''' @@ -66,27 +43,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_combine_complex_nested_aggregation_patterns[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(comprehensive_metrics, 1)), countMerge(tupleElement(comprehensive_metrics, 2)), sumMerge(tupleElement(comprehensive_metrics, 3)), avgMerge(tupleElement(comprehensive_metrics, 4))) AS comprehensive_metrics, source AS source - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(equals(events.event, %(hogql_val_0)s)), sumStateIf(1, equals(events.event, %(hogql_val_1)s)), avgStateIf(1, equals(events.event, %(hogql_val_2)s))) AS comprehensive_metrics, if(notEquals(toJSONString(events.properties.^campaign_source), '{}'), toJSONString(events.properties.^campaign_source), if(isNull(events.properties.campaign_source), NULL, if(startsWith(dynamicType(events.properties.campaign_source), 'DateTime'), replaceOne(toString(events.properties.campaign_source), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.campaign_source), 'Array'), startsWith(dynamicType(events.properties.campaign_source), 'Map'), startsWith(dynamicType(events.properties.campaign_source), 'Tuple')), toJSONString(events.properties.campaign_source), toString(events.properties.campaign_source))))) AS source - FROM events_json AS events - WHERE and(equals(events.team_id, 420), ifNull(equals(if(notEquals(toJSONString(events.properties.^campaign_source), '{}'), toJSONString(events.properties.^campaign_source), if(isNull(events.properties.campaign_source), NULL, if(startsWith(dynamicType(events.properties.campaign_source), 'DateTime'), replaceOne(toString(events.properties.campaign_source), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.campaign_source), 'Array'), startsWith(dynamicType(events.properties.campaign_source), 'Map'), startsWith(dynamicType(events.properties.campaign_source), 'Tuple')), toJSONString(events.properties.campaign_source), toString(events.properties.campaign_source))))), %(hogql_val_3)s), 0)) - GROUP BY source) - GROUP BY source - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(comprehensive_metrics, 1)), countMerge(tupleElement(comprehensive_metrics, 2)), sumMerge(tupleElement(comprehensive_metrics, 3)), avgMerge(tupleElement(comprehensive_metrics, 4))) AS comprehensive_metrics, source AS source - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(equals(events.event, %(hogql_val_4)s)), sumStateIf(1, equals(events.event, %(hogql_val_5)s)), avgStateIf(1, equals(events.event, %(hogql_val_6)s))) AS comprehensive_metrics, if(notEquals(toJSONString(events.properties.^campaign_source), '{}'), toJSONString(events.properties.^campaign_source), if(isNull(events.properties.campaign_source), NULL, if(startsWith(dynamicType(events.properties.campaign_source), 'DateTime'), replaceOne(toString(events.properties.campaign_source), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.campaign_source), 'Array'), startsWith(dynamicType(events.properties.campaign_source), 'Map'), startsWith(dynamicType(events.properties.campaign_source), 'Tuple')), toJSONString(events.properties.campaign_source), toString(events.properties.campaign_source))))) AS source - FROM events_json AS events - WHERE and(equals(events.team_id, 420), ifNull(equals(if(notEquals(toJSONString(events.properties.^campaign_source), '{}'), toJSONString(events.properties.^campaign_source), if(isNull(events.properties.campaign_source), NULL, if(startsWith(dynamicType(events.properties.campaign_source), 'DateTime'), replaceOne(toString(events.properties.campaign_source), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.campaign_source), 'Array'), startsWith(dynamicType(events.properties.campaign_source), 'Map'), startsWith(dynamicType(events.properties.campaign_source), 'Tuple')), toJSONString(events.properties.campaign_source), toString(events.properties.campaign_source))))), %(hogql_val_7)s), 0)) - GROUP BY source) - GROUP BY source - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_combine_funnel_analysis_segments ''' @@ -104,23 +60,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_combine_funnel_analysis_segments[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(funnel_metrics, 1)), countMerge(tupleElement(funnel_metrics, 2)), countMerge(tupleElement(funnel_metrics, 3))) AS funnel_metrics - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(equals(events.event, %(hogql_val_0)s)), countStateIf(equals(events.event, %(hogql_val_1)s))) AS funnel_metrics - FROM events_json AS events - WHERE and(equals(events.team_id, 420), ifNull(equals(if(notEquals(toJSONString(events.properties.^device_type), '{}'), toJSONString(events.properties.^device_type), if(isNull(events.properties.device_type), NULL, if(startsWith(dynamicType(events.properties.device_type), 'DateTime'), replaceOne(toString(events.properties.device_type), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.device_type), 'Array'), startsWith(dynamicType(events.properties.device_type), 'Map'), startsWith(dynamicType(events.properties.device_type), 'Tuple')), toJSONString(events.properties.device_type), toString(events.properties.device_type))))), %(hogql_val_2)s), 0))) - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(funnel_metrics, 1)), countMerge(tupleElement(funnel_metrics, 2)), countMerge(tupleElement(funnel_metrics, 3))) AS funnel_metrics - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(equals(events.event, %(hogql_val_3)s)), countStateIf(equals(events.event, %(hogql_val_4)s))) AS funnel_metrics - FROM events_json AS events - WHERE and(equals(events.team_id, 420), ifNull(equals(if(notEquals(toJSONString(events.properties.^device_type), '{}'), toJSONString(events.properties.^device_type), if(isNull(events.properties.device_type), NULL, if(startsWith(dynamicType(events.properties.device_type), 'DateTime'), replaceOne(toString(events.properties.device_type), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.device_type), 'Array'), startsWith(dynamicType(events.properties.device_type), 'Map'), startsWith(dynamicType(events.properties.device_type), 'Tuple')), toJSONString(events.properties.device_type), toString(events.properties.device_type))))), %(hogql_val_5)s), 0))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_combine_mixed_transformation_stages_with_tuples ''' @@ -142,27 +81,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_combine_mixed_transformation_stages_with_tuples[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(user_metrics, 1)), countMerge(tupleElement(user_metrics, 2))) AS user_metrics, tuple(sumMerge(tupleElement(activity_metrics, 1)), avgMerge(tupleElement(activity_metrics, 2))) AS activity_metrics, date AS date - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_metrics, tuple(sumState(1), avgState(1)) AS activity_metrics, toDate(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS date - FROM events_json AS events - WHERE and(equals(events.team_id, 420), less(toDate(toTimeZone(events.timestamp, %(hogql_val_1)s)), %(hogql_val_2)s)) - GROUP BY date) - GROUP BY date - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(user_metrics, 1)), countMerge(tupleElement(user_metrics, 2))) AS user_metrics, tuple(sumMerge(tupleElement(activity_metrics, 1)), avgMerge(tupleElement(activity_metrics, 2))) AS activity_metrics, date AS date - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_metrics, tuple(sumState(1), avgState(1)) AS activity_metrics, toDate(toTimeZone(events.timestamp, %(hogql_val_3)s)) AS date - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_4)s, 6, %(hogql_val_5)s))) - GROUP BY date) - GROUP BY date - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_combine_multiple_time_periods_with_conditional_aggregations ''' @@ -180,23 +98,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_combine_multiple_time_periods_with_conditional_aggregations[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(metrics, 1)), countMerge(tupleElement(metrics, 2)), sumMerge(tupleElement(metrics, 3))) AS metrics - FROM ( - SELECT tuple(uniqStateIf(events.distinct_id, equals(events.event, %(hogql_val_0)s)), countStateIf(equals(events.event, %(hogql_val_1)s)), sumStateIf(1, 1)) AS metrics - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_2)s, 6, %(hogql_val_3)s)), less(events.timestamp, toDateTime64(%(hogql_val_4)s, 6, %(hogql_val_5)s)))) - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(metrics, 1)), countMerge(tupleElement(metrics, 2)), sumMerge(tupleElement(metrics, 3))) AS metrics - FROM ( - SELECT tuple(uniqStateIf(events.distinct_id, equals(events.event, %(hogql_val_6)s)), countStateIf(equals(events.event, %(hogql_val_7)s)), sumStateIf(1, 1)) AS metrics - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_8)s, 6, %(hogql_val_9)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_combine_regular_and_state_aggregations_mixed ''' @@ -214,23 +115,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_combine_regular_and_state_aggregations_mixed[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(metrics, 1)), countMerge(tupleElement(metrics, 2)), sumMerge(tupleElement(metrics, 3))) AS metrics, %(hogql_val_4)s AS source_type - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState(), sumStateIf(1, equals(events.event, %(hogql_val_0)s))) AS metrics, %(hogql_val_1)s AS source_type - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_2)s, 6, %(hogql_val_3)s)))) - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(metrics, 1)), countMerge(tupleElement(metrics, 2)), sumMerge(tupleElement(metrics, 3))) AS metrics, %(hogql_val_9)s AS source_type - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState(), sumStateIf(1, equals(events.event, %(hogql_val_5)s))) AS metrics, %(hogql_val_6)s AS source_type - FROM events_json AS events - WHERE and(equals(events.team_id, 420), less(events.timestamp, toDateTime64(%(hogql_val_7)s, 6, %(hogql_val_8)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_combine_three_data_sources_with_mixed_tuples ''' @@ -260,35 +144,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_combine_three_data_sources_with_mixed_tuples[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(user_metrics, 1)), countMerge(tupleElement(user_metrics, 2))) AS user_metrics, tuple(sumMerge(tupleElement(value_metrics, 1)), avgMerge(tupleElement(value_metrics, 2))) AS value_metrics, host AS host - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_metrics, tuple(sumState(1), avgState(1)) AS value_metrics, events.properties.`$host` AS host - FROM events_json AS events - WHERE and(equals(events.team_id, 420), less(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s))) - GROUP BY host) - GROUP BY host - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(user_metrics, 1)), countMerge(tupleElement(user_metrics, 2))) AS user_metrics, tuple(sumMerge(tupleElement(value_metrics, 1)), avgMerge(tupleElement(value_metrics, 2))) AS value_metrics, host AS host - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_metrics, tuple(sumState(1), avgState(1)) AS value_metrics, events.properties.`$host` AS host - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_2)s, 6, %(hogql_val_3)s)), less(events.timestamp, toDateTime64(%(hogql_val_4)s, 6, %(hogql_val_5)s))) - GROUP BY host) - GROUP BY host - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(user_metrics, 1)), countMerge(tupleElement(user_metrics, 2))) AS user_metrics, tuple(sumMerge(tupleElement(value_metrics, 1)), avgMerge(tupleElement(value_metrics, 2))) AS value_metrics, host AS host - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_metrics, tuple(sumState(1), avgState(1)) AS value_metrics, events.properties.`$host` AS host - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_6)s, 6, %(hogql_val_7)s))) - GROUP BY host) - GROUP BY host - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_combine_two_different_state_queries_into_one_merge_query ''' @@ -306,23 +161,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_combine_two_different_state_queries_into_one_merge_query[new_events_schema] - ''' - - SELECT countMerge(total_pageviews) AS total_pageviews - FROM ( - SELECT countState() AS total_pageviews - FROM events_json AS events - WHERE and(equals(events.team_id, 420), less(events.timestamp, minus(now64(6, %(hogql_val_0)s), toIntervalDay(1))))) - LIMIT 50000 UNION ALL - SELECT countMerge(total_pageviews) AS total_pageviews - FROM ( - SELECT countState() AS total_pageviews - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, minus(now64(6, %(hogql_val_1)s), toIntervalDay(1))))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_combine_web_analytics_historical_and_realtime_data ''' @@ -344,27 +182,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_combine_web_analytics_historical_and_realtime_data[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(daily_metrics, 1)), countMerge(tupleElement(daily_metrics, 2)), sumMerge(tupleElement(daily_metrics, 3))) AS daily_metrics, date AS date - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState(), sumStateIf(1, equals(events.event, %(hogql_val_0)s))) AS daily_metrics, toDate(toTimeZone(events.timestamp, %(hogql_val_1)s)) AS date - FROM events_json AS events - WHERE and(equals(events.team_id, 420), less(toDate(toTimeZone(events.timestamp, %(hogql_val_2)s)), %(hogql_val_3)s)) - GROUP BY date) - GROUP BY date - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(daily_metrics, 1)), countMerge(tupleElement(daily_metrics, 2)), sumMerge(tupleElement(daily_metrics, 3))) AS daily_metrics, date AS date - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState(), sumStateIf(1, equals(events.event, %(hogql_val_4)s))) AS daily_metrics, toDate(toTimeZone(events.timestamp, %(hogql_val_5)s)) AS date - FROM events_json AS events - WHERE and(equals(events.team_id, 420), equals(toDate(toTimeZone(events.timestamp, %(hogql_val_6)s)), %(hogql_val_7)s)) - GROUP BY date) - GROUP BY date - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_complex_tuple_union_all_with_grouping ''' @@ -386,27 +203,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_complex_tuple_union_all_with_grouping[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(stats_tuple, 1)), countMerge(tupleElement(stats_tuple, 2)), sumMerge(tupleElement(stats_tuple, 3))) AS stats_tuple, date_key AS date_key - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(1), sumState(1)) AS stats_tuple, toDate(toTimeZone(events.timestamp, %(hogql_val_0)s)) AS date_key - FROM events_json AS events - WHERE and(equals(events.team_id, 420), less(toDate(toTimeZone(events.timestamp, %(hogql_val_1)s)), %(hogql_val_2)s)) - GROUP BY date_key) - GROUP BY date_key - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(stats_tuple, 1)), countMerge(tupleElement(stats_tuple, 2)), sumMerge(tupleElement(stats_tuple, 3))) AS stats_tuple, date_key AS date_key - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(1), sumState(1)) AS stats_tuple, toDate(toTimeZone(events.timestamp, %(hogql_val_3)s)) AS date_key - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(toDate(toTimeZone(events.timestamp, %(hogql_val_4)s)), %(hogql_val_5)s)) - GROUP BY date_key) - GROUP BY date_key - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_filtered_aggregation ''' @@ -418,17 +214,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_filtered_aggregation[new_events_schema] - ''' - - SELECT uniqMerge(unique_users) AS unique_users, countMerge(total_events) AS total_events - FROM ( - SELECT uniqState(events.distinct_id) AS unique_users, countState() AS total_events - FROM events_json AS events - WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_0)s))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_merge_wraps_works_with_more_complex_queries ''' @@ -445,22 +230,6 @@ LIMIT 10 ''' # --- -# name: TestStateTransforms.test_merge_wraps_works_with_more_complex_queries[new_events_schema] - ''' - - SELECT uniqMerge(unique_users) AS unique_users, countMerge(total_events) AS total_events, host AS host - FROM ( - SELECT uniqState(events.distinct_id) AS unique_users, countState() AS total_events, events.properties.`$host` AS host - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s))) - GROUP BY host - ORDER BY total_events DESC - LIMIT 10) - GROUP BY host - ORDER BY total_events DESC - LIMIT 10 - ''' -# --- # name: TestStateTransforms.test_multi_level_tuple_union_all_transformation ''' @@ -478,23 +247,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_multi_level_tuple_union_all_transformation[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(user_metrics, 1)), countMerge(tupleElement(user_metrics, 2))) AS user_metrics, tuple(sumMerge(tupleElement(session_metrics, 1)), avgMerge(tupleElement(session_metrics, 2))) AS session_metrics - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_metrics, tuple(sumState(1), avgState(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^duration), '{}'), toJSONString(events.properties.^duration), if(isNull(events.properties.duration), NULL, if(startsWith(dynamicType(events.properties.duration), 'DateTime'), replaceOne(toString(events.properties.duration), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.duration), 'Array'), startsWith(dynamicType(events.properties.duration), 'Map'), startsWith(dynamicType(events.properties.duration), 'Tuple')), toJSONString(events.properties.duration), toString(events.properties.duration))))), %(hogql_val_0)s))) AS session_metrics - FROM events_json AS events - WHERE and(equals(events.team_id, 420), less(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s)))) - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(user_metrics, 1)), countMerge(tupleElement(user_metrics, 2))) AS user_metrics, tuple(sumMerge(tupleElement(session_metrics, 1)), avgMerge(tupleElement(session_metrics, 2))) AS session_metrics - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_metrics, tuple(sumState(1), avgState(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^duration), '{}'), toJSONString(events.properties.^duration), if(isNull(events.properties.duration), NULL, if(startsWith(dynamicType(events.properties.duration), 'DateTime'), replaceOne(toString(events.properties.duration), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.duration), 'Array'), startsWith(dynamicType(events.properties.duration), 'Map'), startsWith(dynamicType(events.properties.duration), 'Tuple')), toJSONString(events.properties.duration), toString(events.properties.duration))))), %(hogql_val_3)s))) AS session_metrics - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_4)s, 6, %(hogql_val_5)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_multiple_tuples_with_state_aggregations ''' @@ -506,17 +258,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_multiple_tuples_with_state_aggregations[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(user_stats, 1)), countMerge(tupleElement(user_stats, 2))) AS user_stats, tuple(sumMerge(tupleElement(metric_stats, 1)), avgMerge(tupleElement(metric_stats, 2))) AS metric_stats - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_stats, tuple(sumState(1), avgState(1)) AS metric_stats - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_nested_aggregations_in_subquery ''' @@ -531,20 +272,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_nested_aggregations_in_subquery[new_events_schema] - ''' - - SELECT sumMerge(total_filtered_count) AS total_filtered_count - FROM ( - SELECT sumState(filtered_count) AS total_filtered_count - FROM ( - SELECT countIf(equals(events.event, %(hogql_val_0)s)) AS filtered_count - FROM events_json AS events - WHERE equals(events.team_id, 420) - GROUP BY events.distinct_id)) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_nested_functions_aggregations_and_conversions ''' @@ -558,19 +285,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_nested_functions_aggregations_and_conversions[new_events_schema] - ''' - - SELECT host AS host, uniqMerge(unique_users) AS unique_users, sumMerge(click_count) AS click_count, avgMerge(avg_duration) AS avg_duration - FROM ( - SELECT events.properties.`$host` AS host, uniqState(events.distinct_id) AS unique_users, sumStateIf(1, equals(events.event, %(hogql_val_0)s)) AS click_count, avgState(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^session_duration), '{}'), toJSONString(events.properties.^session_duration), if(isNull(events.properties.session_duration), NULL, if(startsWith(dynamicType(events.properties.session_duration), 'DateTime'), replaceOne(toString(events.properties.session_duration), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.session_duration), 'Array'), startsWith(dynamicType(events.properties.session_duration), 'Map'), startsWith(dynamicType(events.properties.session_duration), 'Tuple')), toJSONString(events.properties.session_duration), toString(events.properties.session_duration))))), %(hogql_val_1)s)) AS avg_duration - FROM events_json AS events - WHERE equals(events.team_id, 420) - GROUP BY host) - GROUP BY host - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_nested_subquery_with_tuple_union_all ''' @@ -587,22 +301,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_nested_subquery_with_tuple_union_all[new_events_schema] - ''' - - SELECT sumMerge(final_users) AS final_users, avgMerge(final_avg_duration) AS final_avg_duration - FROM ( - SELECT sumState(combined_stats.total_users) AS final_users, avgState(combined_stats.avg_duration) AS final_avg_duration - FROM ( - SELECT tuple(uniqState(events.distinct_id), avgState(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^session_duration), '{}'), toJSONString(events.properties.^session_duration), if(isNull(events.properties.session_duration), NULL, if(startsWith(dynamicType(events.properties.session_duration), 'DateTime'), replaceOne(toString(events.properties.session_duration), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.session_duration), 'Array'), startsWith(dynamicType(events.properties.session_duration), 'Map'), startsWith(dynamicType(events.properties.session_duration), 'Tuple')), toJSONString(events.properties.session_duration), toString(events.properties.session_duration))))), %(hogql_val_0)s))) AS user_duration_tuple, tupleElement(user_duration_tuple, 1) AS total_users, tupleElement(user_duration_tuple, 2) AS avg_duration - FROM events_json AS events - WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_1)s), less(events.timestamp, toDateTime64(%(hogql_val_2)s, 6, %(hogql_val_3)s))) UNION ALL - SELECT tuple(uniqState(events.distinct_id), avgState(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^session_duration), '{}'), toJSONString(events.properties.^session_duration), if(isNull(events.properties.session_duration), NULL, if(startsWith(dynamicType(events.properties.session_duration), 'DateTime'), replaceOne(toString(events.properties.session_duration), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.session_duration), 'Array'), startsWith(dynamicType(events.properties.session_duration), 'Map'), startsWith(dynamicType(events.properties.session_duration), 'Tuple')), toJSONString(events.properties.session_duration), toString(events.properties.session_duration))))), %(hogql_val_4)s))) AS user_duration_tuple, tupleElement(user_duration_tuple, 1) AS total_users, tupleElement(user_duration_tuple, 2) AS avg_duration - FROM events_json AS events - WHERE and(equals(events.team_id, 420), equals(events.event, %(hogql_val_5)s), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_6)s, 6, %(hogql_val_7)s)))) AS combined_stats) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_preserve_group_by ''' @@ -613,16 +311,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_preserve_group_by[new_events_schema] - ''' - - SELECT events.properties.`$pathname` AS pathname, uniqState(events.distinct_id) AS unique_users, countState() AS total_events - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s))) - GROUP BY pathname - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_preserve_query_without_aggregations ''' @@ -632,15 +320,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_preserve_query_without_aggregations[new_events_schema] - ''' - - SELECT events.distinct_id AS distinct_id, events.properties.`$pathname` AS pathname - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_query_similar_to_web_overview_query_transformation_sql ''' @@ -655,20 +334,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_query_similar_to_web_overview_query_transformation_sql[new_events_schema] - ''' - - SELECT sumMerge(total_pageviews) AS total_pageviews, uniqMerge(unique_users) AS unique_users - FROM ( - SELECT sumState(pageview_count) AS total_pageviews, uniqState(user_id) AS unique_users - FROM ( - SELECT events.distinct_id AS user_id, countIf(equals(events.event, %(hogql_val_0)s)) AS pageview_count - FROM events_json AS events - WHERE equals(events.team_id, 420) - GROUP BY events.distinct_id)) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_query_with_as_constants ''' @@ -680,17 +345,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_query_with_as_constants[new_events_schema] - ''' - - SELECT uniqMerge(unique_users) AS unique_users, NULL AS previous_unique_users, countMerge(total_events) AS total_events, 123 AS constant_value - FROM ( - SELECT uniqState(events.distinct_id) AS unique_users, NULL AS previous_unique_users, countState() AS total_events, 123 AS constant_value - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_three_way_mixed_aggregation_stages ''' @@ -714,29 +368,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_three_way_mixed_aggregation_stages[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(pageview_metrics, 1)), countMerge(tupleElement(pageview_metrics, 2))) AS pageview_metrics - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(equals(events.event, %(hogql_val_0)s))) AS pageview_metrics - FROM events_json AS events - WHERE and(equals(events.team_id, 420), less(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s)))) - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(pageview_metrics, 1)), countMerge(tupleElement(pageview_metrics, 2))) AS pageview_metrics - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(equals(events.event, %(hogql_val_3)s))) AS pageview_metrics - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_4)s, 6, %(hogql_val_5)s)), less(events.timestamp, toDateTime64(%(hogql_val_6)s, 6, %(hogql_val_7)s)))) - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(pageview_metrics, 1)), countMerge(tupleElement(pageview_metrics, 2))) AS pageview_metrics - FROM ( - SELECT tuple(uniqState(events.distinct_id), countStateIf(equals(events.event, %(hogql_val_8)s))) AS pageview_metrics - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_9)s, 6, %(hogql_val_10)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_transform_nested_expressions ''' @@ -746,15 +377,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_transform_nested_expressions[new_events_schema] - ''' - - SELECT uniqState(events.distinct_id) AS unique_users, countState(if(equals(events.event, %(hogql_val_0)s), 1, 0)) AS pageview_count - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_transform_simple_query_to_state_aggregations ''' @@ -764,15 +386,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_transform_simple_query_to_state_aggregations[new_events_schema] - ''' - - SELECT uniqState(events.distinct_id) AS unique_users, countState() AS total_events - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_transformation_from_regular_to_state_then_merge_with_tuples ''' @@ -784,17 +397,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_transformation_from_regular_to_state_then_merge_with_tuples[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(user_stats, 1)), countMerge(tupleElement(user_stats, 2))) AS user_stats, tuple(sumMerge(tupleElement(metric_stats, 1)), avgMerge(tupleElement(metric_stats, 2))) AS metric_stats - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_stats, tuple(sumState(1), avgState(1)) AS metric_stats - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_tuple_in_subquery_remains_unchanged ''' @@ -808,19 +410,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_tuple_in_subquery_remains_unchanged[new_events_schema] - ''' - - SELECT sumMerge(aggregated_total) AS aggregated_total - FROM ( - SELECT sumState(filtered_metrics.total_count) AS aggregated_total - FROM ( - SELECT tuple(uniq(events.distinct_id), count()) AS user_metrics, sum(1) AS total_count - FROM events_json AS events - WHERE equals(events.team_id, 420)) AS filtered_metrics) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_tuple_union_all_with_different_conditions ''' @@ -838,23 +427,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_tuple_union_all_with_different_conditions[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(pageview_stats, 1)), countMerge(tupleElement(pageview_stats, 2))) AS pageview_stats, tuple(uniqMerge(tupleElement(click_stats, 1)), countMerge(tupleElement(click_stats, 2))) AS click_stats - FROM ( - SELECT tuple(uniqStateIf(events.distinct_id, equals(events.event, %(hogql_val_0)s)), countStateIf(equals(events.event, %(hogql_val_1)s))) AS pageview_stats, tuple(uniqStateIf(events.distinct_id, equals(events.event, %(hogql_val_2)s)), countStateIf(equals(events.event, %(hogql_val_3)s))) AS click_stats - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_4)s, 6, %(hogql_val_5)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_6)s, 6, %(hogql_val_7)s)))) - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(pageview_stats, 1)), countMerge(tupleElement(pageview_stats, 2))) AS pageview_stats, tuple(uniqMerge(tupleElement(click_stats, 1)), countMerge(tupleElement(click_stats, 2))) AS click_stats - FROM ( - SELECT tuple(uniqStateIf(events.distinct_id, equals(events.event, %(hogql_val_8)s)), countStateIf(equals(events.event, %(hogql_val_9)s))) AS pageview_stats, tuple(uniqStateIf(events.distinct_id, equals(events.event, %(hogql_val_10)s)), countStateIf(equals(events.event, %(hogql_val_11)s))) AS click_stats - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_12)s, 6, %(hogql_val_13)s)), lessOrEquals(events.timestamp, toDateTime64(%(hogql_val_14)s, 6, %(hogql_val_15)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_tuple_with_conditional_aggregations ''' @@ -866,17 +438,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_tuple_with_conditional_aggregations[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(conditional_stats, 1)), countMerge(tupleElement(conditional_stats, 2)), sumMerge(tupleElement(conditional_stats, 3))) AS conditional_stats - FROM ( - SELECT tuple(uniqStateIf(events.distinct_id, 1), countStateIf(1), sumStateIf(1, 1)) AS conditional_stats - FROM events_json AS events - WHERE equals(events.team_id, 420)) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_tuple_with_constants_and_state_aggregations ''' @@ -888,17 +449,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_tuple_with_constants_and_state_aggregations[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(mixed_tuple, 1)), tupleElement(mixed_tuple, 2), countMerge(tupleElement(mixed_tuple, 3)), tupleElement(mixed_tuple, 4), tupleElement(mixed_tuple, 5)) AS mixed_tuple - FROM ( - SELECT tuple(uniqState(events.distinct_id), NULL, countState(), %(hogql_val_0)s, 42) AS mixed_tuple - FROM events_json AS events - WHERE equals(events.team_id, 420)) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_tuple_with_mixed_aggregations_and_non_aggregations ''' @@ -910,17 +460,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_tuple_with_mixed_aggregations_and_non_aggregations[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(mixed_stats, 1)), tupleElement(mixed_stats, 2), countMerge(tupleElement(mixed_stats, 3))) AS mixed_stats, sumMerge(total_sum) AS total_sum - FROM ( - SELECT tuple(uniqState(events.distinct_id), %(hogql_val_0)s, countState()) AS mixed_stats, sumState(1) AS total_sum - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_tuple_with_state_aggregations ''' @@ -934,19 +473,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_tuple_with_state_aggregations[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(user_stats, 1)), countMerge(tupleElement(user_stats, 2))) AS user_stats, host AS host - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_stats, events.properties.`$host` AS host - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s))) - GROUP BY host) - GROUP BY host - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_union_all_state_queries_into_one_merge_query ''' @@ -964,23 +490,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_union_all_state_queries_into_one_merge_query[new_events_schema] - ''' - - SELECT countMerge(total_pageviews) AS total_pageviews - FROM ( - SELECT countState() AS total_pageviews - FROM events_json AS events - WHERE and(equals(events.team_id, 420), less(events.timestamp, minus(now64(6, %(hogql_val_0)s), toIntervalDay(1))))) - LIMIT 50000 UNION ALL - SELECT countMerge(total_pageviews) AS total_pageviews - FROM ( - SELECT countState() AS total_pageviews - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, minus(now64(6, %(hogql_val_1)s), toIntervalDay(1))))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_union_all_tuples_with_state_aggregations ''' @@ -998,23 +507,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_union_all_tuples_with_state_aggregations[new_events_schema] - ''' - - SELECT tuple(uniqMerge(tupleElement(user_stats, 1)), countMerge(tupleElement(user_stats, 2))) AS user_stats, tuple(sumMerge(tupleElement(metric_stats, 1)), avgMerge(tupleElement(metric_stats, 2))) AS metric_stats, %(hogql_val_3)s AS data_source - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_stats, tuple(sumState(1), avgState(1)) AS metric_stats, %(hogql_val_0)s AS data_source - FROM events_json AS events - WHERE and(equals(events.team_id, 420), less(events.timestamp, toDateTime64(%(hogql_val_1)s, 6, %(hogql_val_2)s)))) - LIMIT 50000 UNION ALL - SELECT tuple(uniqMerge(tupleElement(user_stats, 1)), countMerge(tupleElement(user_stats, 2))) AS user_stats, tuple(sumMerge(tupleElement(metric_stats, 1)), avgMerge(tupleElement(metric_stats, 2))) AS metric_stats, %(hogql_val_7)s AS data_source - FROM ( - SELECT tuple(uniqState(events.distinct_id), countState()) AS user_stats, tuple(sumState(1), avgState(1)) AS metric_stats, %(hogql_val_4)s AS data_source - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_5)s, 6, %(hogql_val_6)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_wrap_state_query_in_merge_query ''' @@ -1026,17 +518,6 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_wrap_state_query_in_merge_query[new_events_schema] - ''' - - SELECT uniqMerge(unique_users) AS unique_users, countMerge(total_events) AS total_events - FROM ( - SELECT uniqState(events.distinct_id) AS unique_users, countState() AS total_events - FROM events_json AS events - WHERE and(equals(events.team_id, 420), greaterOrEquals(events.timestamp, toDateTime64(%(hogql_val_0)s, 6, %(hogql_val_1)s)))) - LIMIT 50000 - ''' -# --- # name: TestStateTransforms.test_wrapper_query_aggregation_with_groupby ''' @@ -1050,16 +531,3 @@ LIMIT 50000 ''' # --- -# name: TestStateTransforms.test_wrapper_query_aggregation_with_groupby[new_events_schema] - ''' - - SELECT host AS host, countMerge(total_count) AS total_count, countMerge(click_count) AS click_count - FROM ( - SELECT events.properties.`$host` AS host, countState() AS total_count, countStateIf(equals(events.event, %(hogql_val_0)s)) AS click_count - FROM events_json AS events - WHERE equals(events.team_id, 420) - GROUP BY host) - GROUP BY host - LIMIT 50000 - ''' -# --- diff --git a/posthog/hogql_queries/ai/test/__snapshots__/test_event_taxonomy_query_runner.ambr b/posthog/hogql_queries/ai/test/__snapshots__/test_event_taxonomy_query_runner.ambr index 4433c87d0a00..8820b809780e 100644 --- a/posthog/hogql_queries/ai/test/__snapshots__/test_event_taxonomy_query_runner.ambr +++ b/posthog/hogql_queries/ai/test/__snapshots__/test_event_taxonomy_query_runner.ambr @@ -40,49 +40,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestEventTaxonomyQueryRunner.test_event_taxonomy_query_runner[new_events_schema] - ''' - SELECT key AS key, - arrayMap(item -> item.3, arraySlice(reverse(arraySort(item -> tuple(item.1, item.2, item.3), groupArray(tuple(value_count, latest_seen, value)))), 1, 5)) AS - values, - count(DISTINCT value) AS total_count - FROM - (SELECT key, - value, - count() AS value_count, - max(timestamp) AS latest_seen - FROM - (SELECT JSONExtractKeysAndValues(concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}'), 'String') AS kv, - toTimeZone(events.timestamp, 'UTC') AS timestamp - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), greaterOrEquals(events.timestamp, minus(now64(6, 'UTC'), toIntervalDay(30))), equals(events.event, 'event1')) - ORDER BY toTimeZone(events.timestamp, 'UTC') DESC - LIMIT 100) ARRAY - JOIN (kv).1 AS key, - (kv).2 AS value - WHERE and(not(match(key, '(\\$set|\\$time|\\$set_once|\\$sent_at|distinct_id|\\$ip|\\$feature\\/|\\$feature_enrollment\\/|\\$feature_interaction\\/|\\$product_tour|__|survey_dismiss|survey_responded|phjs|partial_filter_chosen|changed_action|window-id|changed_event|partial_filter)')), isNotNull(value), ifNull(notEquals(value, ''), 1)) - GROUP BY key, - value) - GROUP BY key - ORDER BY total_count DESC, - key ASC - LIMIT 501 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestEventTaxonomyQueryRunner.test_property_taxonomy_handles_numeric_property_values ''' SELECT key AS key, @@ -122,45 +79,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestEventTaxonomyQueryRunner.test_property_taxonomy_handles_numeric_property_values[new_events_schema] - ''' - SELECT key AS key, - arrayMap(item -> item.3, arraySlice(reverse(arraySort(item -> tuple(item.1, item.2, item.3), groupArray(tuple(value_count, latest_seen, value)))), 1, 5)) AS - values, - count(DISTINCT value) AS total_count - FROM - (SELECT key, - value, - count() AS value_count, - max(timestamp) AS latest_seen - FROM - (SELECT [tuple('zero_duration_recording_count_in_period', JSONExtractString(ifNull(if(notEquals(toJSONString(events.properties.^zero_duration_recording_count_in_period), '{}'), toJSONString(events.properties.^zero_duration_recording_count_in_period), if(isNull(events.properties.zero_duration_recording_count_in_period), NULL, if(startsWith(dynamicType(events.properties.zero_duration_recording_count_in_period), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(events.properties.zero_duration_recording_count_in_period), ' ', 'T')), ''), '"'), toJSONString(events.properties.zero_duration_recording_count_in_period)))), '')))] AS kv, - toTimeZone(events.timestamp, 'UTC') AS timestamp - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), greaterOrEquals(events.timestamp, minus(now64(6, 'UTC'), toIntervalDay(30))), equals(events.event, 'organization usage report'), notEquals(JSONExtractString(ifNull(if(notEquals(toJSONString(events.properties.^zero_duration_recording_count_in_period), '{}'), toJSONString(events.properties.^zero_duration_recording_count_in_period), if(isNull(events.properties.zero_duration_recording_count_in_period), NULL, if(startsWith(dynamicType(events.properties.zero_duration_recording_count_in_period), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(events.properties.zero_duration_recording_count_in_period), ' ', 'T')), ''), '"'), toJSONString(events.properties.zero_duration_recording_count_in_period)))), '')), ''))) ARRAY - JOIN (kv).1 AS key, - (kv).2 AS value - WHERE and(isNotNull(value), ifNull(notEquals(value, ''), 1)) - GROUP BY key, - value) - GROUP BY key - ORDER BY total_count DESC, - key ASC - LIMIT 501 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestEventTaxonomyQueryRunner.test_retrieves_action_properties ''' SELECT key AS key, @@ -202,46 +120,3 @@ use_hive_partitioning=0 ''' # --- -# name: TestEventTaxonomyQueryRunner.test_retrieves_action_properties[new_events_schema] - ''' - SELECT key AS key, - arrayMap(item -> item.3, arraySlice(reverse(arraySort(item -> tuple(item.1, item.2, item.3), groupArray(tuple(value_count, latest_seen, value)))), 1, 5)) AS - values, - count(DISTINCT value) AS total_count - FROM - (SELECT key, - value, - count() AS value_count, - max(timestamp) AS latest_seen - FROM - (SELECT JSONExtractKeysAndValues(concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}'), 'String') AS kv, - toTimeZone(events.timestamp, 'UTC') AS timestamp - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), greaterOrEquals(events.timestamp, minus(now64(6, 'UTC'), toIntervalDay(30))), equals(events.event, '$pageview')) - ORDER BY toTimeZone(events.timestamp, 'UTC') DESC - LIMIT 100) ARRAY - JOIN (kv).1 AS key, - (kv).2 AS value - WHERE and(not(match(key, '(\\$set|\\$time|\\$set_once|\\$sent_at|distinct_id|\\$ip|\\$feature\\/|\\$feature_enrollment\\/|\\$feature_interaction\\/|\\$product_tour|__|survey_dismiss|survey_responded|phjs|partial_filter_chosen|changed_action|window-id|changed_event|partial_filter)')), isNotNull(value), ifNull(notEquals(value, ''), 1)) - GROUP BY key, - value) - GROUP BY key - ORDER BY total_count DESC, - key ASC - LIMIT 501 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- diff --git a/posthog/hogql_queries/ai/test/__snapshots__/test_suggested_questions_query_runner.ambr b/posthog/hogql_queries/ai/test/__snapshots__/test_suggested_questions_query_runner.ambr index e45fec59743a..fd5132ddbcda 100644 --- a/posthog/hogql_queries/ai/test/__snapshots__/test_suggested_questions_query_runner.ambr +++ b/posthog/hogql_queries/ai/test/__snapshots__/test_suggested_questions_query_runner.ambr @@ -22,26 +22,3 @@ use_hive_partitioning=0 ''' # --- -# name: TestSuggestedQuestionsQueryRunner.test_suggested_questions_hit_openai[new_events_schema] - ''' - SELECT events.event AS event, - count() AS count - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), greaterOrEquals(events.timestamp, minus(now64(6, 'UTC'), toIntervalDay(30))), notIn(events.event, ['$pageleave', '$autocapture', '$$heatmap', '$copy_autocapture', '$set', '$opt_in', '$feature_flag_called', '$feature_view', '$feature_interaction', '$element_viewed', '$capture_metrics', '$create_alias', '$merge_dangerously', '$groupidentify', 'mcp_tool_call', 'mcp_tools_list', 'mcp_initialize', 'mcp_resources_list', 'mcp_resource_read', 'mcp_prompts_list', 'mcp_prompt_get', 'mcp_custom', 'posthog_identify', 'mcp init', 'mcp_mcpcat:identify', 'mcp_posthog:identify', 'mcp_tool_called', 'mcp tool call', 'mcp tool response', '$snapshot'])) - GROUP BY events.event - ORDER BY count DESC, events.event ASC - LIMIT 501 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- diff --git a/posthog/hogql_queries/ai/test/__snapshots__/test_team_taxonomy_query_runner.ambr b/posthog/hogql_queries/ai/test/__snapshots__/test_team_taxonomy_query_runner.ambr index acdcd3248cca..8f0eb8b08e1d 100644 --- a/posthog/hogql_queries/ai/test/__snapshots__/test_team_taxonomy_query_runner.ambr +++ b/posthog/hogql_queries/ai/test/__snapshots__/test_team_taxonomy_query_runner.ambr @@ -22,26 +22,3 @@ use_hive_partitioning=0 ''' # --- -# name: TestTeamTaxonomyQueryRunner.test_taxonomy_query_runner[new_events_schema] - ''' - SELECT events.event AS event, - count() AS count - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), greaterOrEquals(events.timestamp, minus(now64(6, 'UTC'), toIntervalDay(30))), notIn(events.event, ['$pageleave', '$autocapture', '$$heatmap', '$copy_autocapture', '$set', '$opt_in', '$feature_flag_called', '$feature_view', '$feature_interaction', '$element_viewed', '$capture_metrics', '$create_alias', '$merge_dangerously', '$groupidentify', 'mcp_tool_call', 'mcp_tools_list', 'mcp_initialize', 'mcp_resources_list', 'mcp_resource_read', 'mcp_prompts_list', 'mcp_prompt_get', 'mcp_custom', 'posthog_identify', 'mcp init', 'mcp_mcpcat:identify', 'mcp_posthog:identify', 'mcp_tool_called', 'mcp tool call', 'mcp tool response', '$snapshot'])) - GROUP BY events.event - ORDER BY count DESC, events.event ASC - LIMIT 501 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- diff --git a/posthog/hogql_queries/ai/test/__snapshots__/test_trace_query_runner.ambr b/posthog/hogql_queries/ai/test/__snapshots__/test_trace_query_runner.ambr index 30e6e474982b..5feb8ab911dd 100644 --- a/posthog/hogql_queries/ai/test/__snapshots__/test_trace_query_runner.ambr +++ b/posthog/hogql_queries/ai/test/__snapshots__/test_trace_query_runner.ambr @@ -235,67 +235,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestTraceQueryRunner.test_event_property_filters[new_events_schema] - ''' - SELECT deduped.trace_id AS id, - any(deduped.session_id) AS ai_session_id, - min(deduped.timestamp) AS first_timestamp, - max(deduped.timestamp) AS last_timestamp, - ifNull(nullIf(argMinIf(deduped.distinct_id, deduped.timestamp, equals(deduped.event, '$ai_trace')), ''), argMin(deduped.distinct_id, deduped.timestamp)) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(deduped.latency, 0), 0), notEquals(deduped.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(deduped.latency, 0), 0), equals(deduped.event, '$ai_generation'))), 0)), sumIf(deduped.latency, and(equals(deduped.event, '$ai_generation'), ifNull(greater(deduped.latency, 0), 0))), sumIf(deduped.latency, or(isNull(deduped.parent_id), ifNull(equals(deduped.parent_id, deduped.trace_id), isNull(deduped.parent_id) - and isNull(deduped.trace_id))))), 2) AS total_latency, - nullIf(sumIf(deduped.input_tokens, in(deduped.event, tuple('$ai_generation', '$ai_embedding'))), 0) AS input_tokens, - nullIf(sumIf(deduped.output_tokens, in(deduped.event, tuple('$ai_generation', '$ai_embedding'))), 0) AS output_tokens, - nullIf(round(sumIf(deduped.input_cost_usd, in(deduped.event, tuple('$ai_generation', '$ai_embedding'))), 10), 0) AS input_cost, - nullIf(round(sumIf(deduped.output_cost_usd, in(deduped.event, tuple('$ai_generation', '$ai_embedding'))), 10), 0) AS output_cost, - nullIf(round(sumIf(deduped.total_cost_usd, in(deduped.event, tuple('$ai_generation', '$ai_embedding'))), 10), 0) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(deduped.uuid, deduped.event, deduped.timestamp, deduped.properties, deduped.input, deduped.output, deduped.output_choices, deduped.input_state, deduped.output_state, deduped.tools), notEquals(deduped.event, '$ai_trace')))) AS events, - argMinIf(deduped.input_state, deduped.timestamp, equals(deduped.event, '$ai_trace')) AS input_state, - argMinIf(deduped.output_state, deduped.timestamp, equals(deduped.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(nullIf(deduped.span_name, ''), nullIf(deduped.trace_name, '')), deduped.timestamp, equals(deduped.event, '$ai_trace')), argMin(ifNull(nullIf(deduped.span_name, ''), nullIf(deduped.trace_name, '')), deduped.timestamp)) AS trace_name - FROM - (SELECT __ai_events_fallback.uuid AS uuid, - __ai_events_fallback.event AS event, - toTimeZone(__ai_events_fallback.timestamp, 'UTC') AS timestamp, - __ai_events_fallback.distinct_id AS distinct_id, - concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(__ai_events_fallback.properties)))), ','), '}') AS properties, - __ai_events_fallback.properties.`$ai_trace_id` AS trace_id, - __ai_events_fallback.properties.`$ai_session_id` AS session_id, - __ai_events_fallback.properties.`$ai_parent_id` AS parent_id, - if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_span_name`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_span_name`), if(isNull(__ai_events_fallback.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_span_name`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_span_name`), toString(__ai_events_fallback.properties.`$ai_span_name`))))) AS span_name, - if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_trace_name`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_trace_name`), if(isNull(__ai_events_fallback.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_trace_name`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_trace_name`), toString(__ai_events_fallback.properties.`$ai_trace_name`))))) AS trace_name, - accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_latency`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_latency`), if(isNull(__ai_events_fallback.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_latency`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_latency`), toString(__ai_events_fallback.properties.`$ai_latency`))))), 'Float64'), 'Float64') AS latency, - accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_input_tokens`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_input_tokens`), if(isNull(__ai_events_fallback.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_input_tokens`), toString(__ai_events_fallback.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64') AS input_tokens, - accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_output_tokens`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_output_tokens`), if(isNull(__ai_events_fallback.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_output_tokens`), toString(__ai_events_fallback.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64') AS output_tokens, - accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_input_cost_usd`), if(isNull(__ai_events_fallback.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_input_cost_usd`), toString(__ai_events_fallback.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64') AS input_cost_usd, - accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_output_cost_usd`), if(isNull(__ai_events_fallback.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_output_cost_usd`), toString(__ai_events_fallback.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64') AS output_cost_usd, - accurateCastOrNull(accurateCastOrNull(__ai_events_fallback.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64') AS total_cost_usd, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_input`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_input`), if(isNull(__ai_events_fallback.properties.`$ai_input`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_input`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_input`)))), '')) AS input, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_output`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_output`), if(isNull(__ai_events_fallback.properties.`$ai_output`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_output`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_output`)))), '')) AS output, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_output_choices`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_output_choices`), if(isNull(__ai_events_fallback.properties.`$ai_output_choices`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_choices`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_output_choices`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_output_choices`)))), '')) AS output_choices, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_input_state`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_input_state`), if(isNull(__ai_events_fallback.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_state`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_input_state`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_input_state`)))), '')) AS input_state, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_output_state`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_output_state`), if(isNull(__ai_events_fallback.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_state`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_output_state`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_output_state`)))), '')) AS output_state, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_tools`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_tools`), if(isNull(__ai_events_fallback.properties.`$ai_tools`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_tools`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_tools`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_tools`)))), '')) AS tools - FROM events_json AS __ai_events_fallback - WHERE and(equals(__ai_events_fallback.team_id, 99999), in(__ai_events_fallback.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(__ai_events_fallback.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(__ai_events_fallback.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC'))), equals(__ai_events_fallback.properties.`$ai_trace_id`, 'trace1'), ifNull(equals(if(notEquals(toJSONString(__ai_events_fallback.properties.^foo), '{}'), toJSONString(__ai_events_fallback.properties.^foo), if(isNull(__ai_events_fallback.properties.foo), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.foo), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.foo), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.foo), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.foo), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.foo), 'Tuple')), toJSONString(__ai_events_fallback.properties.foo), toString(__ai_events_fallback.properties.foo))))), 'barz'), 0))) - LIMIT 1 BY __ai_events_fallback.uuid) AS deduped - GROUP BY deduped.trace_id - LIMIT 1 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestTraceQueryRunner.test_person_property_filters ''' SELECT deduped.trace_id AS id, @@ -526,81 +465,3 @@ use_hive_partitioning=0 ''' # --- -# name: TestTraceQueryRunner.test_person_property_filters[new_events_schema] - ''' - SELECT deduped.trace_id AS id, - any(deduped.session_id) AS ai_session_id, - min(deduped.timestamp) AS first_timestamp, - max(deduped.timestamp) AS last_timestamp, - ifNull(nullIf(argMinIf(deduped.distinct_id, deduped.timestamp, equals(deduped.event, '$ai_trace')), ''), argMin(deduped.distinct_id, deduped.timestamp)) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(deduped.latency, 0), 0), notEquals(deduped.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(deduped.latency, 0), 0), equals(deduped.event, '$ai_generation'))), 0)), sumIf(deduped.latency, and(equals(deduped.event, '$ai_generation'), ifNull(greater(deduped.latency, 0), 0))), sumIf(deduped.latency, or(isNull(deduped.parent_id), ifNull(equals(deduped.parent_id, deduped.trace_id), isNull(deduped.parent_id) - and isNull(deduped.trace_id))))), 2) AS total_latency, - nullIf(sumIf(deduped.input_tokens, in(deduped.event, tuple('$ai_generation', '$ai_embedding'))), 0) AS input_tokens, - nullIf(sumIf(deduped.output_tokens, in(deduped.event, tuple('$ai_generation', '$ai_embedding'))), 0) AS output_tokens, - nullIf(round(sumIf(deduped.input_cost_usd, in(deduped.event, tuple('$ai_generation', '$ai_embedding'))), 10), 0) AS input_cost, - nullIf(round(sumIf(deduped.output_cost_usd, in(deduped.event, tuple('$ai_generation', '$ai_embedding'))), 10), 0) AS output_cost, - nullIf(round(sumIf(deduped.total_cost_usd, in(deduped.event, tuple('$ai_generation', '$ai_embedding'))), 10), 0) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(deduped.uuid, deduped.event, deduped.timestamp, deduped.properties, deduped.input, deduped.output, deduped.output_choices, deduped.input_state, deduped.output_state, deduped.tools), notEquals(deduped.event, '$ai_trace')))) AS events, - argMinIf(deduped.input_state, deduped.timestamp, equals(deduped.event, '$ai_trace')) AS input_state, - argMinIf(deduped.output_state, deduped.timestamp, equals(deduped.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(nullIf(deduped.span_name, ''), nullIf(deduped.trace_name, '')), deduped.timestamp, equals(deduped.event, '$ai_trace')), argMin(ifNull(nullIf(deduped.span_name, ''), nullIf(deduped.trace_name, '')), deduped.timestamp)) AS trace_name - FROM - (SELECT __ai_events_fallback.uuid AS uuid, - __ai_events_fallback.event AS event, - toTimeZone(__ai_events_fallback.timestamp, 'UTC') AS timestamp, - __ai_events_fallback.distinct_id AS distinct_id, - concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(__ai_events_fallback.properties)))), ','), '}') AS properties, - __ai_events_fallback.properties.`$ai_trace_id` AS trace_id, - __ai_events_fallback.properties.`$ai_session_id` AS session_id, - __ai_events_fallback.properties.`$ai_parent_id` AS parent_id, - if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_span_name`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_span_name`), if(isNull(__ai_events_fallback.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_span_name`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_span_name`), toString(__ai_events_fallback.properties.`$ai_span_name`))))) AS span_name, - if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_trace_name`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_trace_name`), if(isNull(__ai_events_fallback.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_trace_name`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_trace_name`), toString(__ai_events_fallback.properties.`$ai_trace_name`))))) AS trace_name, - accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_latency`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_latency`), if(isNull(__ai_events_fallback.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_latency`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_latency`), toString(__ai_events_fallback.properties.`$ai_latency`))))), 'Float64'), 'Float64') AS latency, - accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_input_tokens`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_input_tokens`), if(isNull(__ai_events_fallback.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_input_tokens`), toString(__ai_events_fallback.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64') AS input_tokens, - accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_output_tokens`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_output_tokens`), if(isNull(__ai_events_fallback.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_output_tokens`), toString(__ai_events_fallback.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64') AS output_tokens, - accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_input_cost_usd`), if(isNull(__ai_events_fallback.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_input_cost_usd`), toString(__ai_events_fallback.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64') AS input_cost_usd, - accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_output_cost_usd`), if(isNull(__ai_events_fallback.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(__ai_events_fallback.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(__ai_events_fallback.properties.`$ai_output_cost_usd`), toString(__ai_events_fallback.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64') AS output_cost_usd, - accurateCastOrNull(accurateCastOrNull(__ai_events_fallback.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64') AS total_cost_usd, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_input`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_input`), if(isNull(__ai_events_fallback.properties.`$ai_input`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_input`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_input`)))), '')) AS input, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_output`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_output`), if(isNull(__ai_events_fallback.properties.`$ai_output`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_output`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_output`)))), '')) AS output, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_output_choices`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_output_choices`), if(isNull(__ai_events_fallback.properties.`$ai_output_choices`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_choices`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_output_choices`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_output_choices`)))), '')) AS output_choices, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_input_state`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_input_state`), if(isNull(__ai_events_fallback.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_input_state`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_input_state`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_input_state`)))), '')) AS input_state, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_output_state`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_output_state`), if(isNull(__ai_events_fallback.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_output_state`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_output_state`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_output_state`)))), '')) AS output_state, - JSONExtractRaw(ifNull(if(notEquals(toJSONString(__ai_events_fallback.properties.^`$ai_tools`), '{}'), toJSONString(__ai_events_fallback.properties.^`$ai_tools`), if(isNull(__ai_events_fallback.properties.`$ai_tools`), NULL, if(startsWith(dynamicType(__ai_events_fallback.properties.`$ai_tools`), 'DateTime'), concat('"', ifNull(toString(replaceOne(toString(__ai_events_fallback.properties.`$ai_tools`), ' ', 'T')), ''), '"'), toJSONString(__ai_events_fallback.properties.`$ai_tools`)))), '')) AS tools - FROM events_json AS __ai_events_fallback - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS __ai_events_fallback__override ON equals(__ai_events_fallback.distinct_id, __ai_events_fallback__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'bar'), ''), 'null'), '^"|"$', '') AS properties___bar - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS __ai_events_fallback__person ON equals(if(not(empty(__ai_events_fallback__override.distinct_id)), __ai_events_fallback__override.person_id, __ai_events_fallback.person_id), __ai_events_fallback__person.id) - WHERE and(equals(__ai_events_fallback.team_id, 99999), in(__ai_events_fallback.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(__ai_events_fallback.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(__ai_events_fallback.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC'))), equals(__ai_events_fallback.properties.`$ai_trace_id`, 'trace1'), ifNull(equals(__ai_events_fallback__person.properties___bar, 'foo'), 0))) - LIMIT 1 BY __ai_events_fallback.uuid) AS deduped - GROUP BY deduped.trace_id - LIMIT 1 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- diff --git a/posthog/hogql_queries/ai/test/__snapshots__/test_traces_query_runner.ambr b/posthog/hogql_queries/ai/test/__snapshots__/test_traces_query_runner.ambr index f69981660d2d..e54f6d0d0bb3 100644 --- a/posthog/hogql_queries/ai/test/__snapshots__/test_traces_query_runner.ambr +++ b/posthog/hogql_queries/ai/test/__snapshots__/test_traces_query_runner.ambr @@ -71,78 +71,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_field_mapping[new_events_schema.1] - ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-13 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 01:10:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace1', 'trace2'))) - GROUP BY events.properties.`$ai_trace_id` - ORDER BY first_timestamp DESC - LIMIT 101 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_field_mapping[new_events_schema] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestTracesQueryRunner.test_group_key_filter ''' SELECT groupArray(trace_id) AS trace_ids, @@ -460,64 +388,21 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_group_key_filter[new_events_schema.1] - ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-14 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 02:10:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace_project_1', 'trace_org_b', 'trace_org_a'))) - GROUP BY events.properties.`$ai_trace_id` - ORDER BY first_timestamp DESC - LIMIT 101 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_group_key_filter[new_events_schema.2] +# name: TestTracesQueryRunner.test_pagination ''' SELECT groupArray(trace_id) AS trace_ids, min(first_ts) AS min_timestamp, max(last_ts) AS max_timestamp FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, + (SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS trace_id, min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))), equals(events.`$group_0`, 'org:acme'))) + FROM events + WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))))) GROUP BY trace_id HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) + LIMIT 5) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -532,35 +417,35 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_group_key_filter[new_events_schema.3] +# name: TestTracesQueryRunner.test_pagination.1 ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, + SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS id, + any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_session_id'), ''), 'null'), '^"|"$', '')) AS ai_session_id, min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-14 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 00:10:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace_org_a'))) - GROUP BY events.properties.`$ai_trace_id` + round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) + and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))))))), 2) AS total_latency, + sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, + sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, + round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, + round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, + round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_request_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, + round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_web_search_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, + round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_total_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, + arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), events.properties), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) + and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')))))))) AS events, + argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, + argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, + ifNull(argMinIf(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, + countIf(or(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_error'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_is_error'), ''), 'null'), '^"|"$', ''), 'true'), 0))) AS error_count, + any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'ai_support_impersonated'), ''), 'null'), '^"|"$', '')) AS is_support_trace, + arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), and(equals(events.event, '$ai_generation'), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ''), 1))), ',')))) AS tools + FROM events + WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 05:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 10:10:00', 'UTC'))))), in(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), tuple('trace_10', 'trace_9', 'trace_8', 'trace_7', 'trace_6'))) + GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') ORDER BY first_timestamp DESC - LIMIT 101 + LIMIT 5 OFFSET 0 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -575,21 +460,21 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_group_key_filter[new_events_schema.4] +# name: TestTracesQueryRunner.test_pagination.2 ''' SELECT groupArray(trace_id) AS trace_ids, min(first_ts) AS min_timestamp, max(last_ts) AS max_timestamp FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, + (SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS trace_id, min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))), equals(events.`$group_0`, 'org:widgets'))) + FROM events + WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))))) GROUP BY trace_id HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) + LIMIT 10) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -604,36 +489,36 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_group_key_filter[new_events_schema.5] +# name: TestTracesQueryRunner.test_pagination.3 ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, + SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS id, + any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_session_id'), ''), 'null'), '^"|"$', '')) AS ai_session_id, min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 00:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 01:10:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace_org_b'))) - GROUP BY events.properties.`$ai_trace_id` + round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) + and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))))))), 2) AS total_latency, + sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, + sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, + round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, + round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, + round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_request_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, + round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_web_search_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, + round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_total_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, + arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), events.properties), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) + and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')))))))) AS events, + argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, + argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, + ifNull(argMinIf(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, + countIf(or(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_error'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_is_error'), ''), 'null'), '^"|"$', ''), 'true'), 0))) AS error_count, + any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'ai_support_impersonated'), ''), 'null'), '^"|"$', '')) AS is_support_trace, + arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), and(equals(events.event, '$ai_generation'), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ''), 1))), ',')))) AS tools + FROM events + WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 00:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 10:10:00', 'UTC'))))), in(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), tuple('trace_10', 'trace_9', 'trace_8', 'trace_7', 'trace_6', 'trace_5', 'trace_4', 'trace_3', 'trace_2', 'trace_1'))) + GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') ORDER BY first_timestamp DESC - LIMIT 101 - OFFSET 0 SETTINGS readonly=2, + LIMIT 5 + OFFSET 5 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, @@ -647,21 +532,21 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_group_key_filter[new_events_schema.6] +# name: TestTracesQueryRunner.test_pagination.4 ''' SELECT groupArray(trace_id) AS trace_ids, min(first_ts) AS min_timestamp, max(last_ts) AS max_timestamp FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, + (SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS trace_id, min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))), equals(events.`$group_1`, 'project:alpha'))) + FROM events + WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))))) GROUP BY trace_id HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) + LIMIT 15) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -676,94 +561,36 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_group_key_filter[new_events_schema.7] +# name: TestTracesQueryRunner.test_pagination.5 ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, + SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS id, + any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_session_id'), ''), 'null'), '^"|"$', '')) AS ai_session_id, min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 01:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 02:10:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace_project_1'))) - GROUP BY events.properties.`$ai_trace_id` + round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) + and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))))))), 2) AS total_latency, + sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, + sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, + round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, + round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, + round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_request_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, + round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_web_search_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, + round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_total_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, + arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), events.properties), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) + and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')))))))) AS events, + argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, + argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, + ifNull(argMinIf(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, + countIf(or(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_error'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_is_error'), ''), 'null'), '^"|"$', ''), 'true'), 0))) AS error_count, + any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'ai_support_impersonated'), ''), 'null'), '^"|"$', '')) AS is_support_trace, + arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), and(equals(events.event, '$ai_generation'), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ''), 1))), ',')))) AS tools + FROM events + WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-14 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 10:10:00', 'UTC'))))), in(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), tuple('trace_10', 'trace_9', 'trace_8', 'trace_7', 'trace_6', 'trace_5', 'trace_4', 'trace_3', 'trace_2', 'trace_1', 'trace_0'))) + GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') ORDER BY first_timestamp DESC - LIMIT 101 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_group_key_filter[new_events_schema.8] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))), equals(events.`$group_0`, 'nonexistent'))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_group_key_filter[new_events_schema] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, + LIMIT 5 + OFFSET 10 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, @@ -777,7 +604,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_pagination +# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group ''' SELECT groupArray(trace_id) AS trace_ids, min(first_ts) AS min_timestamp, @@ -787,11 +614,11 @@ min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, max(toTimeZone(events.timestamp, 'UTC')) AS last_ts FROM events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))))) + WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'foo'), ''), 'null'), '^"|"$', ''), 'bar'), 0))) GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) + HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 5) + LIMIT 101) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -806,7 +633,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_pagination.1 +# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group.1 ''' SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS id, any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_session_id'), ''), 'null'), '^"|"$', '')) AS ai_session_id, @@ -831,10 +658,10 @@ any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'ai_support_impersonated'), ''), 'null'), '^"|"$', '')) AS is_support_trace, arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), and(equals(events.event, '$ai_generation'), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ''), 1))), ',')))) AS tools FROM events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 05:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 10:10:00', 'UTC'))))), in(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), tuple('trace_10', 'trace_9', 'trace_8', 'trace_7', 'trace_6'))) + WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))))), in(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), tuple('trace1'))) GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') ORDER BY first_timestamp DESC - LIMIT 5 + LIMIT 101 OFFSET 0 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -849,7 +676,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_pagination.2 +# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group.2 ''' SELECT groupArray(trace_id) AS trace_ids, min(first_ts) AS min_timestamp, @@ -859,11 +686,11 @@ min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, max(toTimeZone(events.timestamp, 'UTC')) AS last_ts FROM events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))))) + WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'foo'), ''), 'null'), '^"|"$', ''), 'baz'), 0))) GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) + HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 10) + LIMIT 101) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -878,7 +705,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_pagination.3 +# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group.3 ''' SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS id, any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_session_id'), ''), 'null'), '^"|"$', '')) AS ai_session_id, @@ -903,11 +730,11 @@ any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'ai_support_impersonated'), ''), 'null'), '^"|"$', '')) AS is_support_trace, arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), and(equals(events.event, '$ai_generation'), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ''), 1))), ',')))) AS tools FROM events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 00:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 10:10:00', 'UTC'))))), in(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), tuple('trace_10', 'trace_9', 'trace_8', 'trace_7', 'trace_6', 'trace_5', 'trace_4', 'trace_3', 'trace_2', 'trace_1'))) + WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))))), in(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), tuple('trace1'))) GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') ORDER BY first_timestamp DESC - LIMIT 5 - OFFSET 5 SETTINGS readonly=2, + LIMIT 101 + OFFSET 0 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, @@ -921,7 +748,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_pagination.4 +# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group.4 ''' SELECT groupArray(trace_id) AS trace_ids, min(first_ts) AS min_timestamp, @@ -931,11 +758,11 @@ min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, max(toTimeZone(events.timestamp, 'UTC')) AS last_ts FROM events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))))) + WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'foo'), ''), 'null'), '^"|"$', ''), 'barz'), 0))) GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) + HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 15) + LIMIT 101) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -950,612 +777,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_pagination.5 - ''' - SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS id, - any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_session_id'), ''), 'null'), '^"|"$', '')) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) - and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_request_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_web_search_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_total_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), events.properties), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) - and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')))))))) AS events, - argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_error'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_is_error'), ''), 'null'), '^"|"$', ''), 'true'), 0))) AS error_count, - any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'ai_support_impersonated'), ''), 'null'), '^"|"$', '')) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), and(equals(events.event, '$ai_generation'), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ''), 1))), ',')))) AS tools - FROM events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-14 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 10:10:00', 'UTC'))))), in(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), tuple('trace_10', 'trace_9', 'trace_8', 'trace_7', 'trace_6', 'trace_5', 'trace_4', 'trace_3', 'trace_2', 'trace_1', 'trace_0'))) - GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') - ORDER BY first_timestamp DESC - LIMIT 5 - OFFSET 10 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_pagination[new_events_schema.1] - ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 05:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 10:10:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace_10', 'trace_9', 'trace_8', 'trace_7', 'trace_6'))) - GROUP BY events.properties.`$ai_trace_id` - ORDER BY first_timestamp DESC - LIMIT 5 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_pagination[new_events_schema.2] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 10) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_pagination[new_events_schema.3] - ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 00:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 10:10:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace_10', 'trace_9', 'trace_8', 'trace_7', 'trace_6', 'trace_5', 'trace_4', 'trace_3', 'trace_2', 'trace_1'))) - GROUP BY events.properties.`$ai_trace_id` - ORDER BY first_timestamp DESC - LIMIT 5 - OFFSET 5 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_pagination[new_events_schema.4] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 15) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_pagination[new_events_schema.5] - ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-14 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-15 10:10:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace_10', 'trace_9', 'trace_8', 'trace_7', 'trace_6', 'trace_5', 'trace_4', 'trace_3', 'trace_2', 'trace_1', 'trace_0'))) - GROUP BY events.properties.`$ai_trace_id` - ORDER BY first_timestamp DESC - LIMIT 5 - OFFSET 10 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_pagination[new_events_schema] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-08 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2025-01-16 00:10:59', 'UTC')))))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-16 00:00:59', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2025-01-09 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 5) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'foo'), ''), 'null'), '^"|"$', ''), 'bar'), 0))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group.1 - ''' - SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS id, - any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_session_id'), ''), 'null'), '^"|"$', '')) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) - and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_request_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_web_search_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_total_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), events.properties), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) - and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')))))))) AS events, - argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_error'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_is_error'), ''), 'null'), '^"|"$', ''), 'true'), 0))) AS error_count, - any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'ai_support_impersonated'), ''), 'null'), '^"|"$', '')) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), and(equals(events.event, '$ai_generation'), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ''), 1))), ',')))) AS tools - FROM events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))))), in(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), tuple('trace1'))) - GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') - ORDER BY first_timestamp DESC - LIMIT 101 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group.2 - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'foo'), ''), 'null'), '^"|"$', ''), 'baz'), 0))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group.3 - ''' - SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS id, - any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_session_id'), ''), 'null'), '^"|"$', '')) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_latency'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), or(isNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) - and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_tokens'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_request_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_web_search_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_total_cost_usd'), ''), 'null'), '^"|"$', ''), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), events.properties), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), ifNull(equals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', '')), toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''))), isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_parent_id'), ''), 'null'), '^"|"$', ''))) - and isNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')))))))) AS events, - argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_input_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_output_state'), ''), 'null'), '^"|"$', ''), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_span_name'), ''), 'null'), '^"|"$', ''), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_name'), ''), 'null'), '^"|"$', '')), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_error'), ''), 'null'), '^"|"$', '')), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_is_error'), ''), 'null'), '^"|"$', ''), 'true'), 0))) AS error_count, - any(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'ai_support_impersonated'), ''), 'null'), '^"|"$', '')) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), and(equals(events.event, '$ai_generation'), isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_tools_called'), ''), 'null'), '^"|"$', '')), ''), 1))), ',')))) AS tools - FROM events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))))), in(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), tuple('trace1'))) - GROUP BY replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') - ORDER BY first_timestamp DESC - LIMIT 101 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group.4 - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '') AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', '')), ifNull(notEquals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$ai_trace_id'), ''), 'null'), '^"|"$', ''), ''), 1), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, 'foo'), ''), 'null'), '^"|"$', ''), 'barz'), 0))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group[new_events_schema.1] - ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace1'))) - GROUP BY events.properties.`$ai_trace_id` - ORDER BY first_timestamp DESC - LIMIT 101 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group[new_events_schema.2] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), ifNull(equals(if(notEquals(toJSONString(events.properties.^foo), '{}'), toJSONString(events.properties.^foo), if(isNull(events.properties.foo), NULL, if(startsWith(dynamicType(events.properties.foo), 'DateTime'), replaceOne(toString(events.properties.foo), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.foo), 'Array'), startsWith(dynamicType(events.properties.foo), 'Map'), startsWith(dynamicType(events.properties.foo), 'Tuple')), toJSONString(events.properties.foo), toString(events.properties.foo))))), 'baz'), 0))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group[new_events_schema.3] - ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace1'))) - GROUP BY events.properties.`$ai_trace_id` - ORDER BY first_timestamp DESC - LIMIT 101 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group[new_events_schema.4] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), ifNull(equals(if(notEquals(toJSONString(events.properties.^foo), '{}'), toJSONString(events.properties.^foo), if(isNull(events.properties.foo), NULL, if(startsWith(dynamicType(events.properties.foo), 'DateTime'), replaceOne(toString(events.properties.foo), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.foo), 'Array'), startsWith(dynamicType(events.properties.foo), 'Map'), startsWith(dynamicType(events.properties.foo), 'Tuple')), toJSONString(events.properties.foo), toString(events.properties.foo))))), 'barz'), 0))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_properties_filter_with_multiple_events_in_group[new_events_schema] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), ifNull(equals(if(notEquals(toJSONString(events.properties.^foo), '{}'), toJSONString(events.properties.^foo), if(isNull(events.properties.foo), NULL, if(startsWith(dynamicType(events.properties.foo), 'DateTime'), replaceOne(toString(events.properties.foo), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.foo), 'Array'), startsWith(dynamicType(events.properties.foo), 'Map'), startsWith(dynamicType(events.properties.foo), 'Tuple')), toJSONString(events.properties.foo), toString(events.properties.foo))))), 'bar'), 0))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_trace_property_filter_for_event_group +# name: TestTracesQueryRunner.test_trace_property_filter_for_event_group ''' SELECT groupArray(trace_id) AS trace_ids, min(first_ts) AS min_timestamp, @@ -1728,176 +950,3 @@ use_hive_partitioning=0 ''' # --- -# name: TestTracesQueryRunner.test_trace_property_filter_for_event_group[new_events_schema.1] - ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:55:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:15:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace1'))) - GROUP BY events.properties.`$ai_trace_id` - ORDER BY first_timestamp DESC - LIMIT 101 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_trace_property_filter_for_event_group[new_events_schema.2] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), ifNull(equals(if(notEquals(toJSONString(events.properties.^foo), '{}'), toJSONString(events.properties.^foo), if(isNull(events.properties.foo), NULL, if(startsWith(dynamicType(events.properties.foo), 'DateTime'), replaceOne(toString(events.properties.foo), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.foo), 'Array'), startsWith(dynamicType(events.properties.foo), 'Map'), startsWith(dynamicType(events.properties.foo), 'Tuple')), toJSONString(events.properties.foo), toString(events.properties.foo))))), 'bar'), 0))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_trace_property_filter_for_event_group[new_events_schema.3] - ''' - SELECT events.properties.`$ai_trace_id` AS id, - any(events.properties.`$ai_session_id`) AS ai_session_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_timestamp, - max(toTimeZone(events.timestamp, 'UTC')) AS last_timestamp, - ifNull(nullIf(argMinIf(events.distinct_id, toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), ''), argMin(events.distinct_id, toTimeZone(events.timestamp, 'UTC'))) AS first_distinct_id, - round(if(and(equals(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), notEquals(events.event, '$ai_generation'))), 0), greater(countIf(and(ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0), equals(events.event, '$ai_generation'))), 0)), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), and(equals(events.event, '$ai_generation'), ifNull(greater(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), 0), 0))), sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_latency`), '{}'), toJSONString(events.properties.^`$ai_latency`), if(isNull(events.properties.`$ai_latency`), NULL, if(startsWith(dynamicType(events.properties.`$ai_latency`), 'DateTime'), replaceOne(toString(events.properties.`$ai_latency`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_latency`), 'Array'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Map'), startsWith(dynamicType(events.properties.`$ai_latency`), 'Tuple')), toJSONString(events.properties.`$ai_latency`), toString(events.properties.`$ai_latency`))))), 'Float64'), 'Float64'), or(isNull(events.properties.`$ai_parent_id`), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`))))), 2) AS total_latency, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_tokens`), '{}'), toJSONString(events.properties.^`$ai_input_tokens`), if(isNull(events.properties.`$ai_input_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_input_tokens`), toString(events.properties.`$ai_input_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS input_tokens, - sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_tokens`), '{}'), toJSONString(events.properties.^`$ai_output_tokens`), if(isNull(events.properties.`$ai_output_tokens`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_tokens`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_tokens`), 'Tuple')), toJSONString(events.properties.`$ai_output_tokens`), toString(events.properties.`$ai_output_tokens`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))) AS output_tokens, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_input_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_input_cost_usd`), if(isNull(events.properties.`$ai_input_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_input_cost_usd`), toString(events.properties.`$ai_input_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS input_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_output_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_output_cost_usd`), if(isNull(events.properties.`$ai_output_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_output_cost_usd`), toString(events.properties.`$ai_output_cost_usd`))))), 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS output_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_request_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_request_cost_usd`), if(isNull(events.properties.`$ai_request_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_request_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_request_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_request_cost_usd`), toString(events.properties.`$ai_request_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS request_cost, - round(sumIf(accurateCastOrNull(if(notEquals(toJSONString(events.properties.^`$ai_web_search_cost_usd`), '{}'), toJSONString(events.properties.^`$ai_web_search_cost_usd`), if(isNull(events.properties.`$ai_web_search_cost_usd`), NULL, if(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'DateTime'), replaceOne(toString(events.properties.`$ai_web_search_cost_usd`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Array'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Map'), startsWith(dynamicType(events.properties.`$ai_web_search_cost_usd`), 'Tuple')), toJSONString(events.properties.`$ai_web_search_cost_usd`), toString(events.properties.`$ai_web_search_cost_usd`))))), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS web_search_cost, - round(sumIf(accurateCastOrNull(accurateCastOrNull(events.properties.`$ai_total_cost_usd`, 'Float64'), 'Float64'), in(events.event, tuple('$ai_generation', '$ai_embedding'))), 10) AS total_cost, - arrayDistinct(arraySort(x -> x.3, groupArrayIf(tuple(events.uuid, events.event, toTimeZone(events.timestamp, 'UTC'), concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}')), or(in(events.event, tuple('$ai_metric', '$ai_feedback')), equals(toString(events.properties.`$ai_parent_id`), toString(events.properties.`$ai_trace_id`)))))) AS events, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_input_state`), '{}'), toJSONString(events.properties.^`$ai_input_state`), if(isNull(events.properties.`$ai_input_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_input_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_input_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_input_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_input_state`), 'Tuple')), toJSONString(events.properties.`$ai_input_state`), toString(events.properties.`$ai_input_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS input_state, - argMinIf(if(notEquals(toJSONString(events.properties.^`$ai_output_state`), '{}'), toJSONString(events.properties.^`$ai_output_state`), if(isNull(events.properties.`$ai_output_state`), NULL, if(startsWith(dynamicType(events.properties.`$ai_output_state`), 'DateTime'), replaceOne(toString(events.properties.`$ai_output_state`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_output_state`), 'Array'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Map'), startsWith(dynamicType(events.properties.`$ai_output_state`), 'Tuple')), toJSONString(events.properties.`$ai_output_state`), toString(events.properties.`$ai_output_state`))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')) AS output_state, - ifNull(argMinIf(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'), equals(events.event, '$ai_trace')), argMin(ifNull(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), if(notEquals(toJSONString(events.properties.^`$ai_trace_name`), '{}'), toJSONString(events.properties.^`$ai_trace_name`), if(isNull(events.properties.`$ai_trace_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_trace_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_trace_name`), 'Tuple')), toJSONString(events.properties.`$ai_trace_name`), toString(events.properties.`$ai_trace_name`)))))), toTimeZone(events.timestamp, 'UTC'))) AS trace_name, - countIf(or(isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_error`), '{}'), toJSONString(events.properties.^`$ai_error`), if(isNull(events.properties.`$ai_error`), NULL, if(startsWith(dynamicType(events.properties.`$ai_error`), 'DateTime'), replaceOne(toString(events.properties.`$ai_error`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_error`), 'Array'), startsWith(dynamicType(events.properties.`$ai_error`), 'Map'), startsWith(dynamicType(events.properties.`$ai_error`), 'Tuple')), toJSONString(events.properties.`$ai_error`), toString(events.properties.`$ai_error`)))))), equals(events.properties.`$ai_is_error`, 'true'))) AS error_count, - any(if(notEquals(toJSONString(events.properties.^ai_support_impersonated), '{}'), toJSONString(events.properties.^ai_support_impersonated), if(isNull(events.properties.ai_support_impersonated), NULL, if(startsWith(dynamicType(events.properties.ai_support_impersonated), 'DateTime'), replaceOne(toString(events.properties.ai_support_impersonated), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.ai_support_impersonated), 'Array'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Map'), startsWith(dynamicType(events.properties.ai_support_impersonated), 'Tuple')), toJSONString(events.properties.ai_support_impersonated), toString(events.properties.ai_support_impersonated)))))) AS is_support_trace, - arrayFilter(x -> ifNull(notEquals(x, ''), 1), arrayDistinct(splitByChar(',', arrayStringConcat(groupArrayIf(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), and(equals(events.event, '$ai_generation'), isNotNull(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ifNull(notEquals(toString(if(notEquals(toJSONString(events.properties.^`$ai_tools_called`), '{}'), toJSONString(events.properties.^`$ai_tools_called`), if(isNull(events.properties.`$ai_tools_called`), NULL, if(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'DateTime'), replaceOne(toString(events.properties.`$ai_tools_called`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Array'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Map'), startsWith(dynamicType(events.properties.`$ai_tools_called`), 'Tuple')), toJSONString(events.properties.`$ai_tools_called`), toString(events.properties.`$ai_tools_called`)))))), ''), 1))), ',')))) AS tools - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), and(in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:52:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:12:00', 'UTC'))))), in(events.properties.`$ai_trace_id`, tuple('trace1'))) - GROUP BY events.properties.`$ai_trace_id` - ORDER BY first_timestamp DESC - LIMIT 101 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_trace_property_filter_for_event_group[new_events_schema.4] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), and(ifNull(equals(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), 'runnable'), 0), ifNull(equals(if(notEquals(toJSONString(events.properties.^foo), '{}'), toJSONString(events.properties.^foo), if(isNull(events.properties.foo), NULL, if(startsWith(dynamicType(events.properties.foo), 'DateTime'), replaceOne(toString(events.properties.foo), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.foo), 'Array'), startsWith(dynamicType(events.properties.foo), 'Map'), startsWith(dynamicType(events.properties.foo), 'Tuple')), toJSONString(events.properties.foo), toString(events.properties.foo))))), 'bar'), 0)))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestTracesQueryRunner.test_trace_property_filter_for_event_group[new_events_schema] - ''' - SELECT groupArray(trace_id) AS trace_ids, - min(first_ts) AS min_timestamp, - max(last_ts) AS max_timestamp - FROM - (SELECT events.properties.`$ai_trace_id` AS trace_id, - min(toTimeZone(events.timestamp, 'UTC')) AS first_ts, - max(toTimeZone(events.timestamp, 'UTC')) AS last_ts - FROM events_json AS events - WHERE and(equals(events.team_id, 99999), in(events.event, tuple('$ai_span', '$ai_generation', '$ai_embedding', '$ai_metric', '$ai_feedback', '$ai_trace')), and(isNotNull(events.properties.`$ai_trace_id`), notEquals(events.properties.`$ai_trace_id`, ''), and(greaterOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-11-30 23:50:00', 'UTC'))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2024-12-01 00:20:00', 'UTC')))), ifNull(equals(if(notEquals(toJSONString(events.properties.^`$ai_span_name`), '{}'), toJSONString(events.properties.^`$ai_span_name`), if(isNull(events.properties.`$ai_span_name`), NULL, if(startsWith(dynamicType(events.properties.`$ai_span_name`), 'DateTime'), replaceOne(toString(events.properties.`$ai_span_name`), ' ', 'T'), if(or(startsWith(dynamicType(events.properties.`$ai_span_name`), 'Array'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Map'), startsWith(dynamicType(events.properties.`$ai_span_name`), 'Tuple')), toJSONString(events.properties.`$ai_span_name`), toString(events.properties.`$ai_span_name`))))), 'runnable'), 0))) - GROUP BY trace_id - HAVING and(lessOrEquals(min(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:10:00', 'UTC'))), greaterOrEquals(max(toTimeZone(events.timestamp, 'UTC')), assumeNotNull(toDateTime('2024-12-01 00:00:00', 'UTC')))) - ORDER BY min(toTimeZone(events.timestamp, 'UTC')) DESC - LIMIT 101) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- diff --git a/posthog/hogql_queries/groups/test/__snapshots__/test_groups_query_runner.ambr b/posthog/hogql_queries/groups/test/__snapshots__/test_groups_query_runner.ambr index 4b1113108488..c1c5a43f7714 100644 --- a/posthog/hogql_queries/groups/test/__snapshots__/test_groups_query_runner.ambr +++ b/posthog/hogql_queries/groups/test/__snapshots__/test_groups_query_runner.ambr @@ -59,33 +59,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestGroupsQueryRunner.test_groups_query_runner_search_ranking - ''' - SELECT coalesce(toString(groups.properties___name), toString(groups.key)) AS `coalesce(toString(properties.name), toString(key))`, - groups.key AS key - FROM - (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'name'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___name, - argMax(toTimeZone(groups.created_at, 'UTC'), toTimeZone(groups._timestamp, 'UTC')) AS created_at, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE and(ifNull(equals(groups.index, 0), 0), or(ifNull(ilike(groups.properties___name, '%test%'), 0), ilike(toString(groups.key), '%test%'))) - ORDER BY multiIf(ifNull(ilike(coalesce(toString(groups.properties___name), toString(groups.key)), 'test'), 0), 0, ifNull(ilike(coalesce(toString(groups.properties___name), toString(groups.key)), 'test%'), 0), 1, 2) ASC, groups.created_at DESC - LIMIT 11 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1 - ''' -# --- # name: TestGroupsQueryRunner.test_groups_query_runner_with_numeric_property ''' SELECT tuple(coalesce(toString(groups.properties___name), toString(groups.key)), toString(groups.key)) AS `tuple(coalesce(toString(properties.name), toString(key)), toString(key))`, @@ -267,33 +240,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestGroupsQueryRunner.test_groups_query_runner_with_search - ''' - SELECT coalesce(toString(groups.properties___name), toString(groups.key)) AS `coalesce(toString(properties.name), toString(key))`, - groups.key AS key - FROM - (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'name'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___name, - argMax(toTimeZone(groups.created_at, 'UTC'), toTimeZone(groups._timestamp, 'UTC')) AS created_at, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE and(ifNull(equals(groups.index, 0), 0), or(ifNull(ilike(groups.properties___name, '%org2%'), 0), ilike(toString(groups.key), '%org2%'))) - ORDER BY multiIf(ifNull(ilike(coalesce(toString(groups.properties___name), toString(groups.key)), 'org2'), 0), 0, ifNull(ilike(coalesce(toString(groups.properties___name), toString(groups.key)), 'org2%'), 0), 1, 2) ASC, groups.created_at DESC - LIMIT 11 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1 - ''' -# --- # name: TestGroupsQueryRunner.test_groups_query_runner_with_string_property ''' SELECT tuple(coalesce(toString(groups.properties___name), toString(groups.key)), toString(groups.key)) AS `tuple(coalesce(toString(properties.name), toString(key)), toString(key))` @@ -384,35 +330,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestGroupsQueryRunner.test_search_ordering_with_user_orderby.1 - ''' - SELECT coalesce(toString(groups.properties___name), toString(groups.key)) AS `coalesce(toString(properties.name), toString(key))`, - groups.key AS key, - groups.properties___arr AS arr - FROM - (SELECT argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'name'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___name, - argMax(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'arr'), ''), 'null'), '^"|"$', ''), toTimeZone(groups._timestamp, 'UTC')) AS properties___arr, - argMax(toTimeZone(groups.created_at, 'UTC'), toTimeZone(groups._timestamp, 'UTC')) AS created_at, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE and(ifNull(equals(groups.index, 0), 0), or(ifNull(ilike(groups.properties___name, '%test%'), 0), ilike(toString(groups.key), '%test%'))) - ORDER BY multiIf(ifNull(ilike(coalesce(toString(groups.properties___name), toString(groups.key)), 'test'), 0), 0, ifNull(ilike(coalesce(toString(groups.properties___name), toString(groups.key)), 'test%'), 0), 1, 2) ASC, groups.created_at DESC - LIMIT 11 - OFFSET 0 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1 - ''' -# --- # name: TestGroupsQueryRunner.test_search_ranking ''' SELECT tuple(coalesce(toString(groups.properties___name), toString(groups.key)), toString(groups.key)) AS `tuple(coalesce(toString(properties.name), toString(key)), toString(key))` diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr index 46304b04155b..a746b2ae080c 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel.ambr @@ -58,65 +58,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_aggregation_with_groups[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - breakdown AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 100 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestFOSSFunnelUDF.test_funnel_aggregation_with_groups_across_persons ''' SELECT sum(step_1) AS step_1, @@ -184,73 +125,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_aggregation_with_groups_across_persons[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - breakdown AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 100 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestFOSSFunnelUDF.test_funnel_conversion_window_seconds ''' SELECT sum(step_1) AS step_1, @@ -383,65 +257,16 @@ use_hive_partitioning=0 ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_conversion_window_seconds[new_events_schema.1] +# name: TestFOSSFunnelUDF.test_funnel_events_with_person_on_events_v2 ''' - SELECT source.id, - source.id AS id, - 1 - FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(3, 15, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 1) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 + + SELECT DISTINCT person_id + FROM events + WHERE team_id = 99999 + AND distinct_id = 'stopped_after_pay' ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_conversion_window_seconds[new_events_schema] +# name: TestFOSSFunnelUDF.test_funnel_events_with_person_on_events_v2.1 ''' SELECT sum(step_1) AS step_1, sum(step_2) AS step_2, @@ -465,7 +290,7 @@ FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(3, 15, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -478,10 +303,10 @@ e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, - if(equals(e.event, 'step one'), 1, 0) AS step_0, - if(equals(e.event, 'step two'), 1, 0) AS step_1, - if(equals(e.event, 'step three'), 1, 0) AS step_2 - FROM events_json AS e + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(and(equals(e.event, '$autocapture'), match(e.elements_chain, '(^|;)button(\\.|$|;|:)'), arrayExists(x -> ifNull(equals(x, 'Pay $10'), 0), e.elements_chain_texts)), 1, 0) AS step_1, + if(and(equals(e.event, '$autocapture'), match(e.elements_chain, '(^|;)a(\\.|$|;|:)'), equals(e.elements_chain_href, '/movie')), 1, 0) AS step_2 + FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id @@ -489,7 +314,7 @@ WHERE equals(person_distinct_id_overrides.team_id, 99999) GROUP BY person_distinct_id_overrides.distinct_id HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('step one', 'step three', 'step two'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$autocapture', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown @@ -515,40 +340,36 @@ use_hive_partitioning=0 ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_events_with_person_on_events_v2 +# name: TestFOSSFunnelUDF.test_funnel_with_precalculated_cohort_step_filter ''' - SELECT DISTINCT person_id - FROM events + SELECT count(DISTINCT person_id) + FROM cohortpeople WHERE team_id = 99999 - AND distinct_id = 'stopped_after_pay' + AND cohort_id = 99999 + AND version = 0 ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_events_with_person_on_events_v2.1 +# name: TestFOSSFunnelUDF.test_funnel_with_precalculated_cohort_step_filter.1 ''' SELECT sum(step_1) AS step_1, sum(step_2) AS step_2, - sum(step_3) AS step_3, arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, groupArray(row_number) AS row_number, final_prop AS final_prop FROM (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, + groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, rowNumberInAllBlocks() AS row_number, breakdown AS final_prop FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, + (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -561,9 +382,11 @@ e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(and(equals(e.event, '$autocapture'), match(e.elements_chain, '(^|;)button(\\.|$|;|:)'), arrayExists(x -> ifNull(equals(x, 'Pay $10'), 0), e.elements_chain_texts)), 1, 0) AS step_1, - if(and(equals(e.event, '$autocapture'), match(e.elements_chain, '(^|;)a(\\.|$|;|:)'), equals(e.elements_chain_href, '/movie')), 1, 0) AS step_2 + if(and(equals(e.event, 'user signed up'), in(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), + (SELECT cohortpeople.person_id AS person_id + FROM cohortpeople + WHERE and(equals(cohortpeople.team_id, 99999), equals(cohortpeople.cohort_id, 99999), equals(cohortpeople.version, 0))))), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, @@ -572,16 +395,14 @@ WHERE equals(person_distinct_id_overrides.team_id, 99999) GROUP BY person_distinct_id_overrides.distinct_id HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$autocapture', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown - ORDER BY step_3 DESC, - step_2 DESC, + ORDER BY step_2 DESC, step_1 DESC) GROUP BY final_prop - ORDER BY step_3 DESC, - step_2 DESC, + ORDER BY step_2 DESC, step_1 DESC LIMIT 100 SETTINGS join_algorithm='auto', readonly=2, @@ -598,7 +419,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_events_with_person_on_events_v2[new_events_schema.1] +# name: TestFOSSFunnelUDF.test_funnel_with_property_groups ''' SELECT sum(step_1) AS step_1, sum(step_2) AS step_2, @@ -607,6 +428,7 @@ arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, groupArray(row_number) AS row_number, final_prop AS final_prop FROM @@ -615,16 +437,13 @@ countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, + groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, rowNumberInAllBlocks() AS row_number, breakdown AS final_prop FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -638,17 +457,28 @@ e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(and(equals(e.event, '$autocapture'), match(e.elements_chain, '(^|;)button(\\.|$|;|:)'), arrayExists(x -> ifNull(equals(x, 'Pay $10'), 0), e.elements_chain_texts)), 1, 0) AS step_1, - if(and(equals(e.event, '$autocapture'), match(e.elements_chain, '(^|;)a(\\.|$|;|:)'), equals(e.elements_chain_href, '/movie')), 1, 0) AS step_2 - FROM events_json AS e + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 + FROM events AS e LEFT OUTER JOIN - (SELECT tupleElement(argMax(tuple(person_distinct_id_overrides.person_id), person_distinct_id_overrides.version), 1) AS person_id, + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id FROM person_distinct_id_overrides WHERE equals(person_distinct_id_overrides.team_id, 99999) GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(tupleElement(argMax(tuple(person_distinct_id_overrides.is_deleted), person_distinct_id_overrides.version), 1), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$autocapture', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age + FROM person + WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 99999) + GROUP BY person.id + HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown @@ -674,27 +504,14 @@ use_hive_partitioning=0 ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_events_with_person_on_events_v2[new_events_schema] +# name: TestFOSSFunnelUDF.test_funnel_with_property_groups.1 ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - sum(step_3) AS step_3, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop + SELECT source.id, + source.id AS id, + 1 FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - breakdown AS final_prop + (SELECT aggregation_target AS actor_id, + actor_id AS id FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, [''] AS prop, @@ -712,9 +529,9 @@ e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(and(equals(e.event, '$autocapture'), match(e.elements_chain, '(^|;)button(\\.|$|;|:)'), arrayExists(x -> ifNull(equals(x, 'Pay $10'), 0), e.elements_chain_texts)), 1, 0) AS step_1, - if(and(equals(e.event, '$autocapture'), match(e.elements_chain, '(^|;)a(\\.|$|;|:)'), equals(e.elements_chain_href, '/movie')), 1, 0) AS step_2 - FROM events_json AS e + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 + FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id @@ -722,43 +539,189 @@ WHERE equals(person_distinct_id_overrides.team_id, 99999) GROUP BY person_distinct_id_overrides.distinct_id HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$autocapture', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age + FROM person + WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 99999) + GROUP BY person.id + HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC - LIMIT 100 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 + WHERE bitTest(steps_bitfield, 0) + ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_with_precalculated_cohort_step_filter +# name: TestFOSSFunnelUDF.test_funnel_with_property_groups.2 ''' - - SELECT count(DISTINCT person_id) - FROM cohortpeople + SELECT source.id, + source.id AS id, + 1 + FROM + (SELECT aggregation_target AS actor_id, + actor_id AS id + FROM + (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, + [''] AS prop, + arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.5 AS steps_bitfield, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age + FROM person + WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 99999) + GROUP BY person.id + HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE bitTest(steps_bitfield, 1) + ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestFOSSFunnelUDF.test_funnel_with_property_groups.3 + ''' + SELECT source.id, + source.id AS id, + 1 + FROM + (SELECT aggregation_target AS actor_id, + actor_id AS id + FROM + (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, + [''] AS prop, + arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.5 AS steps_bitfield, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, + if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 + FROM events AS e + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) + LEFT JOIN + (SELECT person.id AS id, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, + replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age + FROM person + WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), + (SELECT person.id AS id, max(person.version) AS version + FROM person + WHERE equals(person.team_id, 99999) + GROUP BY person.id + HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE bitTest(steps_bitfield, 2) + ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestFOSSFunnelUDF.test_funnel_with_static_cohort_step_filter + ''' + SELECT person_id + FROM person_static_cohort WHERE team_id = 99999 AND cohort_id = 99999 - AND version = 0 + AND person_id IN ['00000000-0000-4000-8000-000000000001'] + GROUP BY person_id ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_with_precalculated_cohort_step_filter.1 +# name: TestFOSSFunnelUDF.test_funnel_with_static_cohort_step_filter.1 ''' SELECT sum(step_1) AS step_1, sum(step_2) AS step_2, @@ -791,9 +754,9 @@ e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(and(equals(e.event, 'user signed up'), in(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), - (SELECT cohortpeople.person_id AS person_id - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 99999), equals(cohortpeople.cohort_id, 99999), equals(cohortpeople.version, 0))))), 1, 0) AS step_0, + (SELECT person_static_cohort.person_id AS person_id + FROM person_static_cohort + WHERE and(equals(person_static_cohort.team_id, 99999), equals(person_static_cohort.cohort_id, 99999))))), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 FROM events AS e LEFT OUTER JOIN @@ -827,7 +790,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_with_precalculated_cohort_step_filter[new_events_schema] +# name: TestFOSSFunnelUDF.test_timezones ''' SELECT sum(step_1) AS step_1, sum(step_2) AS step_2, @@ -854,17 +817,14 @@ af_tuple.5 AS steps_bitfield, aggregation_target AS aggregation_target FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + (SELECT toTimeZone(e.timestamp, 'US/Pacific') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, - if(and(equals(e.event, 'user signed up'), in(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), - (SELECT cohortpeople.person_id AS person_id - FROM cohortpeople - WHERE and(equals(cohortpeople.team_id, 99999), equals(cohortpeople.cohort_id, 99999), equals(cohortpeople.version, 0))))), 1, 0) AS step_0, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e + FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id @@ -872,7 +832,7 @@ WHERE equals(person_distinct_id_overrides.team_id, 99999) GROUP BY person_distinct_id_overrides.distinct_id HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'US/Pacific')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'US/Pacific'))), in(e.event, tuple('paid', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown @@ -896,1544 +856,26 @@ use_hive_partitioning=0 ''' # --- -# name: TestFOSSFunnelUDF.test_funnel_with_property_groups +# name: TestFunnelBreakdownUDF.test_funnel_breakdown_correct_breakdown_props_are_chosen ''' SELECT sum(step_1) AS step_1, sum(step_2) AS step_2, - sum(step_3) AS step_3, arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, groupArray(row_number) AS row_number, final_prop AS final_prop FROM (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, + groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, rowNumberInAllBlocks() AS row_number, - breakdown AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC - LIMIT 100 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFOSSFunnelUDF.test_funnel_with_property_groups.1 - ''' - SELECT source.id, - source.id AS id, - 1 - FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFOSSFunnelUDF.test_funnel_with_property_groups.2 - ''' - SELECT source.id, - source.id AS id, - 1 - FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 1) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFOSSFunnelUDF.test_funnel_with_property_groups.3 - ''' - SELECT source.id, - source.id AS id, - 1 - FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha.com'), 0)), 1, 0) AS step_1, - if(and(equals(e.event, '$pageview'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$current_url'), ''), 'null'), '^"|"$', ''), 'aloha2.com'), 0)), 1, 0) AS step_2 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 2) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFOSSFunnelUDF.test_funnel_with_property_groups[new_events_schema.1] - ''' - SELECT source.id, - source.id AS id, - 1 - FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(and(equals(e.event, '$pageview'), and(equals(e.properties.`$current_url`, 'aloha.com'), isNotNull(e.properties.`$current_url`))), 1, 0) AS step_1, - if(and(equals(e.event, '$pageview'), and(equals(e.properties.`$current_url`, 'aloha2.com'), isNotNull(e.properties.`$current_url`))), 1, 0) AS step_2 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFOSSFunnelUDF.test_funnel_with_property_groups[new_events_schema.2] - ''' - SELECT source.id, - source.id AS id, - 1 - FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(and(equals(e.event, '$pageview'), and(equals(e.properties.`$current_url`, 'aloha.com'), isNotNull(e.properties.`$current_url`))), 1, 0) AS step_1, - if(and(equals(e.event, '$pageview'), and(equals(e.properties.`$current_url`, 'aloha2.com'), isNotNull(e.properties.`$current_url`))), 1, 0) AS step_2 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 1) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFOSSFunnelUDF.test_funnel_with_property_groups[new_events_schema.3] - ''' - SELECT source.id, - source.id AS id, - 1 - FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(and(equals(e.event, '$pageview'), and(equals(e.properties.`$current_url`, 'aloha.com'), isNotNull(e.properties.`$current_url`))), 1, 0) AS step_1, - if(and(equals(e.event, '$pageview'), and(equals(e.properties.`$current_url`, 'aloha2.com'), isNotNull(e.properties.`$current_url`))), 1, 0) AS step_2 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 2) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFOSSFunnelUDF.test_funnel_with_property_groups[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - sum(step_3) AS step_3, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - breakdown AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(and(equals(e.event, '$pageview'), and(equals(e.properties.`$current_url`, 'aloha.com'), isNotNull(e.properties.`$current_url`))), 1, 0) AS step_1, - if(and(equals(e.event, '$pageview'), and(equals(e.properties.`$current_url`, 'aloha2.com'), isNotNull(e.properties.`$current_url`))), 1, 0) AS step_2 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'email'), ''), 'null'), '^"|"$', '') AS properties___email, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, 'age'), ''), 'null'), '^"|"$', '') AS properties___age - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('$pageview', 'user signed up')), or(and(ifNull(ilike(toString(e__person.properties___email), '%.com%'), 0), ifNull(equals(e__person.properties___age, '20'), 0)), or(ifNull(ilike(toString(e__person.properties___email), '%.org%'), 0), ifNull(equals(e__person.properties___age, '28'), 0)))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC - LIMIT 100 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFOSSFunnelUDF.test_funnel_with_static_cohort_step_filter - ''' - SELECT person_id - FROM person_static_cohort - WHERE team_id = 99999 - AND cohort_id = 99999 - AND person_id IN ['00000000-0000-4000-8000-000000000001'] - GROUP BY person_id - ''' -# --- -# name: TestFOSSFunnelUDF.test_funnel_with_static_cohort_step_filter.1 - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - breakdown AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(and(equals(e.event, 'user signed up'), in(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), - (SELECT person_static_cohort.person_id AS person_id - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), equals(person_static_cohort.cohort_id, 99999))))), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 100 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFOSSFunnelUDF.test_funnel_with_static_cohort_step_filter[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - breakdown AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(and(equals(e.event, 'user signed up'), in(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), - (SELECT person_static_cohort.person_id AS person_id - FROM person_static_cohort - WHERE and(equals(person_static_cohort.team_id, 99999), equals(person_static_cohort.cohort_id, 99999))))), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 100 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFOSSFunnelUDF.test_timezones - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - breakdown AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'US/Pacific') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'US/Pacific')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'US/Pacific'))), in(e.event, tuple('paid', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 100 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFOSSFunnelUDF.test_timezones[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - breakdown AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'US/Pacific') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'US/Pacific')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'US/Pacific'))), in(e.event, tuple('paid', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 100 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFunnelBreakdownUDF.test_funnel_breakdown_correct_breakdown_props_are_chosen - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 25), breakdown, ['Other']) AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - argMinIf(prop_basic, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop_basic))) AS prop, - arrayJoin(aggregate_funnel_array(2, 1209600, 'first_touch', 'ordered', [if(empty(prop), [''], prop)], [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(and(equals(e.event, 'buy'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', ''), 'xyz'), 0)), 1, 0) AS step_1, - [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, - prop_basic AS prop - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 26 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFunnelBreakdownUDF.test_funnel_breakdown_correct_breakdown_props_are_chosen[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 25), breakdown, ['Other']) AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - argMinIf(prop_basic, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop_basic))) AS prop, - arrayJoin(aggregate_funnel_array(2, 1209600, 'first_touch', 'ordered', [if(empty(prop), [''], prop)], [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(and(equals(e.event, 'buy'), ifNull(equals(if(notEquals(toJSONString(e.properties.^`$version`), '{}'), toJSONString(e.properties.^`$version`), if(isNull(e.properties.`$version`), NULL, if(startsWith(dynamicType(e.properties.`$version`), 'DateTime'), replaceOne(toString(e.properties.`$version`), ' ', 'T'), if(or(startsWith(dynamicType(e.properties.`$version`), 'Array'), startsWith(dynamicType(e.properties.`$version`), 'Map'), startsWith(dynamicType(e.properties.`$version`), 'Tuple')), toJSONString(e.properties.`$version`), toString(e.properties.`$version`))))), 'xyz'), 0)), 1, 0) AS step_1, - [ifNull(toString(e.properties.`$browser`), '')] AS prop_basic, - prop_basic AS prop - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 26 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFunnelBreakdownUDF.test_funnel_breakdown_correct_breakdown_props_are_chosen_for_step - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 25), breakdown, ['Other']) AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - groupUniqArrayIf(arrayMap(x -> ifNull(x, ''), prop), notEmpty(prop)) AS prop, - arrayJoin(aggregate_funnel_array(2, 1209600, 'step_1', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(and(equals(e.event, 'buy'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', ''), 'xyz'), 0)), 1, 0) AS step_1, - [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, - if(equals(step_1, 1), prop_basic, []) AS prop - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 26 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFunnelBreakdownUDF.test_funnel_breakdown_correct_breakdown_props_are_chosen_for_step[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 25), breakdown, ['Other']) AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - groupUniqArrayIf(arrayMap(x -> ifNull(x, ''), prop), notEmpty(prop)) AS prop, - arrayJoin(aggregate_funnel_array(2, 1209600, 'step_1', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(and(equals(e.event, 'buy'), ifNull(equals(if(notEquals(toJSONString(e.properties.^`$version`), '{}'), toJSONString(e.properties.^`$version`), if(isNull(e.properties.`$version`), NULL, if(startsWith(dynamicType(e.properties.`$version`), 'DateTime'), replaceOne(toString(e.properties.`$version`), ' ', 'T'), if(or(startsWith(dynamicType(e.properties.`$version`), 'Array'), startsWith(dynamicType(e.properties.`$version`), 'Map'), startsWith(dynamicType(e.properties.`$version`), 'Tuple')), toJSONString(e.properties.`$version`), toString(e.properties.`$version`))))), 'xyz'), 0)), 1, 0) AS step_1, - [ifNull(toString(e.properties.`$browser`), '')] AS prop_basic, - if(equals(step_1, 1), prop_basic, []) AS prop - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 26 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFunnelBreakdownUDF.test_funnel_step_multiple_breakdown_snapshot - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 25), breakdown, ['Other']) AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - argMinIf(prop_basic, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop_basic))) AS prop, - arrayJoin(aggregate_funnel_array(2, 1209600, 'first_touch', 'ordered', [if(empty(prop), ['', ''], prop)], [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(equals(e.event, 'buy'), 1, 0) AS step_1, - [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), ''), ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, - prop_basic AS prop - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 26 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFunnelBreakdownUDF.test_funnel_step_multiple_breakdown_snapshot[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 25), breakdown, ['Other']) AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - argMinIf(prop_basic, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop_basic))) AS prop, - arrayJoin(aggregate_funnel_array(2, 1209600, 'first_touch', 'ordered', [if(empty(prop), ['', ''], prop)], [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(equals(e.event, 'buy'), 1, 0) AS step_1, - [ifNull(toString(e.properties.`$browser`), ''), ifNull(toString(if(notEquals(toJSONString(e.properties.^`$version`), '{}'), toJSONString(e.properties.^`$version`), if(isNull(e.properties.`$version`), NULL, if(startsWith(dynamicType(e.properties.`$version`), 'DateTime'), replaceOne(toString(e.properties.`$version`), ' ', 'T'), if(or(startsWith(dynamicType(e.properties.`$version`), 'Array'), startsWith(dynamicType(e.properties.`$version`), 'Map'), startsWith(dynamicType(e.properties.`$version`), 'Tuple')), toJSONString(e.properties.`$version`), toString(e.properties.`$version`)))))), '')] AS prop_basic, - prop_basic AS prop - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 26 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_aggregate_by_groups_breakdown_group_person_on_events - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - sum(step_3) AS step_3, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 25), breakdown, 'Other') AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, ifNull(prop_basic, ''), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [argMinIf(prop_basic, timestamp, isNotNull(prop_basic))] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(equals(e.event, 'play movie'), 1, 0) AS step_1, - if(equals(e.event, 'buy'), 1, 0) AS step_2, - ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, - prop_basic AS prop - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC - LIMIT 26 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_aggregate_by_groups_breakdown_group_person_on_events[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - sum(step_3) AS step_3, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 25), breakdown, 'Other') AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, ifNull(prop_basic, ''), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [argMinIf(prop_basic, timestamp, isNotNull(prop_basic))] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(equals(e.event, 'play movie'), 1, 0) AS step_1, - if(equals(e.event, 'buy'), 1, 0) AS step_2, - ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, - prop_basic AS prop - FROM events_json AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC - LIMIT 26 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_aggregate_by_groups_breakdown_group_person_on_events_poe_v2 - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - sum(step_3) AS step_3, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, - max(total_median_conversion_time) AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(total_conversion_times) OVER ()))])[1] AS total_median_conversion_time, - rowNumberInAllBlocks() AS row_number, - if(ifNull(less(row_number, 25), 0), breakdown, 'Other') AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, ifNull(prop_basic, ''), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [argMinIf(prop_basic, timestamp, isNotNull(prop_basic))] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(equals(e.event, 'play movie'), 1, 0) AS step_1, - if(equals(e.event, 'buy'), 1, 0) AS step_2, - ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, - prop_basic AS prop - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0), ifNull(equals(step_2, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC - LIMIT 26 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - enable_analyzer=1, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - sum(step_3) AS step_3, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 25), breakdown, 'Other') AS final_prop + if(less(row_number, 25), breakdown, ['Other']) AS final_prop FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, ifNull(prop_basic, ''), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [argMinIf(prop_basic, timestamp, isNotNull(prop_basic))] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, + argMinIf(prop_basic, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop_basic))) AS prop, + arrayJoin(aggregate_funnel_array(2, 1209600, 'first_touch', 'ordered', [if(empty(prop), [''], prop)], [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -2447,9 +889,8 @@ e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(equals(e.event, 'play movie'), 1, 0) AS step_1, - if(equals(e.event, 'buy'), 1, 0) AS step_2, - ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, + if(and(equals(e.event, 'buy'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', ''), 'xyz'), 0)), 1, 0) AS step_1, + [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, prop_basic AS prop FROM events AS e LEFT OUTER JOIN @@ -2459,24 +900,14 @@ WHERE equals(person_distinct_id_overrides.team_id, 99999) GROUP BY person_distinct_id_overrides.distinct_id HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) GROUP BY breakdown - ORDER BY step_3 DESC, - step_2 DESC, + ORDER BY step_2 DESC, step_1 DESC) GROUP BY final_prop - ORDER BY step_3 DESC, - step_2 DESC, + ORDER BY step_2 DESC, step_1 DESC LIMIT 26 SETTINGS join_algorithm='auto', readonly=2, @@ -2493,88 +924,26 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group.1 - ''' - SELECT source.id, - source.id AS id - FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, ifNull(prop_basic, ''), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [argMinIf(prop_basic, timestamp, isNotNull(prop_basic))] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(equals(e.event, 'play movie'), 1, 0) AS step_1, - if(equals(e.event, 'buy'), 1, 0) AS step_2, - ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, - prop_basic AS prop - FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE and(bitTest(steps_bitfield, 0), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('finance')))) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group.2 +# name: TestFunnelBreakdownUDF.test_funnel_breakdown_correct_breakdown_props_are_chosen_for_step ''' - SELECT source.id, - source.id AS id + SELECT sum(step_1) AS step_1, + sum(step_2) AS step_2, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, + groupArray(row_number) AS row_number, + final_prop AS final_prop FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id + (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, + countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, + groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, + groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, + rowNumberInAllBlocks() AS row_number, + if(less(row_number, 25), breakdown, ['Other']) AS final_prop FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, ifNull(prop_basic, ''), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [argMinIf(prop_basic, timestamp, isNotNull(prop_basic))] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, + groupUniqArrayIf(arrayMap(x -> ifNull(x, ''), prop), notEmpty(prop)) AS prop, + arrayJoin(aggregate_funnel_array(2, 1209600, 'step_1', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -2588,10 +957,9 @@ e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(equals(e.event, 'play movie'), 1, 0) AS step_1, - if(equals(e.event, 'buy'), 1, 0) AS step_2, - ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, - prop_basic AS prop + if(and(equals(e.event, 'buy'), ifNull(equals(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', ''), 'xyz'), 0)), 1, 0) AS step_1, + [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, + if(equals(step_1, 1), prop_basic, []) AS prop FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, @@ -2600,29 +968,22 @@ WHERE equals(person_distinct_id_overrides.team_id, 99999) GROUP BY person_distinct_id_overrides.distinct_id HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE and(bitTest(steps_bitfield, 1), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('finance')))) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', + GROUP BY breakdown + ORDER BY step_2 DESC, + step_1 DESC) + GROUP BY final_prop + ORDER BY step_2 DESC, + step_1 DESC + LIMIT 26 SETTINGS join_algorithm='auto', readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, + max_bytes_before_external_group_by=23622320128, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, @@ -2631,19 +992,26 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group.3 +# name: TestFunnelBreakdownUDF.test_funnel_step_multiple_breakdown_snapshot ''' - SELECT source.id, - source.id AS id + SELECT sum(step_1) AS step_1, + sum(step_2) AS step_2, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, + groupArray(row_number) AS row_number, + final_prop AS final_prop FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id + (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, + countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, + groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, + groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, + rowNumberInAllBlocks() AS row_number, + if(less(row_number, 25), breakdown, ['Other']) AS final_prop FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, ifNull(prop_basic, ''), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, - [argMinIf(prop_basic, timestamp, isNotNull(prop_basic))] AS prop, - arrayJoin(aggregate_funnel(3, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, + argMinIf(prop_basic, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop_basic))) AS prop, + arrayJoin(aggregate_funnel_array(2, 1209600, 'first_touch', 'ordered', [if(empty(prop), ['', ''], prop)], [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -2657,9 +1025,8 @@ e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'sign up'), 1, 0) AS step_0, - if(equals(e.event, 'play movie'), 1, 0) AS step_1, - if(equals(e.event, 'buy'), 1, 0) AS step_2, - ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, + if(equals(e.event, 'buy'), 1, 0) AS step_1, + [ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$browser'), ''), 'null'), '^"|"$', '')), ''), ifNull(toString(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(e.properties, '$version'), ''), 'null'), '^"|"$', '')), '')] AS prop_basic, prop_basic AS prop FROM events AS e LEFT OUTER JOIN @@ -2669,29 +1036,22 @@ WHERE equals(person_distinct_id_overrides.team_id, 99999) GROUP BY person_distinct_id_overrides.distinct_id HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE and(bitTest(steps_bitfield, 0), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('technology')))) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', + GROUP BY breakdown + ORDER BY step_2 DESC, + step_1 DESC) + GROUP BY final_prop + ORDER BY step_2 DESC, + step_1 DESC + LIMIT 26 SETTINGS join_algorithm='auto', readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, + max_bytes_before_external_group_by=23622320128, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, @@ -2700,13 +1060,27 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group.4 +# name: TestFunnelGroupBreakdownUDF.test_funnel_aggregate_by_groups_breakdown_group_person_on_events ''' - SELECT source.id, - source.id AS id + SELECT sum(step_1) AS step_1, + sum(step_2) AS step_2, + sum(step_3) AS step_3, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, + groupArray(row_number) AS row_number, + final_prop AS final_prop FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id + (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, + countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, + countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, + groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, + groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, + groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, + rowNumberInAllBlocks() AS row_number, + if(less(row_number, 25), breakdown, 'Other') AS final_prop FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, ifNull(prop_basic, ''), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, [argMinIf(prop_basic, timestamp, isNotNull(prop_basic))] AS prop, @@ -2721,7 +1095,7 @@ aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, @@ -2731,13 +1105,6 @@ ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, prop_basic AS prop FROM events AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) LEFT JOIN (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, groups.group_type_index AS index, @@ -2746,21 +1113,24 @@ WHERE and(equals(groups.team_id, 99999), equals(index, 0)) GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-08 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE and(bitTest(steps_bitfield, 1), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('technology')))) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', + GROUP BY breakdown + ORDER BY step_3 DESC, + step_2 DESC, + step_1 DESC) + GROUP BY final_prop + ORDER BY step_3 DESC, + step_2 DESC, + step_1 DESC + LIMIT 26 SETTINGS join_algorithm='auto', readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, + max_bytes_before_external_group_by=23622320128, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, @@ -2769,13 +1139,27 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group[new_events_schema.1] +# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group ''' - SELECT source.id, - source.id AS id + SELECT sum(step_1) AS step_1, + sum(step_2) AS step_2, + sum(step_3) AS step_3, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, + arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, + groupArray(row_number) AS row_number, + final_prop AS final_prop FROM - (SELECT aggregation_target AS actor_id, - actor_id AS id + (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, + countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, + countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, + groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, + groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, + groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, + rowNumberInAllBlocks() AS row_number, + if(less(row_number, 25), breakdown, 'Other') AS final_prop FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, ifNull(prop_basic, ''), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, [argMinIf(prop_basic, timestamp, isNotNull(prop_basic))] AS prop, @@ -2799,7 +1183,7 @@ if(equals(e.event, 'buy'), 1, 0) AS step_2, ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, prop_basic AS prop - FROM events_json AS e + FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id @@ -2818,18 +1202,21 @@ WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE and(bitTest(steps_bitfield, 0), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('finance')))) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', + GROUP BY breakdown + ORDER BY step_3 DESC, + step_2 DESC, + step_1 DESC) + GROUP BY final_prop + ORDER BY step_3 DESC, + step_2 DESC, + step_1 DESC + LIMIT 26 SETTINGS join_algorithm='auto', readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, + max_bytes_before_external_group_by=23622320128, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, @@ -2838,7 +1225,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group[new_events_schema.2] +# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group.1 ''' SELECT source.id, source.id AS id @@ -2868,7 +1255,7 @@ if(equals(e.event, 'buy'), 1, 0) AS step_2, ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, prop_basic AS prop - FROM events_json AS e + FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id @@ -2887,7 +1274,7 @@ WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE and(bitTest(steps_bitfield, 1), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('finance')))) + WHERE and(bitTest(steps_bitfield, 0), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('finance')))) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source ORDER BY source.id ASC LIMIT 101 @@ -2907,7 +1294,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group[new_events_schema.3] +# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group.2 ''' SELECT source.id, source.id AS id @@ -2937,7 +1324,7 @@ if(equals(e.event, 'buy'), 1, 0) AS step_2, ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, prop_basic AS prop - FROM events_json AS e + FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id @@ -2956,7 +1343,7 @@ WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE and(bitTest(steps_bitfield, 0), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('technology')))) + WHERE and(bitTest(steps_bitfield, 1), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('finance')))) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source ORDER BY source.id ASC LIMIT 101 @@ -2976,7 +1363,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group[new_events_schema.4] +# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group.3 ''' SELECT source.id, source.id AS id @@ -3006,7 +1393,7 @@ if(equals(e.event, 'buy'), 1, 0) AS step_2, ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, prop_basic AS prop - FROM events_json AS e + FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id @@ -3025,7 +1412,7 @@ WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE and(bitTest(steps_bitfield, 1), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('technology')))) + WHERE and(bitTest(steps_bitfield, 0), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('technology')))) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source ORDER BY source.id ASC LIMIT 101 @@ -3045,27 +1432,13 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group[new_events_schema] +# name: TestFunnelGroupBreakdownUDF.test_funnel_breakdown_group.4 ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - sum(step_3) AS step_3, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_2_conversion_times)])[1] AS step_2_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_2_conversion_times)])[1] AS step_2_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop + SELECT source.id, + source.id AS id FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 4), 0), 1)) AS step_3, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(timings[2], ifNull(greater(timings[2], 0), 0)) AS step_2_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 2), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 25), breakdown, 'Other') AS final_prop + (SELECT aggregation_target AS actor_id, + actor_id AS id FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, ifNull(prop_basic, ''), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1), multiply(3, step_2)])))) AS events_array, [argMinIf(prop_basic, timestamp, isNotNull(prop_basic))] AS prop, @@ -3089,7 +1462,7 @@ if(equals(e.event, 'buy'), 1, 0) AS step_2, ifNull(toString(e__group_0.properties___industry), '') AS prop_basic, prop_basic AS prop - FROM events_json AS e + FROM events AS e LEFT OUTER JOIN (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id @@ -3108,21 +1481,18 @@ WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('buy', 'play movie', 'sign up'))), or(equals(step_0, 1), equals(step_1, 1), equals(step_2, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_3 DESC, - step_2 DESC, - step_1 DESC - LIMIT 26 SETTINGS join_algorithm='auto', + WHERE and(bitTest(steps_bitfield, 1), equals(arrayFlatten(array(breakdown)), arrayFlatten(array('technology')))) + ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto') AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', readonly=2, max_execution_time=60, allow_experimental_object_type=1, max_ast_elements=4000000, max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, + max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_breakdowns_by_current_url.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_breakdowns_by_current_url.ambr index f6d01aaff05d..412efde766c0 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_breakdowns_by_current_url.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_breakdowns_by_current_url.ambr @@ -67,74 +67,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelBreakdownsByCurrentURL.test_breakdown_by_current_url[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 100), breakdown, ['Other']) AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - argMinIf(prop_basic, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop_basic))) AS prop, - arrayJoin(aggregate_funnel_array(2, 1209600, 'first_touch', 'ordered', [if(empty(prop), [''], prop)], [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'watched movie'), 1, 0) AS step_0, - if(equals(e.event, 'terminate funnel'), 1, 0) AS step_1, - [if(empty(replaceRegexpOne(ifNull(toString(e.properties.`$current_url`), ''), '[\\/?#]*$', '')), '/', replaceRegexpOne(ifNull(toString(e.properties.`$current_url`), ''), '[\\/?#]*$', ''))] AS prop_basic, - prop_basic AS prop - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('terminate funnel', 'watched movie'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 101 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestFunnelBreakdownsByCurrentURL.test_breakdown_by_pathname ''' SELECT sum(step_1) AS step_1, @@ -203,74 +135,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelBreakdownsByCurrentURL.test_breakdown_by_pathname[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 100), breakdown, ['Other']) AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - argMinIf(prop_basic, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop_basic))) AS prop, - arrayJoin(aggregate_funnel_array(2, 1209600, 'first_touch', 'ordered', [if(empty(prop), [''], prop)], [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'watched movie'), 1, 0) AS step_0, - if(equals(e.event, 'terminate funnel'), 1, 0) AS step_1, - [if(empty(replaceRegexpOne(ifNull(toString(e.properties.`$pathname`), ''), '[\\/?#]*$', '')), '/', replaceRegexpOne(ifNull(toString(e.properties.`$pathname`), ''), '[\\/?#]*$', ''))] AS prop_basic, - prop_basic AS prop - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('terminate funnel', 'watched movie'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 101 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestFunnelBreakdownsByCurrentURL.test_breakdown_by_pathname_with_path_cleaning ''' SELECT sum(step_1) AS step_1, @@ -339,74 +203,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelBreakdownsByCurrentURL.test_breakdown_by_pathname_with_path_cleaning[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 100), breakdown, ['Other']) AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - argMinIf(prop_basic, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop_basic))) AS prop, - arrayJoin(aggregate_funnel_array(2, 1209600, 'first_touch', 'ordered', [if(empty(prop), [''], prop)], [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'watched movie'), 1, 0) AS step_0, - if(equals(e.event, 'terminate funnel'), 1, 0) AS step_1, - [if(empty(replaceRegexpOne(replaceRegexpAll(ifNull(toString(e.properties.`$pathname`), ''), '/home.*', '/cleaned-home'), '[\\/?#]*$', '')), '/', replaceRegexpOne(replaceRegexpAll(ifNull(toString(e.properties.`$pathname`), ''), '/home.*', '/cleaned-home'), '[\\/?#]*$', ''))] AS prop_basic, - prop_basic AS prop - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('terminate funnel', 'watched movie'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 101 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestFunnelBreakdownsByCurrentURL.test_breakdown_by_pathname_with_path_cleaning_without_normalization ''' SELECT sum(step_1) AS step_1, @@ -475,71 +271,3 @@ use_hive_partitioning=0 ''' # --- -# name: TestFunnelBreakdownsByCurrentURL.test_breakdown_by_pathname_with_path_cleaning_without_normalization[new_events_schema] - ''' - SELECT sum(step_1) AS step_1, - sum(step_2) AS step_2, - arrayMap(x -> if(isNaN(x), NULL, x), [avgArrayOrNull(step_1_conversion_times)])[1] AS step_1_average_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [medianArrayOrNull(step_1_conversion_times)])[1] AS step_1_median_conversion_time, - arrayMap(x -> if(isNaN(x), NULL, x), [arrayReduce('median', arrayFlatten(groupArray(arrayFlatten(groupArray(total_conversion_times))) OVER ()))])[1] AS total_median_conversion_time, - groupArray(row_number) AS row_number, - final_prop AS final_prop - FROM - (SELECT countIf(ifNull(notEquals(bitAnd(steps_bitfield, 1), 0), 1)) AS step_1, - countIf(ifNull(notEquals(bitAnd(steps_bitfield, 2), 0), 1)) AS step_2, - groupArrayIf(timings[1], ifNull(greater(timings[1], 0), 0)) AS step_1_conversion_times, - groupArrayIf(arraySum(timings), ifNull(greaterOrEquals(step_reached, 1), 0)) AS total_conversion_times, - rowNumberInAllBlocks() AS row_number, - if(less(row_number, 100), breakdown, ['Other']) AS final_prop - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, arrayMap(x -> ifNull(x, ''), prop_basic), arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - argMinIf(prop_basic, timestamp, notEmpty(arrayFilter(x -> notEmpty(x), prop_basic))) AS prop, - arrayJoin(aggregate_funnel_array(2, 1209600, 'first_touch', 'ordered', [if(empty(prop), [''], prop)], [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'watched movie'), 1, 0) AS step_0, - if(equals(e.event, 'terminate funnel'), 1, 0) AS step_1, - [replaceRegexpAll(ifNull(toString(e.properties.`$pathname`), ''), '/home.*', '/cleaned-home')] AS prop_basic, - prop_basic AS prop - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('terminate funnel', 'watched movie'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - GROUP BY breakdown - ORDER BY step_2 DESC, - step_1 DESC) - GROUP BY final_prop - ORDER BY step_2 DESC, - step_1 DESC - LIMIT 101 SETTINGS join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- diff --git a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr index 47699c176c78..3d209c6f99b0 100644 --- a/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr +++ b/posthog/hogql_queries/insights/funnels/test/__snapshots__/test_funnel_correlation.ambr @@ -61,68 +61,6 @@ use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_action_events_are_excluded_from_correlations[new_events_schema] - ''' - WITH funnel_actors AS MATERIALIZED - (SELECT aggregation_target AS actor_id, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, af_tuple.5 AS steps_bitfield, aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(and(equals(e.event, 'user signed up'), ifNull(equals(if(notEquals(toJSONString(e.properties.^key), '{}'), toJSONString(e.properties.^key), if(isNull(e.properties.key), NULL, if(startsWith(dynamicType(e.properties.key), 'DateTime'), replaceOne(toString(e.properties.key), ' ', 'T'), if(or(startsWith(dynamicType(e.properties.key), 'Array'), startsWith(dynamicType(e.properties.key), 'Map'), startsWith(dynamicType(e.properties.key), 'Tuple')), toJSONString(e.properties.key), toString(e.properties.key))))), 'val'), 0)), 1, 0) AS step_0, if(and(equals(e.event, 'paid'), ifNull(equals(if(notEquals(toJSONString(e.properties.^key), '{}'), toJSONString(e.properties.^key), if(isNull(e.properties.key), NULL, if(startsWith(dynamicType(e.properties.key), 'DateTime'), replaceOne(toString(e.properties.key), ' ', 'T'), if(or(startsWith(dynamicType(e.properties.key), 'Array'), startsWith(dynamicType(e.properties.key), 'Map'), startsWith(dynamicType(e.properties.key), 'Tuple')), toJSONString(e.properties.key), toString(e.properties.key))))), 'val'), 0)), 1, 0) AS step_1 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names - SELECT event.event AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM events_json AS event - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS event__override ON equals(event.distinct_id, event__override.distinct_id) - JOIN funnel_actors ON equals(if(not(empty(event__override.distinct_id)), event__override.person_id, event.person_id), funnel_actors.actor_id) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), notIn(event.event, [])) - GROUP BY name - LIMIT 100 - UNION ALL - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties ''' WITH funnel_actors AS @@ -638,11 +576,160 @@ use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties[new_events_schema.1] +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_event_properties_and_groups ''' - SELECT source.id, - source.id AS id, - source.matching_events AS matching_events + WITH funnel_actors AS + (SELECT aggregation_target AS actor_id, + (matched_events_array[1][1]).1 AS timestamp, + nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, + (matched_events_array[1][1]).1 AS first_timestamp, + steps AS steps + FROM + (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, + [''] AS prop, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + af_tuple.1 AS step_reached, + plus(af_tuple.1, 1) AS steps, + af_tuple.2 AS breakdown, + af_tuple.3 AS timings, + af_tuple.4 AS matched_event_uuids_array_array, + groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, + mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, + arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, + af_tuple.5 AS steps_bitfield, + aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, + e.`$group_1` AS aggregation_target, + e.uuid AS uuid, + e.`$session_id` AS `$session_id`, + e.`$window_id` AS `$window_id`, + if(equals(e.event, 'user signed up'), 1, 0) AS step_0, + if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE bitTest(steps_bitfield, 0) + ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), + filtered_events AS + (SELECT events.event AS event, + events.properties AS properties, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + events.`$group_1` AS `$group_1`, + accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$exception_issue_id'), ''), 'null'), '^"|"$', ''), 'UUID') AS event_issue_id, + events__fingerprint_issue_state.issue_id AS issue_id, + events__fingerprint_issue_state.issue_id AS issue_id_v2, + events__fingerprint_issue_state.issue_name AS issue_name, + events__fingerprint_issue_state.issue_description AS issue_description, + events__fingerprint_issue_state.issue_status AS issue_status, + events__fingerprint_issue_state.assigned_user_id AS issue_assigned_user_id, + events__fingerprint_issue_state.assigned_role_id AS issue_assigned_role_id, + events__fingerprint_issue_state.first_seen AS issue_first_seen + FROM events + LEFT OUTER JOIN + (SELECT cityHash64(error_tracking_fingerprint_issue_state.fingerprint) AS fp_hash, + toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_id), error_tracking_fingerprint_issue_state.version), 1)) AS issue_id, + toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_name), error_tracking_fingerprint_issue_state.version), 1)) AS issue_name, + toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_description), error_tracking_fingerprint_issue_state.version), 1)) AS issue_description, + toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_status), error_tracking_fingerprint_issue_state.version), 1)) AS issue_status, + toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.assigned_user_id), error_tracking_fingerprint_issue_state.version), 1)) AS assigned_user_id, + toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.assigned_role_id), error_tracking_fingerprint_issue_state.version), 1)) AS assigned_role_id, + toNullable(tupleElement(argMax(tuple(toTimeZone(error_tracking_fingerprint_issue_state.first_seen, 'UTC')), error_tracking_fingerprint_issue_state.version), 1)) AS first_seen + FROM error_tracking_fingerprint_issue_state + WHERE equals(error_tracking_fingerprint_issue_state.team_id, 99999) + GROUP BY fp_hash + HAVING equals(argMax(error_tracking_fingerprint_issue_state.is_deleted, error_tracking_fingerprint_issue_state.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__fingerprint_issue_state ON equals(cityHash64(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$exception_fingerprint'), ''), 'null'), '^"|"$', '')), events__fingerprint_issue_state.fp_hash) + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), equals(events.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(events.timestamp, 'UTC'), 'UTC'), assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC'))), less(toDateTime(toTimeZone(events.timestamp, 'UTC'), 'UTC'), assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC'))), notIn(events.event, tuple('paid', 'user signed up')), in(events.event, tuple('positively_related', 'negatively_related')))), + 2 AS target_step + SELECT if(ifNull(equals((prop).1, 'Total_Values_In_Query'), 0), 'Total_Values_In_Query', concat(ifNull(toString((prop).1), ''), '::', ifNull(toString((prop).2), ''), '::', ifNull(toString((prop).3), ''))) AS name, + countDistinctIf(actor_id, ifNull(equals(steps, target_step), isNull(steps) + and isNull(target_step))) AS success_count, + countDistinctIf(actor_id, ifNull(notEquals(steps, target_step), isNotNull(steps) + or isNotNull(target_step))) AS failure_count + FROM + (SELECT funnel_actors.actor_id AS actor_id, + funnel_actors.steps AS steps, + arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '', '')], if(empty(event_table.event), [], arrayMap(prop -> tuple(event_table.event, prop.1, prop.2), JSONExtractKeysAndValues(event_table.properties, 'String'))))) AS prop + FROM filtered_events AS event_table + RIGHT JOIN funnel_actors ON and(equals(funnel_actors.actor_id, event_table.`$group_1`), greater(toTimeZone(toDateTime(toTimeZone(event_table.timestamp, 'UTC'), 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toTimeZone(toDateTime(toTimeZone(event_table.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')))))) + GROUP BY name, + prop + HAVING or(equals(name, 'Total_Values_In_Query'), greater(plus(success_count, failure_count), 2)) + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups + ''' + WITH funnel_actors AS MATERIALIZED + (SELECT aggregation_target AS actor_id, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps + FROM + (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, af_tuple.5 AS steps_bitfield, aggregation_target AS aggregation_target + FROM + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) + GROUP BY aggregation_target + HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) + WHERE bitTest(steps_bitfield, 0) + ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), + assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, + assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, + 2 AS target_step, + ['paid', 'user signed up'] AS funnel_step_names + SELECT event.event AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) + and isNull(target_step))) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) + or isNotNull(target_step))) AS failure_count + FROM events AS event + JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), notIn(event.event, [])) + GROUP BY name + LIMIT 100 + UNION ALL + SELECT 'Total_Values_In_Query' AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) + and isNull(target_step))) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) + or isNotNull(target_step))) AS failure_count + FROM funnel_actors + LIMIT 100 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.1 + ''' + SELECT source.actor_id AS actor_id FROM (WITH funnel_actors AS (SELECT aggregation_target AS actor_id, @@ -667,66 +754,51 @@ aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('today', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(equals(step_0, 1), equals(step_1, 1)))) + FROM events AS e + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS id - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, + assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, + assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, + 2 AS target_step, + ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events, + actor_id AS key + FROM events AS event + JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'positively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) + and isNull(target_step))) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, optimize_rewrite_aggregate_function_with_if=0, optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties[new_events_schema.2] +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.2 ''' - SELECT source.id, - source.id AS id, - source.matching_events AS matching_events + SELECT source.actor_id AS actor_id FROM (WITH funnel_actors AS (SELECT aggregation_target AS actor_id, @@ -751,44 +823,31 @@ aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('today', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(equals(step_0, 1), equals(step_1, 1)))) + FROM events AS e + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS id - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.id ASC + assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, + assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, + 2 AS target_step, + ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events, + actor_id AS key + FROM events AS event + JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'positively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) + or isNotNull(target_step))) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + ORDER BY source.actor_id ASC LIMIT 101 OFFSET 0 SETTINGS optimize_aggregation_in_order=1, join_algorithm='auto', @@ -806,11 +865,9 @@ use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties[new_events_schema.3] +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.3 ''' - SELECT source.id, - source.id AS id, - source.matching_events AS matching_events + SELECT source.actor_id AS actor_id FROM (WITH funnel_actors AS (SELECT aggregation_target AS actor_id, @@ -835,44 +892,31 @@ aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('today', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(equals(step_0, 1), equals(step_1, 1)))) + FROM events AS e + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS id - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.id ASC + assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, + assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, + 2 AS target_step, + ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events, + actor_id AS key + FROM events AS event + JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) + and isNull(target_step))) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + ORDER BY source.actor_id ASC LIMIT 101 OFFSET 0 SETTINGS optimize_aggregation_in_order=1, join_algorithm='auto', @@ -890,11 +934,9 @@ use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties[new_events_schema.4] +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.4 ''' - SELECT source.id, - source.id AS id, - source.matching_events AS matching_events + SELECT source.actor_id AS actor_id FROM (WITH funnel_actors AS (SELECT aggregation_target AS actor_id, @@ -919,44 +961,31 @@ aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, '$browser'), ''), 'null'), '^"|"$', '') AS `properties___$browser` - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('today', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(equals(step_0, 1), equals(step_1, 1)))) + FROM events AS e + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS id - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.id ASC + assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, + assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, + 2 AS target_step, + ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events, + actor_id AS key + FROM events AS event + JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) + or isNotNull(target_step))) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + ORDER BY source.actor_id ASC LIMIT 101 OFFSET 0 SETTINGS optimize_aggregation_in_order=1, join_algorithm='auto', @@ -974,213 +1003,40 @@ use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties[new_events_schema] +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.5 ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps + WITH funnel_actors AS MATERIALIZED + (SELECT aggregation_target AS actor_id, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target + (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, af_tuple.5 AS steps_bitfield, aggregation_target AS aggregation_target FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('today', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(equals(step_0, 1), equals(step_1, 1)))) + (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 + FROM events AS e + LEFT JOIN + (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, groups.group_type_index AS index, groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 99999), equals(index, 0)) + GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], arrayZip(['$browser'], [JSONExtractString(persons.person_props, '$browser')]))) AS prop - FROM funnel_actors - JOIN - (SELECT persons.id AS id, - persons.properties AS person_props - FROM - (SELECT person.id AS id, - person.properties AS properties - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))))))) SETTINGS optimize_aggregation_in_order=1) AS persons) AS persons ON equals(persons.id, funnel_actors.actor_id)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties_materialized - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT OUTER JOIN - (SELECT tupleElement(argMax(tuple(person_distinct_id_overrides.person_id), person_distinct_id_overrides.version), 1) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(tupleElement(argMax(tuple(person_distinct_id_overrides.is_deleted), person_distinct_id_overrides.version), 1), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('today', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayZip(['$browser'], [JSONExtractString(persons.person_props, '$browser')])) AS prop - FROM funnel_actors - JOIN - (SELECT persons.id AS id, - persons.properties AS person_props - FROM - (SELECT person.id AS id, - person.properties AS properties - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0))))) SETTINGS optimize_aggregation_in_order=1) AS persons) AS persons ON equals(persons.id, funnel_actors.actor_id)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING notIn((aggregation_target_with_props.prop).1, []) + assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, + assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, + 2 AS target_step, + ['paid', 'user signed up'] AS funnel_step_names + SELECT event.event AS name, + countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) + and isNull(target_step))) AS success_count, + countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) + or isNotNull(target_step))) AS failure_count + FROM events AS event + JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), notIn(event.event, [])) + GROUP BY name LIMIT 100 - UNION ALL WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT OUTER JOIN - (SELECT tupleElement(argMax(tuple(person_distinct_id_overrides.person_id), person_distinct_id_overrides.version), 1) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(tupleElement(argMax(tuple(person_distinct_id_overrides.is_deleted), person_distinct_id_overrides.version), 1), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('today', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up'))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step + UNION ALL SELECT 'Total_Values_In_Query' AS name, countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) and isNull(target_step))) AS success_count, @@ -1195,15 +1051,15 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties_materialized.1 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.6 ''' - SELECT source.id, - source.id AS id, - source.matching_events AS matching_events + SELECT source.actor_id AS actor_id FROM (WITH funnel_actors AS (SELECT aggregation_target AS actor_id, @@ -1211,17 +1067,11 @@ (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -1234,44 +1084,31 @@ aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 FROM events AS e - LEFT OUTER JOIN - (SELECT tupleElement(argMax(tuple(person_distinct_id_overrides.person_id), person_distinct_id_overrides.version), 1) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(tupleElement(argMax(tuple(person_distinct_id_overrides.is_deleted), person_distinct_id_overrides.version), 1), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('today', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS id - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.id ASC + assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, + assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, + 2 AS target_step, + ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events, + actor_id AS key + FROM events AS event + JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) + and isNull(target_step))) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + ORDER BY source.actor_id ASC LIMIT 101 OFFSET 0 SETTINGS optimize_aggregation_in_order=1, join_algorithm='auto', @@ -1283,37 +1120,15 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties_materialized.2 - ''' - SELECT session_replay_events.session_id AS session_id, - min(toTimeZone(session_replay_events.min_first_timestamp, 'UTC')) AS start_time, - max(session_replay_events.retention_period_days) AS retention_period_days, - plus(dateTrunc('DAY', start_time), toIntervalDay(coalesce(retention_period_days, 30))) AS expiry_time - FROM session_replay_events - WHERE and(equals(session_replay_events.team_id, 99999), in(session_replay_events.session_id, [''])) - GROUP BY session_replay_events.session_id - HAVING and(ifNull(greaterOrEquals(expiry_time, toDateTime64('today', 6, 'UTC')), 0), ifNull(equals(max(session_replay_events.is_deleted), 0), 0)) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties_materialized.3 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.7 ''' - SELECT source.id, - source.id AS id, - source.matching_events AS matching_events + SELECT source.actor_id AS actor_id FROM (WITH funnel_actors AS (SELECT aggregation_target AS actor_id, @@ -1321,17 +1136,11 @@ (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -1344,4695 +1153,60 @@ aggregation_target AS aggregation_target FROM (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, + e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 FROM events AS e - LEFT OUTER JOIN - (SELECT tupleElement(argMax(tuple(person_distinct_id_overrides.person_id), person_distinct_id_overrides.version), 1) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(tupleElement(argMax(tuple(person_distinct_id_overrides.is_deleted), person_distinct_id_overrides.version), 1), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('today', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Positive'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS id - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties_materialized.4 - ''' - SELECT session_replay_events.session_id AS session_id, - min(toTimeZone(session_replay_events.min_first_timestamp, 'UTC')) AS start_time, - max(session_replay_events.retention_period_days) AS retention_period_days, - plus(dateTrunc('DAY', start_time), toIntervalDay(coalesce(retention_period_days, 30))) AS expiry_time - FROM session_replay_events - WHERE and(equals(session_replay_events.team_id, 99999), in(session_replay_events.session_id, [''])) - GROUP BY session_replay_events.session_id - HAVING and(ifNull(greaterOrEquals(expiry_time, toDateTime64('today', 6, 'UTC')), 0), ifNull(equals(max(session_replay_events.is_deleted), 0), 0)) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties_materialized.5 - ''' - SELECT source.id, - source.id AS id, - source.matching_events AS matching_events - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT OUTER JOIN - (SELECT tupleElement(argMax(tuple(person_distinct_id_overrides.person_id), person_distinct_id_overrides.version), 1) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(tupleElement(argMax(tuple(person_distinct_id_overrides.is_deleted), person_distinct_id_overrides.version), 1), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('today', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS id - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties_materialized.6 - ''' - SELECT session_replay_events.session_id AS session_id, - min(toTimeZone(session_replay_events.min_first_timestamp, 'UTC')) AS start_time, - max(session_replay_events.retention_period_days) AS retention_period_days, - plus(dateTrunc('DAY', start_time), toIntervalDay(coalesce(retention_period_days, 30))) AS expiry_time - FROM session_replay_events - WHERE and(equals(session_replay_events.team_id, 99999), in(session_replay_events.session_id, [''])) - GROUP BY session_replay_events.session_id - HAVING and(ifNull(greaterOrEquals(expiry_time, toDateTime64('today', 6, 'UTC')), 0), ifNull(equals(max(session_replay_events.is_deleted), 0), 0)) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties_materialized.7 - ''' - SELECT source.id, - source.id AS id, - source.matching_events AS matching_events - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id) AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT OUTER JOIN - (SELECT tupleElement(argMax(tuple(person_distinct_id_overrides.person_id), person_distinct_id_overrides.version), 1) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING ifNull(equals(tupleElement(argMax(tuple(person_distinct_id_overrides.is_deleted), person_distinct_id_overrides.version), 1), 0), 0) SETTINGS optimize_aggregation_in_order=1) AS e__override ON equals(e.distinct_id, e__override.distinct_id) - LEFT JOIN - (SELECT person.id AS id, - nullIf(nullIf(person.`pmat_$browser`, ''), 'null') AS `properties___$browser` - FROM person - WHERE and(equals(person.team_id, 99999), in(tuple(person.id, person.version), - (SELECT person.id AS id, max(person.version) AS version - FROM person - WHERE equals(person.team_id, 99999) - GROUP BY person.id - HAVING and(ifNull(equals(argMax(person.is_deleted, person.version), 0), 0), ifNull(less(argMax(toTimeZone(person.created_at, 'UTC'), person.version), plus(now64(6, 'UTC'), toIntervalDay(1))), 0))))) SETTINGS optimize_aggregation_in_order=1) AS e__person ON equals(if(not(empty(e__override.distinct_id)), e__override.person_id, e.person_id), e__person.id) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('today', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__person.`properties___$browser`, 'Negative'), 0)), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS id - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_basic_funnel_correlation_with_properties_materialized.8 - ''' - SELECT session_replay_events.session_id AS session_id, - min(toTimeZone(session_replay_events.min_first_timestamp, 'UTC')) AS start_time, - max(session_replay_events.retention_period_days) AS retention_period_days, - plus(dateTrunc('DAY', start_time), toIntervalDay(coalesce(retention_period_days, 30))) AS expiry_time - FROM session_replay_events - WHERE and(equals(session_replay_events.team_id, 99999), in(session_replay_events.session_id, [''])) - GROUP BY session_replay_events.session_id - HAVING and(ifNull(greaterOrEquals(expiry_time, toDateTime64('today', 6, 'UTC')), 0), ifNull(equals(max(session_replay_events.is_deleted), 0), 0)) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_event_properties_and_groups - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_1` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - filtered_events AS - (SELECT events.event AS event, - events.properties AS properties, - toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - events.`$group_1` AS `$group_1`, - accurateCastOrNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$exception_issue_id'), ''), 'null'), '^"|"$', ''), 'UUID') AS event_issue_id, - events__fingerprint_issue_state.issue_id AS issue_id, - events__fingerprint_issue_state.issue_id AS issue_id_v2, - events__fingerprint_issue_state.issue_name AS issue_name, - events__fingerprint_issue_state.issue_description AS issue_description, - events__fingerprint_issue_state.issue_status AS issue_status, - events__fingerprint_issue_state.assigned_user_id AS issue_assigned_user_id, - events__fingerprint_issue_state.assigned_role_id AS issue_assigned_role_id, - events__fingerprint_issue_state.first_seen AS issue_first_seen - FROM events - LEFT OUTER JOIN - (SELECT cityHash64(error_tracking_fingerprint_issue_state.fingerprint) AS fp_hash, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_id), error_tracking_fingerprint_issue_state.version), 1)) AS issue_id, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_name), error_tracking_fingerprint_issue_state.version), 1)) AS issue_name, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_description), error_tracking_fingerprint_issue_state.version), 1)) AS issue_description, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_status), error_tracking_fingerprint_issue_state.version), 1)) AS issue_status, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.assigned_user_id), error_tracking_fingerprint_issue_state.version), 1)) AS assigned_user_id, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.assigned_role_id), error_tracking_fingerprint_issue_state.version), 1)) AS assigned_role_id, - toNullable(tupleElement(argMax(tuple(toTimeZone(error_tracking_fingerprint_issue_state.first_seen, 'UTC')), error_tracking_fingerprint_issue_state.version), 1)) AS first_seen - FROM error_tracking_fingerprint_issue_state - WHERE equals(error_tracking_fingerprint_issue_state.team_id, 99999) - GROUP BY fp_hash - HAVING equals(argMax(error_tracking_fingerprint_issue_state.is_deleted, error_tracking_fingerprint_issue_state.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__fingerprint_issue_state ON equals(cityHash64(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$exception_fingerprint'), ''), 'null'), '^"|"$', '')), events__fingerprint_issue_state.fp_hash) - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), equals(events.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(events.timestamp, 'UTC'), 'UTC'), assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC'))), less(toDateTime(toTimeZone(events.timestamp, 'UTC'), 'UTC'), assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC'))), notIn(events.event, tuple('paid', 'user signed up')), in(events.event, tuple('positively_related', 'negatively_related')))), - 2 AS target_step - SELECT if(ifNull(equals((prop).1, 'Total_Values_In_Query'), 0), 'Total_Values_In_Query', concat(ifNull(toString((prop).1), ''), '::', ifNull(toString((prop).2), ''), '::', ifNull(toString((prop).3), ''))) AS name, - countDistinctIf(actor_id, ifNull(equals(steps, target_step), isNull(steps) - and isNull(target_step))) AS success_count, - countDistinctIf(actor_id, ifNull(notEquals(steps, target_step), isNotNull(steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '', '')], if(empty(event_table.event), [], arrayMap(prop -> tuple(event_table.event, prop.1, prop.2), JSONExtractKeysAndValues(event_table.properties, 'String'))))) AS prop - FROM filtered_events AS event_table - RIGHT JOIN funnel_actors ON and(equals(funnel_actors.actor_id, event_table.`$group_1`), greater(toTimeZone(toDateTime(toTimeZone(event_table.timestamp, 'UTC'), 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toTimeZone(toDateTime(toTimeZone(event_table.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')))))) - GROUP BY name, - prop - HAVING or(equals(name, 'Total_Values_In_Query'), greater(plus(success_count, failure_count), 2)) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_event_properties_and_groups[new_events_schema] - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_1` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('explicit_redacted_timestamp', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - filtered_events AS - (SELECT events.event AS event, - concat('{', arrayStringConcat(arrayMap(kv -> concat(toJSONString(kv.1), ':', kv.2), arrayFilter(kv -> kv.2 != 'null' - AND NOT (kv.2 = '[]' - AND has(['$active_feature_flags', '$exception_functions', '$exception_sources', '$exception_types', '$exception_values'], kv.1)), JSONExtractKeysAndValuesRaw(toJSONString(events.properties)))), ','), '}') AS properties, - toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - events.`$group_1` AS `$group_1`, - accurateCastOrNull(events.properties.`$exception_issue_id`, 'UUID') AS event_issue_id, - events__fingerprint_issue_state.issue_id AS issue_id, - events__fingerprint_issue_state.issue_id AS issue_id_v2, - events__fingerprint_issue_state.issue_name AS issue_name, - events__fingerprint_issue_state.issue_description AS issue_description, - events__fingerprint_issue_state.issue_status AS issue_status, - events__fingerprint_issue_state.assigned_user_id AS issue_assigned_user_id, - events__fingerprint_issue_state.assigned_role_id AS issue_assigned_role_id, - events__fingerprint_issue_state.first_seen AS issue_first_seen - FROM events_json AS events - LEFT OUTER JOIN - (SELECT cityHash64(error_tracking_fingerprint_issue_state.fingerprint) AS fp_hash, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_id), error_tracking_fingerprint_issue_state.version), 1)) AS issue_id, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_name), error_tracking_fingerprint_issue_state.version), 1)) AS issue_name, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_description), error_tracking_fingerprint_issue_state.version), 1)) AS issue_description, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.issue_status), error_tracking_fingerprint_issue_state.version), 1)) AS issue_status, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.assigned_user_id), error_tracking_fingerprint_issue_state.version), 1)) AS assigned_user_id, - toNullable(tupleElement(argMax(tuple(error_tracking_fingerprint_issue_state.assigned_role_id), error_tracking_fingerprint_issue_state.version), 1)) AS assigned_role_id, - toNullable(tupleElement(argMax(tuple(toTimeZone(error_tracking_fingerprint_issue_state.first_seen, 'UTC')), error_tracking_fingerprint_issue_state.version), 1)) AS first_seen - FROM error_tracking_fingerprint_issue_state - WHERE equals(error_tracking_fingerprint_issue_state.team_id, 99999) - GROUP BY fp_hash - HAVING equals(argMax(error_tracking_fingerprint_issue_state.is_deleted, error_tracking_fingerprint_issue_state.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__fingerprint_issue_state ON equals(cityHash64(events.properties.`$exception_fingerprint`), events__fingerprint_issue_state.fp_hash) - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), equals(events.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(events.timestamp, 'UTC'), 'UTC'), assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC'))), less(toDateTime(toTimeZone(events.timestamp, 'UTC'), 'UTC'), assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC'))), notIn(events.event, tuple('paid', 'user signed up')), in(events.event, tuple('positively_related', 'negatively_related')))), - 2 AS target_step - SELECT if(ifNull(equals((prop).1, 'Total_Values_In_Query'), 0), 'Total_Values_In_Query', concat(ifNull(toString((prop).1), ''), '::', ifNull(toString((prop).2), ''), '::', ifNull(toString((prop).3), ''))) AS name, - countDistinctIf(actor_id, ifNull(equals(steps, target_step), isNull(steps) - and isNull(target_step))) AS success_count, - countDistinctIf(actor_id, ifNull(notEquals(steps, target_step), isNotNull(steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '', '')], if(empty(event_table.event), [], arrayMap(prop -> tuple(event_table.event, prop.1, prop.2), JSONExtractKeysAndValues(event_table.properties, 'String'))))) AS prop - FROM filtered_events AS event_table - RIGHT JOIN funnel_actors ON and(equals(funnel_actors.actor_id, event_table.`$group_1`), greater(toTimeZone(toDateTime(toTimeZone(event_table.timestamp, 'UTC'), 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toTimeZone(toDateTime(toTimeZone(event_table.timestamp, 'UTC'), 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')))))) - GROUP BY name, - prop - HAVING or(equals(name, 'Total_Values_In_Query'), greater(plus(success_count, failure_count), 2)) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_event_properties_and_groups_materialized - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_1` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names - SELECT concat(ifNull(toString(event_name), ''), '::', ifNull(toString((prop).1), ''), '::', ifNull(toString((prop).2), '')) AS name, - countDistinctIf(actor_id, ifNull(equals(steps, target_step), isNull(steps) - and isNull(target_step))) AS success_count, - countDistinctIf(actor_id, ifNull(notEquals(steps, target_step), isNotNull(steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - event.event AS event_name, - arrayJoin(JSONExtractKeysAndValues(event.properties, 'String')) AS prop - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_1`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), in(event.event, tuple('positively_related', 'negatively_related')))) - GROUP BY name, - prop - HAVING ifNull(greater(plus(success_count, failure_count), 2), 0) - LIMIT 100 - UNION ALL WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_1` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups - ''' - WITH funnel_actors AS MATERIALIZED - (SELECT aggregation_target AS actor_id, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, af_tuple.5 AS steps_bitfield, aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names - SELECT event.event AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), notIn(event.event, [])) - GROUP BY name - LIMIT 100 - UNION ALL - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.1 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'positively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.2 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'positively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.3 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.4 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.5 - ''' - WITH funnel_actors AS MATERIALIZED - (SELECT aggregation_target AS actor_id, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, af_tuple.5 AS steps_bitfield, aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, groups.group_type_index AS index, groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names - SELECT event.event AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), notIn(event.event, [])) - GROUP BY name - LIMIT 100 - UNION ALL - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.6 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups.7 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups[new_events_schema.1] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events_json AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'positively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups[new_events_schema.2] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events_json AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'positively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups[new_events_schema.3] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events_json AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups[new_events_schema.4] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events_json AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups[new_events_schema.5] - ''' - WITH funnel_actors AS MATERIALIZED - (SELECT aggregation_target AS actor_id, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, af_tuple.5 AS steps_bitfield, aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, groups.group_type_index AS index, groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names - SELECT event.event AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM events_json AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), notIn(event.event, [])) - GROUP BY name - LIMIT 100 - UNION ALL - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups[new_events_schema.6] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events_json AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups[new_events_schema.7] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events_json AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups[new_events_schema] - ''' - WITH funnel_actors AS MATERIALIZED - (SELECT aggregation_target AS actor_id, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, af_tuple.3 AS timings, af_tuple.4 AS matched_event_uuids_array_array, groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, af_tuple.5 AS steps_bitfield, aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, e.`$group_0` AS aggregation_target, e.uuid AS uuid, e.`$session_id` AS `$session_id`, e.`$window_id` AS `$window_id`, if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names - SELECT event.event AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM events_json AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), notIn(event.event, [])) - GROUP BY name - LIMIT 100 - UNION ALL - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups_poe_v2 - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names - SELECT event.event AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), notIn(event.event, [])) - GROUP BY name - LIMIT 100 - UNION ALL WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups_poe_v2.1 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'positively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups_poe_v2.2 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'positively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups_poe_v2.3 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups_poe_v2.4 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups_poe_v2.5 - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names - SELECT event.event AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), notIn(event.event, [])) - GROUP BY name - LIMIT 100 - UNION ALL WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'finance'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups_poe_v2.6 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_events_and_groups_poe_v2.7 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, - assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, - 2 AS target_step, - ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM events AS event - JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) - WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), funnel_actors.first_timestamp), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(funnel_actors.final_timestamp, plus(toTimeZone(funnel_actors.first_timestamp, 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) - GROUP BY actor_id - ORDER BY actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')]))) AS prop - FROM funnel_actors - LEFT JOIN - (SELECT groups.key AS key, - groups.properties AS properties - FROM - (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups.1 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups.2 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups.3 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups.4 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups.5 - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], JSONExtractKeysAndValues(groups_0.properties, 'String'))) AS prop - FROM funnel_actors - LEFT JOIN - (SELECT groups.key AS key, - groups.properties AS properties - FROM - (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups[new_events_schema.1] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups[new_events_schema.2] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups[new_events_schema.3] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups[new_events_schema.4] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups[new_events_schema.5] - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], JSONExtractKeysAndValues(groups_0.properties, 'String'))) AS prop - FROM funnel_actors - LEFT JOIN - (SELECT groups.key AS key, - groups.properties AS properties - FROM - (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups[new_events_schema] - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')]))) AS prop - FROM funnel_actors - LEFT JOIN - (SELECT groups.key AS key, - groups.properties AS properties - FROM - (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_materialized - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')])) AS prop - FROM funnel_actors - LEFT JOIN - (SELECT groups.key AS key, - groups.properties AS properties - FROM - (SELECT tupleElement(argMax(tuple(groups.group_properties), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING notIn((aggregation_target_with_props.prop).1, []) - LIMIT 100 - UNION ALL WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_materialized.1 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_materialized.2 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_materialized.3 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_materialized.4 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_materialized.5 - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(JSONExtractKeysAndValues(groups_0.properties, 'String')) AS prop - FROM funnel_actors - LEFT JOIN - (SELECT groups.key AS key, - groups.properties AS properties - FROM - (SELECT tupleElement(argMax(tuple(groups.group_properties), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING notIn((aggregation_target_with_props.prop).1, []) - LIMIT 100 - UNION ALL WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')]))) AS prop - FROM funnel_actors - LEFT JOIN - (SELECT groups.key AS key, - groups.properties AS properties - FROM - (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events.1 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events.2 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events.3 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events.4 - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events.5 - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], JSONExtractKeysAndValues(groups_0.properties, 'String'))) AS prop - FROM funnel_actors - LEFT JOIN - (SELECT groups.key AS key, - groups.properties AS properties - FROM - (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events[new_events_schema.1] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events[new_events_schema.2] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events[new_events_schema.3] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events[new_events_schema.4] - ''' - SELECT source.actor_id AS actor_id - FROM - (WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - matched_events_array[plus(step_reached, 1)] AS matching_events, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step SELECT funnel_actors.actor_id AS actor_id, - any(funnel_actors.matching_events) AS matching_events, - actor_id AS key - FROM funnel_actors - WHERE ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step)) - GROUP BY funnel_actors.actor_id - ORDER BY funnel_actors.actor_id ASC) AS source - ORDER BY source.actor_id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events[new_events_schema.5] - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], JSONExtractKeysAndValues(groups_0.properties, 'String'))) AS prop - FROM funnel_actors - LEFT JOIN - (SELECT groups.key AS key, - groups.properties AS properties - FROM - (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events[new_events_schema] - ''' - WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events_json AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) - or isNotNull(target_step))) AS failure_count - FROM - (SELECT funnel_actors.actor_id AS actor_id, - funnel_actors.steps AS steps, - arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')]))) AS prop - FROM funnel_actors - LEFT JOIN - (SELECT groups.key AS key, - groups.properties AS properties - FROM - (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE equals(groups.team_id, 99999) - GROUP BY groups.group_type_index, - groups.group_key) AS groups - WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props - GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) - LIMIT 100 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 + assumeNotNull(toDateTime('2020-01-01 00:00:00', 'UTC')) AS date_from, + assumeNotNull(toDateTime('2020-01-14 23:59:59', 'UTC')) AS date_to, + 2 AS target_step, + ['paid', 'user signed up'] AS funnel_step_names SELECT funnel_actors.actor_id AS actor_id, + any(funnel_actors.matching_events) AS matching_events, + actor_id AS key + FROM events AS event + JOIN funnel_actors ON equals(funnel_actors.actor_id, event.`$group_0`) + WHERE and(equals(event.team_id, 99999), greaterOrEquals(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_from), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), date_to), equals(event.team_id, 99999), greater(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), toTimeZone(funnel_actors.first_timestamp, 'UTC')), less(toDateTime(toTimeZone(event.timestamp, 'UTC'), 'UTC'), coalesce(toTimeZone(funnel_actors.final_timestamp, 'UTC'), plus(toTimeZone(toTimeZone(funnel_actors.first_timestamp, 'UTC'), 'UTC'), toIntervalDay(14)), date_to)), notIn(event.event, funnel_step_names), equals(event.event, 'negatively_related'), ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) + or isNotNull(target_step))) + GROUP BY actor_id + ORDER BY actor_id ASC) AS source + ORDER BY source.actor_id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups ''' WITH funnel_actors AS (SELECT aggregation_target AS actor_id, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -6052,13 +1226,13 @@ if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), 2 AS target_step - SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) and isNull(target_step))) AS success_count, countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) @@ -6066,70 +1240,22 @@ FROM (SELECT funnel_actors.actor_id AS actor_id, funnel_actors.steps AS steps, - arrayJoin(arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')])) AS prop + arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')]))) AS prop FROM funnel_actors LEFT JOIN (SELECT groups.key AS key, groups.properties AS properties FROM - (SELECT tupleElement(argMax(tuple(groups.group_properties), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties, + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, groups.group_type_index AS index, groups.group_key AS key FROM groups WHERE equals(groups.team_id, 99999) GROUP BY groups.group_type_index, groups.group_key) AS groups - WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING notIn((aggregation_target_with_props.prop).1, []) - LIMIT 100 - UNION ALL WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors + HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -6138,11 +1264,13 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized.1 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups.1 ''' SELECT source.actor_id AS actor_id FROM @@ -6152,17 +1280,11 @@ (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -6190,7 +1312,7 @@ WHERE and(equals(groups.team_id, 99999), equals(index, 0)) GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) @@ -6215,11 +1337,13 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized.2 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups.2 ''' SELECT source.actor_id AS actor_id FROM @@ -6229,17 +1353,11 @@ (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -6267,7 +1385,7 @@ WHERE and(equals(groups.team_id, 99999), equals(index, 0)) GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) @@ -6292,11 +1410,13 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized.3 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups.3 ''' SELECT source.actor_id AS actor_id FROM @@ -6306,17 +1426,11 @@ (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -6344,7 +1458,7 @@ WHERE and(equals(groups.team_id, 99999), equals(index, 0)) GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) @@ -6369,11 +1483,13 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized.4 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups.4 ''' SELECT source.actor_id AS actor_id FROM @@ -6383,17 +1499,11 @@ (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -6421,7 +1531,7 @@ WHERE and(equals(groups.team_id, 99999), equals(index, 0)) GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) @@ -6446,28 +1556,24 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_materialized.5 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups.5 ''' WITH funnel_actors AS (SELECT aggregation_target AS actor_id, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -6487,13 +1593,13 @@ if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), 2 AS target_step - SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) and isNull(target_step))) AS success_count, countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) @@ -6501,70 +1607,22 @@ FROM (SELECT funnel_actors.actor_id AS actor_id, funnel_actors.steps AS steps, - arrayJoin(JSONExtractKeysAndValues(groups_0.properties, 'String')) AS prop + arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], JSONExtractKeysAndValues(groups_0.properties, 'String'))) AS prop FROM funnel_actors LEFT JOIN (SELECT groups.key AS key, groups.properties AS properties FROM - (SELECT tupleElement(argMax(tuple(groups.group_properties), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties, + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, groups.group_type_index AS index, groups.group_key AS key FROM groups WHERE equals(groups.team_id, 99999) GROUP BY groups.group_type_index, groups.group_key) AS groups - WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING notIn((aggregation_target_with_props.prop).1, []) - LIMIT 100 - UNION ALL WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors + HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -6573,28 +1631,24 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events ''' WITH funnel_actors AS (SELECT aggregation_target AS actor_id, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -6614,13 +1668,13 @@ if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), 2 AS target_step - SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) and isNull(target_step))) AS success_count, countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) @@ -6628,70 +1682,22 @@ FROM (SELECT funnel_actors.actor_id AS actor_id, funnel_actors.steps AS steps, - arrayJoin(arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')])) AS prop + arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], arrayZip(['industry'], [JSONExtractString(groups_0.properties, 'industry')]))) AS prop FROM funnel_actors LEFT JOIN (SELECT groups.key AS key, groups.properties AS properties FROM - (SELECT tupleElement(argMax(tuple(groups.group_properties), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties, + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, groups.group_type_index AS index, groups.group_key AS key FROM groups WHERE equals(groups.team_id, 99999) GROUP BY groups.group_type_index, groups.group_key) AS groups - WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING notIn((aggregation_target_with_props.prop).1, []) - LIMIT 100 - UNION ALL WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors + HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -6700,11 +1706,13 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2.1 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events.1 ''' SELECT source.actor_id AS actor_id FROM @@ -6714,17 +1722,11 @@ (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -6752,7 +1754,7 @@ WHERE and(equals(groups.team_id, 99999), equals(index, 0)) GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) @@ -6777,11 +1779,13 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2.2 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events.2 ''' SELECT source.actor_id AS actor_id FROM @@ -6791,17 +1795,11 @@ (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -6829,7 +1827,7 @@ WHERE and(equals(groups.team_id, 99999), equals(index, 0)) GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'positive'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) @@ -6854,11 +1852,13 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2.3 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events.3 ''' SELECT source.actor_id AS actor_id FROM @@ -6868,17 +1868,11 @@ (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -6906,7 +1900,7 @@ WHERE and(equals(groups.team_id, 99999), equals(index, 0)) GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) @@ -6931,11 +1925,13 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2.4 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events.4 ''' SELECT source.actor_id AS actor_id FROM @@ -6945,17 +1941,11 @@ (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -6983,7 +1973,7 @@ WHERE and(equals(groups.team_id, 99999), equals(index, 0)) GROUP BY groups.group_type_index, groups.group_key) AS e__group_0 ON equals(e.`$group_0`, e__group_0.key) - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), ifNull(equals(e__group_0.properties___industry, 'negative'), 0), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) @@ -7008,28 +1998,24 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' # --- -# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events_poe_v2.5 +# name: TestClickhouseFunnelCorrelation.test_funnel_correlation_with_properties_and_groups_person_on_events.5 ''' WITH funnel_actors AS (SELECT aggregation_target AS actor_id, (matched_events_array[1][1]).1 AS timestamp, nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp + steps AS steps FROM (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, + arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(lessOrEquals(length(x.4), 1), equals(x.4, x_before.4), equals(x.4, x_after.4), equals(x.3, x_before.3), equals(x.3, x_after.3), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, af_tuple.1 AS step_reached, plus(af_tuple.1, 1) AS steps, af_tuple.2 AS breakdown, @@ -7049,13 +2035,13 @@ if(equals(e.event, 'user signed up'), 1, 0) AS step_0, if(equals(e.event, 'paid'), 1, 0) AS step_1 FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) + WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(equals(step_0, 1), equals(step_1, 1)))) GROUP BY aggregation_target HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) WHERE bitTest(steps_bitfield, 0) ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), 2 AS target_step - SELECT concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), '')) AS name, + SELECT if(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), 'Total_Values_In_Query', concat(ifNull(toString((aggregation_target_with_props.prop).1), ''), '::', ifNull(toString((aggregation_target_with_props.prop).2), ''))) AS name, countDistinctIf(aggregation_target_with_props.actor_id, ifNull(equals(aggregation_target_with_props.steps, target_step), isNull(aggregation_target_with_props.steps) and isNull(target_step))) AS success_count, countDistinctIf(aggregation_target_with_props.actor_id, ifNull(notEquals(aggregation_target_with_props.steps, target_step), isNotNull(aggregation_target_with_props.steps) @@ -7063,70 +2049,22 @@ FROM (SELECT funnel_actors.actor_id AS actor_id, funnel_actors.steps AS steps, - arrayJoin(JSONExtractKeysAndValues(groups_0.properties, 'String')) AS prop + arrayJoin(arrayConcat([tuple('Total_Values_In_Query', '')], JSONExtractKeysAndValues(groups_0.properties, 'String'))) AS prop FROM funnel_actors LEFT JOIN (SELECT groups.key AS key, groups.properties AS properties FROM - (SELECT tupleElement(argMax(tuple(groups.group_properties), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties, + (SELECT argMax(groups.group_properties, toTimeZone(groups._timestamp, 'UTC')) AS properties, groups.group_type_index AS index, groups.group_key AS key FROM groups WHERE equals(groups.team_id, 99999) GROUP BY groups.group_type_index, groups.group_key) AS groups - WHERE ifNull(equals(groups.index, 0), 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props + WHERE equals(groups.index, 0)) AS groups_0 ON equals(funnel_actors.actor_id, groups_0.key)) AS aggregation_target_with_props GROUP BY (aggregation_target_with_props.prop).1, (aggregation_target_with_props.prop).2 - HAVING notIn((aggregation_target_with_props.prop).1, []) - LIMIT 100 - UNION ALL WITH funnel_actors AS - (SELECT aggregation_target AS actor_id, - (matched_events_array[1][1]).1 AS timestamp, - nullIf((matched_events_array[2][1]).1, 0) AS final_timestamp, - (matched_events_array[1][1]).1 AS first_timestamp, - steps AS steps, - final_timestamp, - first_timestamp - FROM - (SELECT arraySort(t -> t.1, groupArray(tuple(accurateCastOrNull(timestamp, 'Float64'), uuid, '', arrayFilter(x -> ifNull(notEquals(x, 0), 1), [multiply(1, step_0), multiply(2, step_1)])))) AS events_array, - [''] AS prop, - arrayJoin(aggregate_funnel(2, 1209600, 'first_touch', 'ordered', prop, [], arrayFilter((x, x_before, x_after) -> not(and(ifNull(lessOrEquals(length(x.4), 1), 0), ifNull(equals(x.4, x_before.4), isNull(x.4) - and isNull(x_before.4)), ifNull(equals(x.4, x_after.4), isNull(x.4) - and isNull(x_after.4)), ifNull(equals(x.3, x_before.3), isNull(x.3) - and isNull(x_before.3)), ifNull(equals(x.3, x_after.3), isNull(x.3) - and isNull(x_after.3)), ifNull(greater(x.1, x_before.1), 0), ifNull(less(x.1, x_after.1), 0))), events_array, arrayRotateRight(events_array, 1), arrayRotateLeft(events_array, 1)))) AS af_tuple, - af_tuple.1 AS step_reached, - plus(af_tuple.1, 1) AS steps, - af_tuple.2 AS breakdown, - af_tuple.3 AS timings, - af_tuple.4 AS matched_event_uuids_array_array, - groupArray(tuple(timestamp, uuid, `$session_id`, `$window_id`)) AS user_events, - mapFromArrays(arrayMap(x -> x.2, user_events), user_events) AS user_events_map, - arrayMap(matched_event_uuids_array -> arrayMap(event_uuid -> user_events_map[event_uuid], arrayDistinct(matched_event_uuids_array)), matched_event_uuids_array_array) AS matched_events_array, - af_tuple.5 AS steps_bitfield, - aggregation_target AS aggregation_target - FROM - (SELECT toTimeZone(e.timestamp, 'UTC') AS timestamp, - e.`$group_0` AS aggregation_target, - e.uuid AS uuid, - e.`$session_id` AS `$session_id`, - e.`$window_id` AS `$window_id`, - if(equals(e.event, 'user signed up'), 1, 0) AS step_0, - if(equals(e.event, 'paid'), 1, 0) AS step_1 - FROM events AS e - WHERE and(equals(e.team_id, 99999), and(and(greaterOrEquals(e.timestamp, toDateTime64('2020-01-01 00:00:00.000000', 6, 'UTC')), lessOrEquals(e.timestamp, toDateTime64('2020-01-14 23:59:59.999999', 6, 'UTC'))), in(e.event, tuple('paid', 'user signed up')), and(notEquals(toString(aggregation_target), ''), isNotNull(aggregation_target))), or(ifNull(equals(step_0, 1), 0), ifNull(equals(step_1, 1), 0)))) - GROUP BY aggregation_target - HAVING ifNull(greaterOrEquals(step_reached, 0), 0)) - WHERE bitTest(steps_bitfield, 0) - ORDER BY aggregation_target ASC SETTINGS join_algorithm='auto'), - 2 AS target_step - SELECT 'Total_Values_In_Query' AS name, - countDistinctIf(funnel_actors.actor_id, ifNull(equals(funnel_actors.steps, target_step), isNull(funnel_actors.steps) - and isNull(target_step))) AS success_count, - countDistinctIf(funnel_actors.actor_id, ifNull(notEquals(funnel_actors.steps, target_step), isNotNull(funnel_actors.steps) - or isNotNull(target_step))) AS failure_count - FROM funnel_actors + HAVING or(equals((aggregation_target_with_props.prop).1, 'Total_Values_In_Query'), notIn((aggregation_target_with_props.prop).1, [])) LIMIT 100 SETTINGS readonly=2, max_execution_time=60, allow_experimental_object_type=1, @@ -7135,6 +2073,8 @@ max_bytes_before_external_group_by=0, transform_null_in=1, optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, allow_experimental_join_condition=1, use_hive_partitioning=0 ''' From 6c553dd051378bc51a01214211064feaed0d38e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Tue, 11 Aug 2026 10:03:28 +0200 Subject: [PATCH 012/289] refactor(slack): ci vocab vetoes the short-circuit --- posthog/git.py | 4 +- .../ai/slack_app/activities/classifiers.py | 46 ++++----------- .../ai/slack_app/eval_slack_repo_selection.py | 13 +++++ .../tests/ai/test_classify_task_needs_repo.py | 57 ++++++++++++++----- posthog/test/test_git.py | 1 + .../backend/tests/test_guess_repository.py | 22 ++----- 6 files changed, 74 insertions(+), 69 deletions(-) diff --git a/posthog/git.py b/posthog/git.py index 1659295efabb..5ce02b0a0cef 100644 --- a/posthog/git.py +++ b/posthog/git.py @@ -84,7 +84,9 @@ def extract_explicit_repo(text: str, all_repos: list[str]) -> str | None: Links only resolve when the message points at a single connected repo. Two different linked repos is genuine ambiguity, and returning None lets the caller disambiguate (the Slack cascade falls through to its discovery agent, then a repo picker) rather - than silently starting work in whichever was pasted first. + than silently starting work in whichever was pasted first. A link someone labeled + `` reads as a typed token and wins outright; Slack writes its own + labels as `github.com/owner/repo/…`, which read as links. Pure helper (no Django / heavy deps) so any product can import it downward from core. """ diff --git a/posthog/temporal/ai/slack_app/activities/classifiers.py b/posthog/temporal/ai/slack_app/activities/classifiers.py index 94978bf40019..ff6a9f6609af 100644 --- a/posthog/temporal/ai/slack_app/activities/classifiers.py +++ b/posthog/temporal/ai/slack_app/activities/classifiers.py @@ -54,37 +54,6 @@ AGENT_DIRECTED_TIMEOUT_SECONDS = 20.0 AGENT_DIRECTED_MAX_RETRIES = 1 -# Nothing else in a repo is called "flaky", and a workflow-run URL is only ever CI. -_CI_UNAMBIGUOUS_PATTERNS = ( - r"\bde-?flak", - r"\bflak(?:y|e|es|iness)\b", - r"\bmerge queue\b", - r"github\.com/[\w.-]+/[\w.-]+/actions\b", -) -# Both halves required, so "check the test dashboard" stays an analytics ask. -_CI_SUBJECT_PATTERNS = (r"\btests?\b", r"\bspecs?\b", r"\bsuites?\b", r"\bshards?\b", r"\bci\b", r"\bmaster\b") -_CI_FAILURE_PATTERNS = ( - r"\bfail(?:s|ed|ing|ure|ures)?\b", - r"\bred\b", - r"\bbroke(?:n)?\b", - r"\btimed?\s?out\b", - r"\berror(?:s|ing)?\b", -) - - -def _is_ci_failure_ask(normalized: str) -> bool: - """Whether the conversation is about a broken or flaky CI run. - - Checked first because CI work is code work wearing none of the usual tells — a flaky - test report rarely names a file — and because it often mentions a product noun ("the - experiment insight test is flaky") that would short-circuit the classifier to no-repo. - """ - if any(re.search(pattern, normalized) for pattern in _CI_UNAMBIGUOUS_PATTERNS): - return True - return any(re.search(pattern, normalized) for pattern in _CI_SUBJECT_PATTERNS) and any( - re.search(pattern, normalized) for pattern in _CI_FAILURE_PATTERNS - ) - def classify_task_needs_repo( event_text: str, @@ -103,9 +72,6 @@ def classify_task_needs_repo( conversation = "\n".join(f"{msg['user']}: {msg['text']}" for msg in thread_messages) normalized = f"{conversation}\nLatest message: {event_text}".lower() - if _is_ci_failure_ask(normalized): - return True - # Substring match: keep the shortest form that uniquely identifies the # concept without colliding with code-review vocabulary. Plurals are used # only when the singular substring-matches a common non-analytics word @@ -159,6 +125,11 @@ def classify_task_needs_repo( r"\bserializer\b", r"\bviewset\b", r"\bmigration\b", + # A failing test is code work, but it is usually named after the feature it covers, + # so the product terms above would answer no-repo before the model reads the sentence. + r"\bci\b", + r"\bflak(?:y|e|es|iness)\b", + r"\bmerge queue\b", ) if any(term in normalized for term in product_debug_terms) and not any( @@ -184,7 +155,12 @@ def classify_task_needs_repo( "the team's code → no_repo. Important exception: 'wrong data', 'missing events', or " "'numbers look off' in PostHog usually means the team's tracking code is broken (wrong " "event names, identification logic, SDK setup) — that's a code fix in their repo → " - "needs_repo. When in doubt, lean needs_repo=false — code-focused tasks usually carry " + "needs_repo.\n\n" + "A failing, broken, or flaky CI run, test suite, or build is work in the team's own " + "repository → needs_repo, including when the test is named after a PostHog feature " + "('the experiment insight test is flaky'): the subject is their test, not our " + "product.\n\n" + "When in doubt, lean needs_repo=false — code-focused tasks usually carry " "explicit signals (file extensions, 'PR', 'commit', framework names, function or class " "names). Analytics, data, and configuration asks are the common case and should not send " "us hunting for a repository on a guess.\n\n" diff --git a/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py b/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py index 74f204f41f76..387b4b16c34f 100644 --- a/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py +++ b/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py @@ -148,6 +148,19 @@ def status(self) -> Literal["PASS", "FAIL", "SKIP"]: expected_stage="cascade", expected_outcome="auto", ), + Case( + name="ci_run_link", + description="Cascade reads the repo out of a workflow-run link, so a CI ask never reaches the agent.", + text_template="@PostHog is this flaky? https://github.com/{first_repo}/actions/runs/30560492835", + thread_messages=[ + { + "user": "tester", + "text": "@PostHog is this flaky? https://github.com/{first_repo}/actions/runs/30560492835", + } + ], + expected_stage="cascade", + expected_outcome="auto", + ), # --- Haiku gate short-circuits (heuristic + LLM) --------------------------- Case( name="billing_question", diff --git a/posthog/temporal/tests/ai/test_classify_task_needs_repo.py b/posthog/temporal/tests/ai/test_classify_task_needs_repo.py index c8a54f3dbd91..42418be15181 100644 --- a/posthog/temporal/tests/ai/test_classify_task_needs_repo.py +++ b/posthog/temporal/tests/ai/test_classify_task_needs_repo.py @@ -37,25 +37,50 @@ class TestClassifyTaskNeedsRepo: ("analytics_hogql", "write a hogql query to count signups by country", False), ("flag_search", "find the feature flag for the new onboarding", False), ("replay_question", "show me session replays of failed checkouts", False), - # Real #flakey-tests asks. None names a file or says "PR"/"commit", and - # several carry a product noun that used to short-circuit them to no-repo. - ( - "ci_is_this_flaky", - "is this flaky? https://github.com/posthog/posthog/actions/runs/30560492835/job/90936416640", - True, - ), - ("ci_master_broken", "django tests failing on master, investigate", True), - ("ci_rust_in_pr", "why is rust CI failing in this PR?", True), - ("ci_product_noun_no_longer_short_circuits", "the experiment insight test is flaky", True), - ("ci_merge_queue", "seeing a lot of pending failures in the trunk merge queue", True), - # The failure word alone is not CI — this is still a product ask. - ("product_failure_without_ci_subject", "the dashboard fails to load for this user", False), ] ) def test_heuristic_classification(self, _name, text, expected): result = classify_task_needs_repo(text, [{"user": "Alessandro", "text": text}]) assert result is expected + @parameterized.expand( + [ + # Real #flakey-tests asks, each carrying a product noun that short-circuits + # the heuristic to no-repo unless the CI vocabulary vetoes it. + ("flaky_test_named_after_a_feature", "the experiment insight test is flaky"), + ("merge_queue", "the merge queue keeps failing on the experiment insight tests"), + ("ci_on_a_product_pr", "CI is red on the dashboard PR, can you take a look"), + ] + ) + def test_ci_vocabulary_leaves_the_call_to_the_llm(self, _name, text): + assert self._run_with_llm_content(text, '{"needs_repo": true}') is True + + @parameterized.expand( + [ + # Both halves of a CI ask, split across a thread the way people actually talk. + # A vocabulary that pairs any subject word with any failure word reads these as + # CI and spends a discovery-agent sandbox run on an analytics question. + ( + "tests_and_errors_in_an_analytics_thread", + [ + {"user": "amy", "text": "we ran some tests on the signup funnel yesterday"}, + {"user": "bo", "text": "the numbers look off, error rate is way up in the dashboard"}, + ], + "why did conversion drop?", + ), + ( + "master_chatter_beside_a_product_bug", + [ + {"user": "amy", "text": "just merged that to master"}, + {"user": "bo", "text": "the survey widget throws an error on mobile"}, + ], + "what does the data say?", + ), + ] + ) + def test_product_ask_short_circuits_before_the_llm(self, _name, thread_messages, event_text): + assert self._run_with_llm_content(event_text, '{"needs_repo": true}', thread_messages) is False + def test_llm_path_returns_true_when_model_says_needs_repo(self): """Ask with no heuristic signal — classifier must defer to the LLM.""" text = "open a PR in posthog/posthog to fix this serializer" @@ -84,7 +109,9 @@ def test_llm_response_shapes(self, _name, content, expected): result = self._run_with_llm_content(text, content) assert result is expected - def _run_with_llm_content(self, text: str, content: str) -> bool: + def _run_with_llm_content( + self, text: str, content: str, thread_messages: list[dict[str, str]] | None = None + ) -> bool: fake_response = MagicMock() fake_response.choices = [MagicMock(message=MagicMock(content=content))] fake_client = MagicMock() @@ -93,7 +120,7 @@ def _run_with_llm_content(self, text: str, content: str) -> bool: "posthog.temporal.ai.slack_app.activities.classifiers.get_llm_client", return_value=fake_client, ): - return classify_task_needs_repo(text, [{"user": "Alessandro", "text": text}]) + return classify_task_needs_repo(text, thread_messages or [{"user": "Alessandro", "text": text}]) def test_llm_failure_defaults_to_false(self): """A flaky LLM call must not wall users behind the Connect-GitHub gate.""" diff --git a/posthog/test/test_git.py b/posthog/test/test_git.py index b3e6638689aa..c325c735fad6 100644 --- a/posthog/test/test_git.py +++ b/posthog/test/test_git.py @@ -40,6 +40,7 @@ class TestExtractExplicitRepo: "fix posthog/posthog-js — context: https://github.com/posthog/posthog/pull/1", "posthog/posthog-js", ), + ("two_bare_tokens_first_wins", "check posthog/posthog-js then posthog/posthog", "posthog/posthog-js"), ( "two_linked_repos_is_ambiguous", "https://github.com/posthog/posthog/pull/1 broke https://github.com/posthog/posthog-js/actions/runs/2", diff --git a/products/slack_app/backend/tests/test_guess_repository.py b/products/slack_app/backend/tests/test_guess_repository.py index 07655dd6c949..e0da9e7e280d 100644 --- a/products/slack_app/backend/tests/test_guess_repository.py +++ b/products/slack_app/backend/tests/test_guess_repository.py @@ -319,25 +319,11 @@ def test_prewarm_calls_get_full_repo_names(self, mock_slack_cls, mock_github_cla class TestExtractExplicitRepo: - @parameterized.expand( - [ - ("simple", "fix posthog/posthog-js please", "posthog/posthog-js"), - ("no_match", "hello world", None), - ("case_insensitive", "check PostHog/PostHog", "posthog/posthog"), - ("github_url", "see https://github.com/posthog/posthog/issues/1", "posthog/posthog"), - ("backticks", "please fix `posthog/posthog-js`", "posthog/posthog-js"), - ( - "slack_link_label", - "use ", - "posthog/posthog-js", - ), - ("multiple_first_wins", "check posthog/posthog-js then posthog/posthog", "posthog/posthog-js"), - ("with_bot_mention", "<@U123> fix posthog/posthog-js", "posthog/posthog-js"), - ] - ) - def test_extract_explicit_repo(self, _name, text, expected): + # Matching is covered where the helper lives, in posthog/test/test_git.py. All this + # wrapper adds is stripping the bot mention off the front of the Slack message. + def test_strips_bot_mention_before_matching(self): repos = ["posthog/posthog", "posthog/posthog-js", "posthog/plugin-server"] - assert _extract_explicit_repo(text, repos) == expected + assert _extract_explicit_repo("<@U123> fix posthog/posthog-js", repos) == "posthog/posthog-js" class TestParseRulesCommand: From 8e309a634e7127e7331e3d0a372f8fca560ddd0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Tue, 11 Aug 2026 15:29:27 +0200 Subject: [PATCH 013/289] fix(slack): narrow ci vocabulary, correct contracts --- posthog/git.py | 12 ++++++------ .../temporal/ai/slack_app/activities/classifiers.py | 3 ++- .../tests/ai/test_classify_task_needs_repo.py | 5 ++--- posthog/test/test_git.py | 2 ++ .../tasks/backend/logic/repo_selection/cascade.py | 11 ++++++----- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/posthog/git.py b/posthog/git.py index 5ce02b0a0cef..48b38b9cd578 100644 --- a/posthog/git.py +++ b/posthog/git.py @@ -73,7 +73,7 @@ def _repo_from_github_url(token: str) -> str | None: def extract_explicit_repo(text: str, all_repos: list[str]) -> str | None: - """Return the repo named in `text` that matches a connected repo, if exactly one is. + """Return the connected repo `text` names: the first one typed, or the only one linked. Two tiers of evidence, strongest first: a bare `owner/repo` token, then a `github.com/owner/repo…` URL of any depth (a run, a pull request, a file permalink). @@ -82,11 +82,11 @@ def extract_explicit_repo(text: str, all_repos: list[str]) -> str | None: platform-specific noise (e.g. bot mentions) by the caller. Links only resolve when the message points at a single connected repo. Two different - linked repos is genuine ambiguity, and returning None lets the caller disambiguate - (the Slack cascade falls through to its discovery agent, then a repo picker) rather - than silently starting work in whichever was pasted first. A link someone labeled - `` reads as a typed token and wins outright; Slack writes its own - labels as `github.com/owner/repo/…`, which read as links. + linked repos is genuine ambiguity, and None is the answer every caller can act on: + Slack falls through to its discovery agent and then a repo picker, and the callers + with no such fallback start repo-less rather than on whichever was pasted first. A + link someone labeled `` reads as a typed token and wins outright; + Slack writes its own labels as `github.com/owner/repo/…`, which read as links. Pure helper (no Django / heavy deps) so any product can import it downward from core. """ diff --git a/posthog/temporal/ai/slack_app/activities/classifiers.py b/posthog/temporal/ai/slack_app/activities/classifiers.py index ff6a9f6609af..547b0c8fb003 100644 --- a/posthog/temporal/ai/slack_app/activities/classifiers.py +++ b/posthog/temporal/ai/slack_app/activities/classifiers.py @@ -127,7 +127,8 @@ def classify_task_needs_repo( r"\bmigration\b", # A failing test is code work, but it is usually named after the feature it covers, # so the product terms above would answer no-repo before the model reads the sentence. - r"\bci\b", + # Both match the whole thread, so keep them narrow enough that passing chatter cannot + # veto an analytics ask: a bare "ci" also means confidence interval here. r"\bflak(?:y|e|es|iness)\b", r"\bmerge queue\b", ) diff --git a/posthog/temporal/tests/ai/test_classify_task_needs_repo.py b/posthog/temporal/tests/ai/test_classify_task_needs_repo.py index 42418be15181..2352058f955f 100644 --- a/posthog/temporal/tests/ai/test_classify_task_needs_repo.py +++ b/posthog/temporal/tests/ai/test_classify_task_needs_repo.py @@ -45,11 +45,10 @@ def test_heuristic_classification(self, _name, text, expected): @parameterized.expand( [ - # Real #flakey-tests asks, each carrying a product noun that short-circuits - # the heuristic to no-repo unless the CI vocabulary vetoes it. + # Each ask carries a product noun that short-circuits the heuristic to + # no-repo unless the CI vocabulary vetoes it first. ("flaky_test_named_after_a_feature", "the experiment insight test is flaky"), ("merge_queue", "the merge queue keeps failing on the experiment insight tests"), - ("ci_on_a_product_pr", "CI is red on the dashboard PR, can you take a look"), ] ) def test_ci_vocabulary_leaves_the_call_to_the_llm(self, _name, text): diff --git a/posthog/test/test_git.py b/posthog/test/test_git.py index c325c735fad6..1f958656bf0c 100644 --- a/posthog/test/test_git.py +++ b/posthog/test/test_git.py @@ -35,6 +35,8 @@ class TestExtractExplicitRepo: ("unconnected_repo_url", "see https://github.com/acme/widgets/pull/1", None), ("lookalike_host", "see https://mygithub.com/posthog/posthog/pull/1", None), ("host_prefix_spoof", "see https://github.com.evil.tld/posthog/posthog", None), + ("org_url_names_no_repo", "see https://github.com/posthog", None), + ("unparseable_url_is_not_an_error", "see https://[::1/posthog/posthog", None), ( "bare_token_beats_later_url", "fix posthog/posthog-js — context: https://github.com/posthog/posthog/pull/1", diff --git a/products/tasks/backend/logic/repo_selection/cascade.py b/products/tasks/backend/logic/repo_selection/cascade.py index 7078d2c56df0..136bafbd3e25 100644 --- a/products/tasks/backend/logic/repo_selection/cascade.py +++ b/products/tasks/backend/logic/repo_selection/cascade.py @@ -22,10 +22,10 @@ def cascade_select_repository( """Pick a connected repository without the sandbox-backed selection agent. Resolves only the trivial cases: with ``single_repo_wins``, a lone connected repo is taken - directly; otherwise the message has to name a connected ``owner/repo`` explicitly. Anything - ambiguous returns `None` and the caller starts a repo-less run rather than paying for agentic - discovery. Selection must never block that run from starting, so this never raises — every - failure degrades to "no repo". + directly; otherwise the message has to name a connected ``owner/repo``, as a typed token or + as a GitHub link to exactly one connected repo. Anything ambiguous returns `None` and the + caller starts a repo-less run rather than paying for agentic discovery. Selection must never + block that run from starting, so this never raises — every failure degrades to "no repo". ``user_id`` is passed as the requester, so their own connected GitHub stands in when the team has no team-level integration (their own credentials, not a cross-account leak), letting them @@ -58,6 +58,7 @@ async def select_repository_for_message( The sandbox conversation open path must stay fast: it runs before the Run is created, so we avoid the repo-selection LLM agent here. A lone connected repo is deliberately *not* assumed — - an unprompted mention shouldn't pin a sandbox to a repo the user never named. + an unprompted mention shouldn't pin a sandbox to a repo the user never named. Pasting a link + to one counts as naming it; two different linked repos pin nothing. """ return await database_sync_to_async(cascade_select_repository, thread_sensitive=False)(team_id, user_id, message) From 06b1e5a55a114766ee9fe4062fd119c3728a3dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Tue, 11 Aug 2026 15:32:36 +0200 Subject: [PATCH 014/289] refactor(slack): flatten the repo candidate loop --- posthog/git.py | 25 ++++++++++--------- .../ai/slack_app/activities/classifiers.py | 7 +++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/posthog/git.py b/posthog/git.py index 48b38b9cd578..b0cbcac5a0d1 100644 --- a/posthog/git.py +++ b/posthog/git.py @@ -52,6 +52,10 @@ def get_git_branch() -> Optional[str]: _TOKEN_PUNCTUATION = "`'\"()[]{}<>,.;:!?" _GITHUB_HOSTS = frozenset({"github.com", "www.github.com"}) +_REPO_TOKEN = re.compile(r"[\w.-]+/[\w.-]+") +# Slack formats links as and either side can carry the repo, so `|` separates +# candidates the same way whitespace does. +_CANDIDATE_SEPARATOR = re.compile(r"[\s|]+") def _repo_from_github_url(token: str) -> str | None: @@ -96,19 +100,16 @@ def extract_explicit_repo(text: str, all_repos: list[str]) -> str | None: normalized_repos = {repo.lower(): repo for repo in all_repos} linked: set[str] = set() - for token in text.split(): - # Slack formats links as ; either side can carry the repo, so try both. - for candidate in (part.strip(_TOKEN_PUNCTUATION) for part in token.split("|")): - if not candidate: - continue + for part in _CANDIDATE_SEPARATOR.split(text): + candidate = part.strip(_TOKEN_PUNCTUATION) + if not candidate: + continue - if re.fullmatch(r"[\w.-]+/[\w.-]+", candidate): - match = normalized_repos.get(candidate.lower()) - if match: - return match + if _REPO_TOKEN.fullmatch(candidate) and (match := normalized_repos.get(candidate.lower())): + return match - from_url = _repo_from_github_url(candidate) - if from_url and (match := normalized_repos.get(from_url.lower())): - linked.add(match) + from_url = _repo_from_github_url(candidate) + if from_url and (match := normalized_repos.get(from_url.lower())): + linked.add(match) return next(iter(linked)) if len(linked) == 1 else None diff --git a/posthog/temporal/ai/slack_app/activities/classifiers.py b/posthog/temporal/ai/slack_app/activities/classifiers.py index 547b0c8fb003..eea931b21653 100644 --- a/posthog/temporal/ai/slack_app/activities/classifiers.py +++ b/posthog/temporal/ai/slack_app/activities/classifiers.py @@ -125,10 +125,9 @@ def classify_task_needs_repo( r"\bserializer\b", r"\bviewset\b", r"\bmigration\b", - # A failing test is code work, but it is usually named after the feature it covers, - # so the product terms above would answer no-repo before the model reads the sentence. - # Both match the whole thread, so keep them narrow enough that passing chatter cannot - # veto an analytics ask: a bare "ci" also means confidence interval here. + # A failing test is code work, but it is named after the feature it covers, so the + # product terms above would answer no-repo first. Keep these narrow: they match the + # whole thread, and a bare "ci" would also catch confidence intervals. r"\bflak(?:y|e|es|iness)\b", r"\bmerge queue\b", ) From bb1d2a1bfac55539a0e7f98af38d04d9fc5eb899 Mon Sep 17 00:00:00 2001 From: Adam Bowker Date: Wed, 12 Aug 2026 11:51:01 -0400 Subject: [PATCH 015/289] feat(tasks): publish compute rate card v1, effective 2026-08-24 Populates COMPUTE_RATE_CARDS with the first cloud compute rate card: $0.000075 per CPU core-second and $0.000008 per GiB-second, effective 2026-08-24T00:00:00Z. Sessions bill $0 before the effective timestamp, so merging ahead of the date is safe. Generated-By: PostHog Desktop Task-Id: 77a2de36-80bf-455c-98a4-c0979f710100 --- .../tasks/backend/logic/services/sandbox_pricing.py | 12 ++++++++++-- products/tasks/backend/tests/test_sandbox_pricing.py | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 products/tasks/backend/tests/test_sandbox_pricing.py diff --git a/products/tasks/backend/logic/services/sandbox_pricing.py b/products/tasks/backend/logic/services/sandbox_pricing.py index a243b16b0227..17b17d885e15 100644 --- a/products/tasks/backend/logic/services/sandbox_pricing.py +++ b/products/tasks/backend/logic/services/sandbox_pricing.py @@ -1,6 +1,6 @@ from collections.abc import Sequence from dataclasses import dataclass -from datetime import datetime +from datetime import UTC, datetime from decimal import ROUND_UP, Decimal from django.utils import timezone @@ -49,7 +49,15 @@ def total_cost_usd(self) -> Decimal: return self.cpu_cost_usd + self.memory_cost_usd -COMPUTE_RATE_CARDS: tuple[ComputeRateCard, ...] = () +COMPUTE_RATE_CARDS: tuple[ComputeRateCard, ...] = ( + ComputeRateCard( + version="v1", + effective_at=datetime(2026, 8, 24, tzinfo=UTC), + expires_at=None, + cpu_core_second_usd=Decimal("0.000075"), + memory_gib_second_usd=Decimal("0.000008"), + ), +) @dataclass(frozen=True) diff --git a/products/tasks/backend/tests/test_sandbox_pricing.py b/products/tasks/backend/tests/test_sandbox_pricing.py new file mode 100644 index 000000000000..3a70414cc006 --- /dev/null +++ b/products/tasks/backend/tests/test_sandbox_pricing.py @@ -0,0 +1,5 @@ +from products.tasks.backend.logic.services.sandbox_pricing import COMPUTE_RATE_CARDS, validate_compute_rate_cards + + +def test_published_compute_rate_cards_are_valid() -> None: + assert validate_compute_rate_cards(COMPUTE_RATE_CARDS) == COMPUTE_RATE_CARDS From bc3f7b23f94b9a785206e28a2075c64e1c8cd9c6 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Wed, 12 Aug 2026 12:36:18 -0400 Subject: [PATCH 016/289] fix(persons): claim stranded person rows on create instead of duplicating them Person UUIDs are deterministic (uuidv5 of team_id:distinct_id), and production has no unique index on posthog_person (team_id, uuid). On teams whose posthog_persondistinctid rows were removed outside the write path, a returning user's create therefore minted a second person row with an identical (team_id, uuid), silently. For teams on the new PERSON_CREATE_CLAIM_TEAM_ALLOWLIST, person creation now claims an existing live row that holds the event's (team_id, uuid) and has no live distinct-ID mapping: such a row is unreachable by the product, and the deterministic uuid means it was originally created for this same distinct ID. The claim resets the row from the current event, bumps its version, and attaches the new mapping at version 0, so ClickHouse converges with no override row. Teams off the allowlist run the existing query untouched; the probe is not free and only teams with stranded rows benefit. Concurrency safety rides on the unique (team_id, distinct_id) mapping index, not on any posthog_person index: a concurrent creator for the same uuid carries the same primary distinct ID, collides there, and the whole single statement rolls back into the existing CreationConflict retry path. Deliberately a separate allowlist from PERSON_MERGE_TOMBSTONE_TEAM_ALLOWLIST: the tombstone query's ON CONFLICT (team_id, uuid) needs a unique index for arbiter inference and would hard-fail every create in environments that lack it. Co-Authored-By: Claude Fable 5 --- nodejs/src/common/persons/metrics.ts | 9 + ...s-person-repository-stranded-claim.test.ts | 316 ++++++++++++++++++ .../postgres-person-repository.ts | 272 +++++++++++++++ nodejs/src/ingestion/config.ts | 7 + nodejs/src/servers/ingestion-api-server.ts | 1 + .../src/servers/ingestion-general-server.ts | 1 + 6 files changed, 606 insertions(+) create mode 100644 nodejs/src/common/persons/repositories/postgres-person-repository-stranded-claim.test.ts diff --git a/nodejs/src/common/persons/metrics.ts b/nodejs/src/common/persons/metrics.ts index f6d4b11aa622..6eb9bacaad36 100644 --- a/nodejs/src/common/persons/metrics.ts +++ b/nodejs/src/common/persons/metrics.ts @@ -184,6 +184,15 @@ export const personProfileBatchIgnoredPropertiesCounter = new Counter({ labelNames: ['property'], }) +export const personCreateStrandedClaimCounter = new Counter({ + name: 'person_create_stranded_claim_total', + help: 'Person creations routed through the stranded-row claim statement, by outcome', + // claimed: adopted an unreachable row holding this (team_id, uuid) + // inserted: no row held the uuid, fresh insert + // inserted_duplicate: a reachable person already held the uuid, so the insert created a duplicate key + labelNames: ['outcome'], +}) + export const personJsonFieldSizeHistogram = new Histogram({ name: 'person_json_field_size_bytes', help: 'Approximate size in bytes of serialized JSON fields (using string length as proxy for performance)', diff --git a/nodejs/src/common/persons/repositories/postgres-person-repository-stranded-claim.test.ts b/nodejs/src/common/persons/repositories/postgres-person-repository-stranded-claim.test.ts new file mode 100644 index 000000000000..dc5b9268dfdb --- /dev/null +++ b/nodejs/src/common/persons/repositories/postgres-person-repository-stranded-claim.test.ts @@ -0,0 +1,316 @@ +import { DateTime } from 'luxon' + +import { PERSONS_OUTPUT, PERSON_DISTINCT_IDS_OUTPUT } from '~/common/outputs/persons' +import { personCreateStrandedClaimCounter } from '~/common/persons/metrics' +import { closeHub, createHub } from '~/common/utils/db/hub' +import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres' +import { parseJSON } from '~/common/utils/json-parse' +import { UUIDT } from '~/common/utils/utils' +import { resetTestDatabase } from '~/tests/helpers/sql' +import { Hub, Team } from '~/types' + +import { PostgresPersonRepository } from './postgres-person-repository' +import { TEST_TIMESTAMP, fetchDistinctIdValues, getFirstTeam } from './test-helpers' + +jest.mock('~/common/utils/logger') + +// Production does NOT have the unique (team_id, uuid) index the tracked schema declares +// (posthog_person_new_uuid_idx is non-unique in both prod regions), which is what lets +// duplicate persons exist there at all. The tests below recreate that reality; with the +// tracked schema's unique index in place, the duplicate scenarios cannot even be seeded +// and every claim test would pass vacuously. +async function makeUuidIndexNonUnique(postgres: PostgresRouter): Promise { + const { rows } = await postgres.query( + PostgresUse.PERSONS_WRITE, + `SELECT i.indisunique FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid + WHERE c.relname = 'posthog_person_new_uuid_idx'`, + [], + 'checkUuidIndex' + ) + if (rows.length > 0 && !rows[0].indisunique) { + return + } + await postgres.query(PostgresUse.PERSONS_WRITE, `DROP INDEX IF EXISTS posthog_person_new_uuid_idx`, [], 'dropIdx') + await postgres.query( + PostgresUse.PERSONS_WRITE, + `CREATE INDEX posthog_person_new_uuid_idx ON posthog_person (team_id, uuid)`, + [], + 'createIdx' + ) +} + +async function restoreUniqueUuidIndex(postgres: PostgresRouter): Promise { + // The last test leaves duplicate (team_id, uuid) rows behind; clear them or the + // unique index cannot build. + await postgres.query(PostgresUse.PERSONS_WRITE, `DELETE FROM posthog_persondistinctid`, [], 'clearPdi') + await postgres.query(PostgresUse.PERSONS_WRITE, `DELETE FROM posthog_person`, [], 'clearPersons') + await postgres.query(PostgresUse.PERSONS_WRITE, `DROP INDEX IF EXISTS posthog_person_new_uuid_idx`, [], 'dropIdx') + await postgres.query( + PostgresUse.PERSONS_WRITE, + `CREATE UNIQUE INDEX posthog_person_new_uuid_idx ON posthog_person (team_id, uuid)`, + [], + 'createIdx' + ) +} + +describe('PostgresPersonRepository stranded-row claim', () => { + let hub: Hub + let postgres: PostgresRouter + let repository: PostgresPersonRepository + let team: Team + + beforeEach(async () => { + hub = await createHub() + await resetTestDatabase() + postgres = hub.postgres + await makeUuidIndexNonUnique(postgres) + repository = new PostgresPersonRepository(postgres, { + calculatePropertiesSize: 0, + personCreateClaimTeamAllowlist: '*', + }) + team = await getFirstTeam(postgres) + personCreateStrandedClaimCounter.reset() + }) + + afterEach(async () => { + await closeHub(hub) + jest.clearAllMocks() + }) + + afterAll(async () => { + // Other suites share this worker's database and expect the tracked schema. + const cleanupHub = await createHub() + await restoreUniqueUuidIndex(cleanupHub.postgres) + await closeHub(cleanupHub) + }) + + // A person row with no distinct-ID mapping, the way rows stranded by out-of-band + // mapping deletion look in production. + async function seedStrandedPerson( + teamId: number, + uuid: string, + overrides: { version?: number; properties?: Record; isIdentified?: boolean } = {} + ): Promise { + const { rows } = await postgres.query( + PostgresUse.PERSONS_WRITE, + `INSERT INTO posthog_person ( + created_at, properties, properties_last_updated_at, properties_last_operation, + team_id, is_user_id, is_identified, uuid, version + ) VALUES ($1, $2, '{}', '{}', $3, NULL, $4, $5, $6) RETURNING id`, + [ + DateTime.fromISO('2023-06-01T00:00:00.000Z').toISO(), + JSON.stringify(overrides.properties ?? { stale: 'value' }), + teamId, + overrides.isIdentified ?? false, + uuid, + overrides.version ?? 3, + ], + 'seedStrandedPerson' + ) + return Number(rows[0].id) + } + + async function addMapping(teamId: number, personId: number, distinctId: string, isDeleted = false): Promise { + await postgres.query( + PostgresUse.PERSONS_WRITE, + `INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id, version, is_deleted) + VALUES ($1, $2, $3, 0, $4)`, + [distinctId, personId, teamId, isDeleted], + 'seedMapping' + ) + } + + async function fetchPersonRows(teamId: number, uuid: string): Promise<{ id: number; version: number }[]> { + const { rows } = await postgres.query( + PostgresUse.PERSONS_WRITE, + `SELECT id, version FROM posthog_person WHERE team_id = $1 AND uuid = $2 ORDER BY id`, + [teamId, uuid], + 'fetchPersonRows' + ) + return rows.map((r: any) => ({ id: Number(r.id), version: Number(r.version) })) + } + + async function createPerson( + uuid: string, + distinctId: string, + properties: Record = {} + ): ReturnType { + return await repository.createPerson(TEST_TIMESTAMP, properties, {}, {}, team.id, null, true, uuid, { + distinctId, + }) + } + + async function getCounterValue(outcome: string): Promise { + const metric = await personCreateStrandedClaimCounter.get() + return metric.values.find((v) => v.labels.outcome === outcome)?.value ?? 0 + } + + it('the fixture allows duplicate (team_id, uuid) rows, like production', async () => { + const uuid = new UUIDT().toString() + await seedStrandedPerson(team.id, uuid) + // With the tracked schema's unique index this second insert would throw, and + // every scenario below would be untestable. + await expect(seedStrandedPerson(team.id, uuid)).resolves.toEqual(expect.any(Number)) + }) + + it('claims the stranded row: same row id, reset properties, bumped version, new mapping', async () => { + const uuid = new UUIDT().toString() + const strandedId = await seedStrandedPerson(team.id, uuid, { version: 7, properties: { stale: 'yes' } }) + + const result = await createPerson(uuid, 'returning-user', { fresh: 'yes' }) + + expect(result.success).toBe(true) + if (!result.success) { + return + } + expect(result.created).toBe(true) + expect(Number(result.person.id)).toBe(strandedId) + expect(result.person.uuid).toBe(uuid) + expect(result.person.properties).toEqual({ fresh: 'yes' }) + expect(result.person.version).toBe(8) + expect(result.person.is_identified).toBe(true) + + // Exactly one row holds the uuid, and the mapping points at it. + expect(await fetchPersonRows(team.id, uuid)).toEqual([{ id: strandedId, version: 8 }]) + expect(await fetchDistinctIdValues(postgres, result.person)).toEqual(['returning-user']) + expect(await getCounterValue('claimed')).toBe(1) + }) + + it('emits the person update and the distinct-id mapping to Kafka on a claim', async () => { + const uuid = new UUIDT().toString() + await seedStrandedPerson(team.id, uuid) + + const result = await createPerson(uuid, 'returning-user') + expect(result.success).toBe(true) + if (!result.success) { + return + } + + const personMessages = result.messages.filter((m) => m.output === PERSONS_OUTPUT) + const didMessages = result.messages.filter((m) => m.output === PERSON_DISTINCT_IDS_OUTPUT) + expect(personMessages).toHaveLength(1) + expect(didMessages).toHaveLength(1) + + const didPayload = parseJSON(didMessages[0].value!.toString()) + expect(didPayload).toEqual({ + person_id: uuid, + team_id: team.id, + distinct_id: 'returning-user', + // Version 0 keeps the mapping out of the ClickHouse overrides view: events + // stamped with this deterministic uuid already point at the right person. + version: 0, + is_deleted: 0, + }) + }) + + it('claims exactly one row (the oldest) when several stranded rows share the uuid', async () => { + const uuid = new UUIDT().toString() + const olderId = await seedStrandedPerson(team.id, uuid, { version: 1 }) + const newerId = await seedStrandedPerson(team.id, uuid, { version: 5 }) + + const result = await createPerson(uuid, 'returning-user') + expect(result.success).toBe(true) + if (!result.success) { + return + } + + expect(Number(result.person.id)).toBe(olderId) + const rows = await fetchPersonRows(team.id, uuid) + // The newer stranded row is untouched; repair tooling owns it. + expect(rows).toEqual([ + { id: olderId, version: 2 }, + { id: newerId, version: 5 }, + ]) + expect(await getCounterValue('claimed')).toBe(1) + }) + + it('never claims a row the product can still reach', async () => { + const uuid = new UUIDT().toString() + const reachableId = await seedStrandedPerson(team.id, uuid, { version: 4 }) + await addMapping(team.id, reachableId, 'other-distinct-id') + + const result = await createPerson(uuid, 'second-distinct-id') + expect(result.success).toBe(true) + if (!result.success) { + return + } + + // Pre-existing behavior for this (bounded, known) case: a duplicate row is + // created rather than ingestion failing. The metric is the alarm. + expect(Number(result.person.id)).not.toBe(reachableId) + const rows = await fetchPersonRows(team.id, uuid) + expect(rows).toHaveLength(2) + expect(rows[0]).toEqual({ id: reachableId, version: 4 }) + expect(await getCounterValue('inserted_duplicate')).toBe(1) + expect(await getCounterValue('claimed')).toBe(0) + }) + + it('treats a row whose only mapping is soft-deleted as claimable', async () => { + const uuid = new UUIDT().toString() + const strandedId = await seedStrandedPerson(team.id, uuid) + await addMapping(team.id, strandedId, 'dead-distinct-id', true) + + const result = await createPerson(uuid, 'live-distinct-id') + expect(result.success).toBe(true) + if (!result.success) { + return + } + + expect(Number(result.person.id)).toBe(strandedId) + expect(await getCounterValue('claimed')).toBe(1) + }) + + it('inserts normally when nothing holds the uuid', async () => { + const uuid = new UUIDT().toString() + const result = await createPerson(uuid, 'brand-new-user', { a: 1 }) + + expect(result.success).toBe(true) + if (!result.success) { + return + } + expect(result.created).toBe(true) + expect(result.person.uuid).toBe(uuid) + expect(result.person.version).toBe(0) + expect(await fetchDistinctIdValues(postgres, result.person)).toEqual(['brand-new-user']) + expect(await getCounterValue('inserted')).toBe(1) + expect(await getCounterValue('claimed')).toBe(0) + }) + + it('returns CreationConflict when the distinct ID is already mapped', async () => { + const uuid = new UUIDT().toString() + const first = await createPerson(uuid, 'contested-distinct-id') + expect(first.success).toBe(true) + + // Same repository-level contract as the legacy path: the caller resolves the + // conflict by re-fetching the person by distinct ID. + const second = await createPerson(new UUIDT().toString(), 'contested-distinct-id') + expect(second.success).toBe(false) + if (second.success) { + return + } + expect(second.error).toBe('CreationConflict') + }) + + it('does not claim for teams outside the allowlist', async () => { + const legacyRepository = new PostgresPersonRepository(postgres, { + calculatePropertiesSize: 0, + // personCreateClaimTeamAllowlist deliberately unset + }) + const uuid = new UUIDT().toString() + const strandedId = await seedStrandedPerson(team.id, uuid) + + const result = await legacyRepository.createPerson(TEST_TIMESTAMP, {}, {}, {}, team.id, null, true, uuid, { + distinctId: 'returning-user', + }) + expect(result.success).toBe(true) + if (!result.success) { + return + } + + // Legacy behavior preserved: a duplicate row, the stranded one untouched. + expect(Number(result.person.id)).not.toBe(strandedId) + expect(await fetchPersonRows(team.id, uuid)).toHaveLength(2) + expect(await getCounterValue('claimed')).toBe(0) + expect(await getCounterValue('inserted')).toBe(0) + }) +}) diff --git a/nodejs/src/common/persons/repositories/postgres-person-repository.ts b/nodejs/src/common/persons/repositories/postgres-person-repository.ts index 3260a8d6e252..256ccc423ccf 100644 --- a/nodejs/src/common/persons/repositories/postgres-person-repository.ts +++ b/nodejs/src/common/persons/repositories/postgres-person-repository.ts @@ -5,6 +5,7 @@ import { buildIntegerMatcher } from '~/common/config/config' import { PERSON_DISTINCT_IDS_OUTPUT } from '~/common/outputs/persons' import { oversizedPersonPropertiesTrimmedCounter, + personCreateStrandedClaimCounter, personJsonFieldSizeHistogram, personPropertiesSizeViolationCounter, } from '~/common/persons/metrics' @@ -81,6 +82,17 @@ export interface PostgresPersonRepositoryOptions { personPropertiesTrimTargetBytes: number /** Teams whose merge deletes tombstone the person row instead of hard-deleting it ('*' for all) */ personMergeTombstoneTeamAllowlist: string + /** + * Teams whose person creation claims an existing unreachable row holding the same + * (team_id, uuid) instead of inserting a duplicate. Person UUIDs are deterministic + * (uuidv5 of team_id:distinct_id), so on teams where posthog_persondistinctid rows + * were destroyed outside the write path, a returning user's create would otherwise + * mint a second row with an identical (team_id, uuid). Scoped to affected teams + * because the claim probe adds an index lookup to the hottest write path. + * NOT the tombstone allowlist: that one routes to a query whose ON CONFLICT + * (team_id, uuid) arbiter requires a unique index production does not have yet. + */ + personCreateClaimTeamAllowlist: string } const DEFAULT_OPTIONS: PostgresPersonRepositoryOptions = { @@ -88,6 +100,7 @@ const DEFAULT_OPTIONS: PostgresPersonRepositoryOptions = { personPropertiesDbConstraintLimitBytes: DEFAULT_PERSON_PROPERTIES_DB_CONSTRAINT_LIMIT_BYTES, personPropertiesTrimTargetBytes: DEFAULT_PERSON_PROPERTIES_TRIM_TARGET_BYTES, personMergeTombstoneTeamAllowlist: '', + personCreateClaimTeamAllowlist: '', } export class PostgresPersonRepository @@ -95,6 +108,7 @@ export class PostgresPersonRepository { private options: PostgresPersonRepositoryOptions private isTombstoneTeam: ValueMatcher + private isClaimTeam: ValueMatcher constructor( private postgres: PostgresRouter, @@ -102,6 +116,7 @@ export class PostgresPersonRepository ) { this.options = { ...DEFAULT_OPTIONS, ...options } this.isTombstoneTeam = buildIntegerMatcher(this.options.personMergeTombstoneTeamAllowlist, true) + this.isClaimTeam = buildIntegerMatcher(this.options.personCreateClaimTeamAllowlist, true) } private async handleOversizedPersonProperties( @@ -533,6 +548,21 @@ export class PostgresPersonRepository // Teams outside the tombstone rollout run the query shipped on master, // untouched: clearing the allowlist is a full rollback to it. if (!this.isTombstoneTeam(teamId)) { + if (this.isClaimTeam(teamId)) { + return await this.createPersonWithStrandedClaim( + createdAt, + properties, + propertiesLastUpdatedAt, + propertiesLastOperation, + teamId, + isUserId, + isIdentified, + uuid, + primaryDistinctId, + extraDistinctIds, + tx + ) + } return await this.createPersonLegacy( createdAt, properties, @@ -773,6 +803,248 @@ export class PostgresPersonRepository } } + /** + * createPersonLegacy plus one behavior change: when a live posthog_person row already + * holds this (team_id, uuid) and no live distinct-ID mapping points at it, that row is + * unreachable by the product (persons resolve only via distinct_id -> posthog_persondistinctid + * -> person_id), so it is claimed - reset from this event and given the new mapping - + * instead of a second row being inserted with an identical (team_id, uuid). + * + * The uuid is deterministic (uuidv5 of `${teamId}:${primaryDistinctId}`), so a claimed row + * was originally created for this same distinct ID; the claim reunites a person with its + * own row. The mapping keeps version 0, exactly like a fresh insert: the ClickHouse + * overrides view only consumes versions > 0, and events already stamped with this uuid + * point at the right person either way. + * + * Concurrency safety does not depend on any posthog_person index: a concurrent creator + * for the same uuid necessarily carries the same primary distinct ID, so its mapping + * insert collides on the unique (team_id, distinct_id) index and rolls this whole + * single statement back, surfacing as CreationConflict just like the legacy path. + */ + private async createPersonWithStrandedClaim( + createdAt: DateTime, + properties: Properties, + propertiesLastUpdatedAt: PropertiesLastUpdatedAt, + propertiesLastOperation: PropertiesLastOperation, + teamId: number, + isUserId: number | null, + isIdentified: boolean, + uuid: string, + primaryDistinctId: { distinctId: string; version?: number }, + extraDistinctIds: { distinctId: string; version?: number }[] = [], + tx?: TransactionClient + ): Promise { + const distinctIds = [primaryDistinctId, ...extraDistinctIds] + for (const distinctId of distinctIds) { + distinctId.version ||= 0 + } + + // Fresh inserts start at version 0; a claim continues the claimed row's counter so + // its ClickHouse row (same uuid) is overwritten rather than outranked. + const personVersion = 0 + + try { + const sanitizedProperties = sanitizeJsonbValue(properties) + const sanitizedPropertiesLastUpdatedAt = sanitizeJsonbValue(propertiesLastUpdatedAt) + const sanitizedPropertiesLastOperation = sanitizeJsonbValue(propertiesLastOperation) + + if (typeof sanitizedProperties === 'string') { + personJsonFieldSizeHistogram + .labels({ operation: 'createPerson', field: 'properties' }) + .observe(sanitizedProperties.length) + } + if (typeof sanitizedPropertiesLastUpdatedAt === 'string') { + personJsonFieldSizeHistogram + .labels({ operation: 'createPerson', field: 'properties_last_updated_at' }) + .observe(sanitizedPropertiesLastUpdatedAt.length) + } + if (typeof sanitizedPropertiesLastOperation === 'string') { + personJsonFieldSizeHistogram + .labels({ operation: 'createPerson', field: 'properties_last_operation' }) + .observe(sanitizedPropertiesLastOperation.length) + } + + // For new persons, set last_seen_at to the hour-rounded createdAt + const lastSeenAt = createdAt.startOf('hour') + + // holders is one probe on the (team_id, uuid) index; the reachability check is one + // probe per holder on the (team_id, person_id) index. Duplicate groups can hold + // several unreachable rows, so claimable takes exactly one (the oldest); repair + // tooling resolves the rest. The claim resets properties from this event rather + // than reviving the stranded row's - deliberate, matching the tombstone revival + // path, so data a deletion may have targeted is not resurrected. + // The mapping insert has no ON CONFLICT: a collision must abort the whole + // statement, same as the legacy path. + const query = ` + WITH holders AS ( + SELECT p.id, + EXISTS ( + SELECT 1 FROM posthog_persondistinctid d + WHERE d.team_id = $5 AND d.person_id = p.id AND d.is_deleted = false + ) AS reachable + FROM posthog_person p + WHERE p.team_id = $5 AND p.uuid = $8 AND p.is_deleted = false + ), + claimable AS ( + SELECT p.id FROM posthog_person p + WHERE p.team_id = $5 + AND p.id IN (SELECT h.id FROM holders h WHERE NOT h.reachable) + ORDER BY p.id + LIMIT 1 + FOR UPDATE + ), + claimed AS ( + UPDATE posthog_person p SET + created_at = $1, + properties = $2, + properties_last_updated_at = $3, + properties_last_operation = $4, + is_user_id = $6, + is_identified = $7, + version = COALESCE(p.version, 0)::numeric + 1, + last_seen_at = $10 + FROM claimable c + WHERE p.team_id = $5 AND p.id = c.id + RETURNING ${PERSON_COLUMNS_PREFIXED} + ), + inserted AS ( + INSERT INTO posthog_person ( + created_at, properties, properties_last_updated_at, properties_last_operation, + team_id, is_user_id, is_identified, uuid, version, last_seen_at + ) + SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 + WHERE NOT EXISTS (SELECT 1 FROM claimable) + RETURNING ${PERSON_COLUMNS} + ), + person AS ( + SELECT *, true AS was_claimed FROM claimed + UNION ALL + SELECT *, false AS was_claimed FROM inserted + ), + inserted_distinct_ids AS ( + -- NOTE: Keep this in sync with the posthog_persondistinctid INSERT in addDistinctId + INSERT INTO posthog_persondistinctid (distinct_id, person_id, team_id, version) + SELECT d.distinct_id, p.id, $5, d.version + FROM person p + CROSS JOIN unnest($11::text[], $12::bigint[]) AS d(distinct_id, version) + RETURNING id, distinct_id, version + ) + SELECT + p.*, + (SELECT count(*)::int FROM holders h WHERE h.reachable) AS reachable_holder_count, + ( + SELECT COALESCE(jsonb_agg(jsonb_build_object('id', d.id::text, 'distinct_id', d.distinct_id, 'version', d.version)), '[]'::jsonb) + FROM inserted_distinct_ids d + ) AS distinct_id_rows + FROM person p;` + + const { rows } = await this.postgres.query< + RawPerson & { + was_claimed: boolean + reachable_holder_count: number + distinct_id_rows: { id: string; distinct_id: string; version: number }[] + } + >( + tx ?? PostgresUse.PERSONS_WRITE, + query, + [ + createdAt.toISO(), + sanitizedProperties, + sanitizedPropertiesLastUpdatedAt, + sanitizedPropertiesLastOperation, + teamId, + isUserId, + isIdentified, + uuid, + personVersion, + lastSeenAt.toISO(), + distinctIds.map(({ distinctId }) => distinctId), + distinctIds.map(({ version }) => version), + ], + 'insertPersonWithStrandedClaim', + 'warn' + ) + + const { + was_claimed: wasClaimed, + reachable_holder_count: reachableHolderCount, + distinct_id_rows: distinctIdRows, + ...personRow + } = rows[0] + const person = this.toPerson(personRow) + + if (wasClaimed) { + personCreateStrandedClaimCounter.inc({ outcome: 'claimed' }) + } else if (reachableHolderCount > 0) { + // A reachable person already holds this uuid via a different distinct ID, so + // this insert created a duplicate (team_id, uuid) - the pre-existing behavior. + // Loud on purpose: these rows block the unique index build. + personCreateStrandedClaimCounter.inc({ outcome: 'inserted_duplicate' }) + logger.warn('Created person duplicates a reachable (team_id, uuid)', { + team_id: teamId, + person_uuid: uuid, + reachable_holder_count: reachableHolderCount, + }) + } else { + personCreateStrandedClaimCounter.inc({ outcome: 'inserted' }) + } + + const kafkaMessages: PersonMessage[] = [generateKafkaPersonUpdateMessage(person)] + + for (const row of distinctIdRows) { + kafkaMessages.push({ + output: PERSON_DISTINCT_IDS_OUTPUT, + value: Buffer.from( + JSON.stringify({ + person_id: person.uuid, + team_id: teamId, + distinct_id: row.distinct_id, + version: Number(row.version), + is_deleted: 0, + }) + ), + }) + } + + return { + success: true, + person, + messages: kafkaMessages, + created: true, + } + } catch (error) { + // Same conflict contract as the legacy path: a unique violation means a + // concurrent creator won the mapping, and the caller re-fetches by distinct ID. + if (error instanceof Error && error.message.includes('unique constraint')) { + return { + success: false, + error: 'CreationConflict', + distinctIds: distinctIds.map((d) => d.distinctId), + } + } + + if (this.isPropertiesSizeConstraintViolation(error)) { + personPropertiesSizeViolationCounter.inc({ + violation_type: 'create_person_size_violation', + }) + + logger.warn('Rejecting person properties create/update, exceeds size limit', { + team_id: teamId, + person_id: undefined, + violation_type: 'create_person_size_violation', + }) + + throw new PersonPropertiesSizeViolationError( + `Person properties create would exceed size limit`, + teamId, + undefined + ) + } + + throw error + } + } + // Master's createPerson, kept byte-for-byte for teams outside the tombstone // rollout. Remove together with the allowlist once tombstone mode is the default. private async createPersonLegacy( diff --git a/nodejs/src/ingestion/config.ts b/nodejs/src/ingestion/config.ts index 7ab0f1b63177..deb1db765bc2 100644 --- a/nodejs/src/ingestion/config.ts +++ b/nodejs/src/ingestion/config.ts @@ -177,6 +177,12 @@ export type IngestionConsumerConfig = { // recreated person revives above its own tombstone. Comma-separated team IDs, or '*' for all // teams; empty means no teams. PERSON_MERGE_TOMBSTONE_TEAM_ALLOWLIST: string + // Teams whose person creation claims an existing unreachable posthog_person row holding + // the same deterministic (team_id, uuid) instead of inserting a duplicate row. Scope to + // teams whose distinct-ID mappings were destroyed outside the write path (stranded rows); + // for everyone else the probe is wasted load on the hottest write statement. + // Comma-separated team IDs, or '*' for all teams; empty means no teams. + PERSON_CREATE_CLAIM_TEAM_ALLOWLIST: string // Group batch writing config GROUP_BATCH_WRITING_USE_BATCH_UPDATES: boolean @@ -323,6 +329,7 @@ export function getDefaultIngestionConsumerConfig(): IngestionConsumerConfig { PERSON_MERGE_FOLD_ENABLED: false, PERSON_MERGE_FOLD_TEAM_ALLOWLIST: '*', PERSON_MERGE_TOMBSTONE_TEAM_ALLOWLIST: '', + PERSON_CREATE_CLAIM_TEAM_ALLOWLIST: '', // Group batch writing config GROUP_BATCH_WRITING_USE_BATCH_UPDATES: true, diff --git a/nodejs/src/servers/ingestion-api-server.ts b/nodejs/src/servers/ingestion-api-server.ts index 9d83eec26ace..33ce1b12f407 100644 --- a/nodejs/src/servers/ingestion-api-server.ts +++ b/nodejs/src/servers/ingestion-api-server.ts @@ -248,6 +248,7 @@ export class IngestionApiServer implements NodeServer { const postgresPersonRepository = new PostgresPersonRepository(this.postgres, { calculatePropertiesSize: this.config.PERSON_UPDATE_CALCULATE_PROPERTIES_SIZE, personMergeTombstoneTeamAllowlist: this.config.PERSON_MERGE_TOMBSTONE_TEAM_ALLOWLIST, + personCreateClaimTeamAllowlist: this.config.PERSON_CREATE_CLAIM_TEAM_ALLOWLIST, }) const personRepository = buildPersonRepository( personhogClient, diff --git a/nodejs/src/servers/ingestion-general-server.ts b/nodejs/src/servers/ingestion-general-server.ts index ddfebdf596e2..1c7ac6980302 100644 --- a/nodejs/src/servers/ingestion-general-server.ts +++ b/nodejs/src/servers/ingestion-general-server.ts @@ -221,6 +221,7 @@ export class IngestionGeneralServer implements NodeServer { const postgresPersonRepository = new PostgresPersonRepository(this.postgres, { calculatePropertiesSize: this.config.PERSON_UPDATE_CALCULATE_PROPERTIES_SIZE, personMergeTombstoneTeamAllowlist: this.config.PERSON_MERGE_TOMBSTONE_TEAM_ALLOWLIST, + personCreateClaimTeamAllowlist: this.config.PERSON_CREATE_CLAIM_TEAM_ALLOWLIST, }) const personRepository = buildPersonRepository( personhogClient, From c3a0784056326466763435924e8174aa87167418 Mon Sep 17 00:00:00 2001 From: Adam Bowker Date: Wed, 12 Aug 2026 12:46:41 -0400 Subject: [PATCH 017/289] fix(tasks): move rate card v1 effective to 2026-08-25 12:00 UTC 2026-08-25 12:00 UTC is 8am ET on launch day. Generated-By: PostHog Desktop Task-Id: 77a2de36-80bf-455c-98a4-c0979f710100 --- products/tasks/backend/logic/services/sandbox_pricing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/tasks/backend/logic/services/sandbox_pricing.py b/products/tasks/backend/logic/services/sandbox_pricing.py index 17b17d885e15..380c1d2fa731 100644 --- a/products/tasks/backend/logic/services/sandbox_pricing.py +++ b/products/tasks/backend/logic/services/sandbox_pricing.py @@ -52,7 +52,7 @@ def total_cost_usd(self) -> Decimal: COMPUTE_RATE_CARDS: tuple[ComputeRateCard, ...] = ( ComputeRateCard( version="v1", - effective_at=datetime(2026, 8, 24, tzinfo=UTC), + effective_at=datetime(2026, 8, 25, 12, tzinfo=UTC), expires_at=None, cpu_core_second_usd=Decimal("0.000075"), memory_gib_second_usd=Decimal("0.000008"), From 7f082003b41d85c059e21d14c7bb3b134c02fd53 Mon Sep 17 00:00:00 2001 From: Kyle Swank Date: Wed, 12 Aug 2026 15:04:00 -0400 Subject: [PATCH 018/289] fix(data-warehouse): keep BigQuery's temporary dataset visible after save A hyphenated source field is declared as "temporary-dataset" but persisted by dataclasses.asdict() as "temporary_dataset", so the settings form looked for a key the API never returned and fell back to the wizard default. Restore declared field names on read, covering both stored spellings, and deep-merge switch groups on update so toggling one no longer drops a required nested value. Generated-By: PostHog Desktop Task-Id: 0c6e6b1e-37a5-4df1-b195-808e8865c54b --- .../views/external_data_source.py | 122 +++++++++++++++++- .../sources/tests/test_generated_configs.py | 7 +- .../tests/api/test_external_data_source.py | 111 ++++++++++++++++ 3 files changed, 233 insertions(+), 7 deletions(-) diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source.py b/products/warehouse_sources/backend/presentation/views/external_data_source.py index 0eba098a33ad..704cad0f2cf5 100644 --- a/products/warehouse_sources/backend/presentation/views/external_data_source.py +++ b/products/warehouse_sources/backend/presentation/views/external_data_source.py @@ -45,6 +45,7 @@ from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.utils import action +from posthog.dataclasses import frozen from posthog.event_usage import EventSource, get_event_source, is_wizard_self_driving_program, report_user_action from posthog.exceptions_capture import capture_exception from posthog.models.integration import Integration @@ -277,6 +278,67 @@ def _add_name_variants(target: set[str], name: str) -> None: target.add(normalised) +@frozen +class DeclaredFieldNames: + """Declared field names that need special handling when reading or merging job_inputs. + + `hyphenated` are names the source declares with a hyphen. `dataclasses.asdict()` persists + the Python attribute name instead, so stored configs can hold either spelling. + `switch_groups` are switch-group container names, whose stored value is a nested dict. + """ + + hyphenated: set[str] + switch_groups: set[str] + + +def get_declared_field_names(fields: list[FieldType]) -> DeclaredFieldNames: + """Collect hyphenated and switch-group field names, flattened across all nesting levels.""" + hyphenated: set[str] = set() + switch_groups: set[str] = set() + + for field in fields: + if "-" in field.name: + hyphenated.add(field.name) + if isinstance(field, SourceFieldSwitchGroupConfig): + switch_groups.add(field.name) + nested = get_declared_field_names(field.fields) + hyphenated.update(nested.hyphenated) + switch_groups.update(nested.switch_groups) + elif isinstance(field, SourceFieldSelectConfig): + for option in field.options: + if option.fields: + nested = get_declared_field_names(option.fields) + hyphenated.update(nested.hyphenated) + switch_groups.update(nested.switch_groups) + + return DeclaredFieldNames(hyphenated=hyphenated, switch_groups=switch_groups) + + +def restore_declared_field_names(data: dict, hyphenated: set[str]) -> dict: + """Return a copy of data re-keyed to the names the source config declares. + + A hyphenated field round-trips through `dataclasses.asdict()`, which writes the Python + attribute name ("temporary_dataset") rather than the declared one ("temporary-dataset"). + Clients key off the declared name, so restore it. When both spellings are present the + declared one wins, matching how config parsing prefers the alias. + """ + if not hyphenated: + return data + + variants = {name.replace("-", "_"): name for name in hyphenated} + result: dict = {} + for key, value in data.items(): + declared = variants.get(key) + if declared is not None: + if declared in data: + continue + key = declared + if isinstance(value, dict): + value = restore_declared_field_names(value, hyphenated) + result[key] = value + return result + + def get_nonsensitive_and_sensitive_field_names(fields: list[FieldType]) -> tuple[set[str], set[str]]: """Classify source config field names as nonsensitive or sensitive. @@ -396,20 +458,29 @@ def _coerce(value: Any) -> str: _CREATION_ONLY_SECRET_FIELDS = frozenset({"connection_string"}) -def has_preserved_credentials(existing: dict[str, Any], incoming: dict[str, Any], sensitive_fields: set[str]) -> bool: +def has_preserved_credentials( + existing: dict[str, Any], + incoming: dict[str, Any], + sensitive_fields: set[str], + nested_containers: Iterable[str] = _NESTED_AUTH_CONTAINERS, +) -> bool: """True if any stored secret would be reused because the update didn't re-supply it. - Checks both top-level secret fields and the nested auth containers where sources like + Checks both top-level secret fields and the nested containers where sources like ServiceNow, Stripe and Snowflake keep their credentials. Used to force credential re-entry when the connection target changes, so a redirected host can't receive a preserved secret. A secret only counts as preserved when it would survive the merge: an absent container carries the whole existing block over, a same-selection container preserves any field the update omits, and a selection switch replaces the block wholesale. + + Switch groups merge the same way, so callers pass their names too. A switch group carries + no `selection`, which reads as unchanged and lands on the omitted-field check — the branch + that matches how the merge treats them. """ if any(existing.get(key) and not incoming.get(key) for key in sensitive_fields): return True - for container_key in _NESTED_AUTH_CONTAINERS: + for container_key in nested_containers: existing_container = existing.get(container_key) if not isinstance(existing_container, dict): continue @@ -930,7 +1001,9 @@ def to_representation(self, instance): if "require_tls" not in tunnel: tunnel["require_tls"] = {"enabled": True} - representation["job_inputs"] = strip_sensitive_from_dict(job_inputs, nonsensitive, sensitive) + stripped = strip_sensitive_from_dict(job_inputs, nonsensitive, sensitive) + declared = get_declared_field_names(source.get_source_config.fields) + representation["job_inputs"] = restore_declared_field_names(stripped, declared.hyphenated) return representation def get_last_run_at(self, instance: ExternalDataSource) -> str | None: @@ -1045,6 +1118,7 @@ def update(self, instance: ExternalDataSource, validated_data: Any) -> Any: source_type_model = ExternalDataSourceType(instance.source_type) source = SourceRegistry.get_source(source_type_model) sensitive_fields = get_sensitive_field_names(source.get_source_config.fields) + declared_field_names = get_declared_field_names(source.get_source_config.fields) discovered_schemas: list[SourceSchema] | None = None new_job_inputs = {**existing_job_inputs, **incoming_job_inputs} @@ -1106,7 +1180,10 @@ def update(self, instance: ExternalDataSource, validated_data: Any) -> Any: if connection_host_changed or ssh_tunnel_changed or job_inputs_host_added: gate_sensitive_fields = sensitive_fields - _CREATION_ONLY_SECRET_FIELDS preserved_credentials = has_preserved_credentials( - existing_job_inputs, incoming_job_inputs, gate_sensitive_fields + existing_job_inputs, + incoming_job_inputs, + gate_sensitive_fields, + nested_containers=(*_NESTED_AUTH_CONTAINERS, *declared_field_names.switch_groups), ) if preserved_credentials or preserved_row_backed_credentials: if ssh_tunnel_changed: @@ -1144,6 +1221,41 @@ def update(self, instance: ExternalDataSource, validated_data: Any) -> Any: merged_container[key] = existing_container[key] new_job_inputs[container_key] = merged_container + # Switch groups are nested containers too. The settings form submits only the fields the + # user touched and skips a disabled group's children, so a payload that just flips + # `enabled` would otherwise replace the whole stored group and drop a required nested + # value that validation then rejects. Switching a group off keeps its stored value — + # the user hasn't asked to forget it, and consumers gate on `enabled` before reading it. + for group_key in declared_field_names.switch_groups: + # A group declared with a hyphen can be stored under either spelling (see + # `restore_declared_field_names`), so look for both on each side. + group_key_variants = (group_key, group_key.replace("-", "_")) + incoming_key = next((key for key in group_key_variants if key in incoming_job_inputs), None) + if incoming_key is None: + continue + incoming_group = incoming_job_inputs[incoming_key] + if not isinstance(incoming_group, dict): + raise ValidationError({"job_inputs": {group_key: "Must be an object."}}) + existing_group = next( + ( + existing_job_inputs[key] + for key in group_key_variants + if isinstance(existing_job_inputs.get(key), dict) + ), + None, + ) + if existing_group is None: + continue + merged_group = {**existing_group, **incoming_group} + # No switch group declares a secret today, but keep the carry-over so one could. + for key in sensitive_fields: + if existing_group.get(key) and not incoming_group.get(key): + merged_group[key] = existing_group[key] + # Drop the other spelling so parsing can't see two competing groups. + for key in group_key_variants: + new_job_inputs.pop(key, None) + new_job_inputs[incoming_key] = merged_group + incoming_ssh_tunnel = incoming_job_inputs.get("ssh_tunnel") if existing_ssh_tunnel and incoming_ssh_tunnel is not None: ssh_tunnel_host_changed = "host" in incoming_ssh_tunnel and incoming_ssh_tunnel[ diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.py index 3c19aa7f4a20..234e05907dbe 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_generated_configs.py @@ -58,7 +58,10 @@ ) -def test_bigquery_config(): +# The form submits "temporary-dataset", but dataclasses.asdict() persists "temporary_dataset", +# so a source that has been saved once is read back under the second spelling. +@pytest.mark.parametrize("temporary_dataset_key", ["temporary-dataset", "temporary_dataset"]) +def test_bigquery_config(temporary_dataset_key: str): config = BigQuerySourceConfig.from_dict( { "key_file": { @@ -69,7 +72,7 @@ def test_bigquery_config(): "token_uri": "token_uri", }, "dataset_id": "dataset_id", - "temporary-dataset": {"enabled": False, "temporary_dataset_id": ""}, + temporary_dataset_key: {"enabled": False, "temporary_dataset_id": ""}, "dataset_project": {"enabled": False, "dataset_project_id": ""}, } ) diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py index f91cd24655c6..3f6286ded2ac 100644 --- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py +++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py @@ -57,9 +57,11 @@ from products.warehouse_sources.backend.presentation.views.external_data_schema import ExternalDataSchemaSerializer from products.warehouse_sources.backend.presentation.views.external_data_source import ( ExternalDataSourceViewSet, + get_declared_field_names, get_direct_connection_metadata, get_nonsensitive_and_sensitive_field_names, get_oauth_integration_kinds, + restore_declared_field_names, strip_sensitive_from_dict, ) from products.warehouse_sources.backend.temporal.data_imports.sources import SourceRegistry @@ -8965,6 +8967,40 @@ def test_strip_preserves_aliased_switch_group_from_to_dict(self): assert result["temporary_dataset"]["enabled"] is True assert result["temporary_dataset"]["temporary_dataset_id"] == "tmp-dataset" + def test_restore_prefers_the_declared_name_when_both_spellings_are_stored(self): + fields: list[FieldType] = [ + SourceFieldSwitchGroupConfig( + name="temporary-dataset", + label="Temporary dataset", + default=False, + fields=cast( + list[FieldType], + [ + SourceFieldInputConfig( + name="temporary_dataset_id", + label="Dataset ID", + placeholder="", + required=True, + type=SourceFieldInputConfigType.TEXT, + secret=False, + ), + ], + ), + ), + ] + declared = get_declared_field_names(fields) + + result = restore_declared_field_names( + { + "temporary-dataset": {"temporary_dataset_id": "declared"}, + "temporary_dataset": {"temporary_dataset_id": "persisted"}, + }, + declared.hyphenated, + ) + + assert result["temporary-dataset"]["temporary_dataset_id"] == "declared" + assert "temporary_dataset" not in result + def test_all_registered_sources_have_valid_classification(self): for source in SourceRegistry.get_all_sources().values(): config = source.get_source_config @@ -13210,3 +13246,78 @@ def test_patch_repo_change_reconciles_webhooks_and_prunes_mapping(self, _mock_va removed_webhook_row.refresh_from_db() assert removed_webhook_row.deleted is True or removed_webhook_row.should_sync is False + + +class TestBigQuerySwitchGroups(APIBaseTest): + def _create_bigquery_source(self, job_inputs: dict[str, Any]) -> ExternalDataSource: + return ExternalDataSource.objects.create( + team_id=self.team.pk, + source_id=str(uuid.uuid4()), + connection_id=str(uuid.uuid4()), + destination_id=str(uuid.uuid4()), + source_type="BigQuery", + created_by=self.user, + prefix="bq", + job_inputs={ + "key_file": { + "project_id": "project_id", + "private_key_id": "private_key_id", + "private_key": "private_key", + "client_email": "client_email", + "token_uri": "token_uri", + }, + "dataset_id": "my_dataset", + **job_inputs, + }, + ) + + # A source saved since migration 0807 stores "temporary_dataset"; one untouched since stores + # "temporary-dataset". The settings form only looks for the declared, hyphenated name. + @parameterized.expand([("temporary-dataset",), ("temporary_dataset",)]) + def test_temporary_dataset_reads_back_under_the_declared_name(self, stored_key: str) -> None: + source = self._create_bigquery_source({stored_key: {"enabled": True, "temporary_dataset_id": "tmp_dataset"}}) + + response = self.client.get(f"/api/environments/{self.team.pk}/external_data_sources/{source.pk}/") + assert response.status_code == status.HTTP_200_OK + + job_inputs = response.json()["job_inputs"] + assert job_inputs["temporary-dataset"]["temporary_dataset_id"] == "tmp_dataset" + assert job_inputs["temporary-dataset"]["enabled"] == "True" + # A leftover second spelling would win the settings form's merge and blank the field. + assert "temporary_dataset" not in job_inputs + + @parameterized.expand( + [ + ("temporary-dataset", "temporary_dataset", False, "temporary_dataset_id", "tmp_dataset"), + ("use_custom_region", "use_custom_region", False, "region", "us-east1"), + ("dataset_project", "dataset_project", True, "dataset_project_id", "other_project"), + ] + ) + def test_toggling_a_switch_group_keeps_its_stored_value( + self, group_key: str, stored_key: str, enabled: bool, nested_key: str, nested_value: str + ) -> None: + source = self._create_bigquery_source( + { + "temporary_dataset": {"enabled": True, "temporary_dataset_id": "tmp_dataset"}, + "use_custom_region": {"enabled": True, "region": "us-east1"}, + "dataset_project": {"enabled": False, "dataset_project_id": "other_project"}, + } + ) + + with patch( + "products.warehouse_sources.backend.temporal.data_imports.sources.bigquery.source.BigQuerySource.validate_credentials", + return_value=(True, None), + ): + response = self.client.patch( + f"/api/environments/{self.team.pk}/external_data_sources/{source.pk}/", + data={"job_inputs": {group_key: {"enabled": enabled}}}, + ) + + assert response.status_code == status.HTTP_200_OK, response.json() + + source.refresh_from_db() + config = BigQuerySourceConfig.from_dict(source.job_inputs) + group = getattr(config, stored_key) + assert group is not None + assert group.enabled is enabled + assert getattr(group, nested_key) == nested_value From 26e8e4f3f517b7939d4ca93aabdac742c37a47ba Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Thu, 13 Aug 2026 12:07:46 -0400 Subject: [PATCH 019/289] fix(persons): re-verify liveness under lock in the stranded-row claim The claimable CTE now rechecks is_deleted = false under the FOR UPDATE row lock, so a row tombstoned between snapshot and lock falls through to a fresh insert instead of being claimed without clearing its flag. Also drops a stray ::numeric cast from the version bump and adds coverage for tombstoned holders and multi-distinct-ID claims. Co-Authored-By: Claude Fable 5 --- ...s-person-repository-stranded-claim.test.ts | 70 ++++++++++++++++++- .../postgres-person-repository.ts | 7 +- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/nodejs/src/common/persons/repositories/postgres-person-repository-stranded-claim.test.ts b/nodejs/src/common/persons/repositories/postgres-person-repository-stranded-claim.test.ts index dc5b9268dfdb..44b675818a65 100644 --- a/nodejs/src/common/persons/repositories/postgres-person-repository-stranded-claim.test.ts +++ b/nodejs/src/common/persons/repositories/postgres-person-repository-stranded-claim.test.ts @@ -89,14 +89,19 @@ describe('PostgresPersonRepository stranded-row claim', () => { async function seedStrandedPerson( teamId: number, uuid: string, - overrides: { version?: number; properties?: Record; isIdentified?: boolean } = {} + overrides: { + version?: number + properties?: Record + isIdentified?: boolean + isDeleted?: boolean + } = {} ): Promise { const { rows } = await postgres.query( PostgresUse.PERSONS_WRITE, `INSERT INTO posthog_person ( created_at, properties, properties_last_updated_at, properties_last_operation, - team_id, is_user_id, is_identified, uuid, version - ) VALUES ($1, $2, '{}', '{}', $3, NULL, $4, $5, $6) RETURNING id`, + team_id, is_user_id, is_identified, uuid, version, is_deleted + ) VALUES ($1, $2, '{}', '{}', $3, NULL, $4, $5, $6, $7) RETURNING id`, [ DateTime.fromISO('2023-06-01T00:00:00.000Z').toISO(), JSON.stringify(overrides.properties ?? { stale: 'value' }), @@ -104,6 +109,7 @@ describe('PostgresPersonRepository stranded-row claim', () => { overrides.isIdentified ?? false, uuid, overrides.version ?? 3, + overrides.isDeleted ?? false, ], 'seedStrandedPerson' ) @@ -260,6 +266,64 @@ describe('PostgresPersonRepository stranded-row claim', () => { expect(await getCounterValue('claimed')).toBe(1) }) + it('never claims a tombstoned (is_deleted) row: falls through to a fresh insert', async () => { + const uuid = new UUIDT().toString() + const tombstonedId = await seedStrandedPerson(team.id, uuid, { version: 6, isDeleted: true }) + + const result = await createPerson(uuid, 'returning-user') + expect(result.success).toBe(true) + if (!result.success) { + return + } + + // The tombstoned row is not revived; the create behaves as if it were absent. + expect(Number(result.person.id)).not.toBe(tombstonedId) + expect(result.person.version).toBe(0) + expect(await fetchPersonRows(team.id, uuid)).toEqual([ + { id: tombstonedId, version: 6 }, + { id: Number(result.person.id), version: 0 }, + ]) + expect(await getCounterValue('claimed')).toBe(0) + expect(await getCounterValue('inserted')).toBe(1) + }) + + it('claims with extra distinct IDs: every mapping lands on the claimed row', async () => { + const uuid = new UUIDT().toString() + const strandedId = await seedStrandedPerson(team.id, uuid) + + const result = await repository.createPerson( + TEST_TIMESTAMP, + {}, + {}, + {}, + team.id, + null, + true, + uuid, + { distinctId: 'primary-id' }, + [{ distinctId: 'extra-id', version: 2 }] + ) + expect(result.success).toBe(true) + if (!result.success) { + return + } + + expect(Number(result.person.id)).toBe(strandedId) + expect((await fetchDistinctIdValues(postgres, result.person)).sort()).toEqual(['extra-id', 'primary-id']) + + const didPayloads = result.messages + .filter((m) => m.output === PERSON_DISTINCT_IDS_OUTPUT) + .map((m) => parseJSON(m.value!.toString())) + expect(didPayloads).toEqual( + expect.arrayContaining([ + expect.objectContaining({ distinct_id: 'primary-id', version: 0, person_id: uuid }), + expect.objectContaining({ distinct_id: 'extra-id', version: 2, person_id: uuid }), + ]) + ) + expect(didPayloads).toHaveLength(2) + expect(await getCounterValue('claimed')).toBe(1) + }) + it('inserts normally when nothing holds the uuid', async () => { const uuid = new UUIDT().toString() const result = await createPerson(uuid, 'brand-new-user', { a: 1 }) diff --git a/nodejs/src/common/persons/repositories/postgres-person-repository.ts b/nodejs/src/common/persons/repositories/postgres-person-repository.ts index 256ccc423ccf..94422dd3d974 100644 --- a/nodejs/src/common/persons/repositories/postgres-person-repository.ts +++ b/nodejs/src/common/persons/repositories/postgres-person-repository.ts @@ -886,8 +886,13 @@ export class PostgresPersonRepository WHERE p.team_id = $5 AND p.uuid = $8 AND p.is_deleted = false ), claimable AS ( + -- is_deleted is re-verified here (not only in holders) because under READ + -- COMMITTED, FOR UPDATE follows a concurrent update to the row's new version + -- and rechecks only this WHERE; without it, a row tombstoned between snapshot + -- and lock would be claimed without clearing its is_deleted flag. SELECT p.id FROM posthog_person p WHERE p.team_id = $5 + AND p.is_deleted = false AND p.id IN (SELECT h.id FROM holders h WHERE NOT h.reachable) ORDER BY p.id LIMIT 1 @@ -901,7 +906,7 @@ export class PostgresPersonRepository properties_last_operation = $4, is_user_id = $6, is_identified = $7, - version = COALESCE(p.version, 0)::numeric + 1, + version = COALESCE(p.version, 0) + 1, last_seen_at = $10 FROM claimable c WHERE p.team_id = $5 AND p.id = c.id From 07ac93109328281cb454f1e900d370d6b789f092 Mon Sep 17 00:00:00 2001 From: Matt Pua Date: Thu, 13 Aug 2026 13:00:25 -0400 Subject: [PATCH 020/289] fix(homepage): allow eight items in homepage grid Generated-By: PostHog Desktop Task-Id: d9ca1400-717b-4f10-b10f-afc3e90af4c2 --- frontend/src/scenes/max/components/CapabilityBadges.tsx | 8 +++----- .../scenes/project-homepage/ai-first/HomepageInput.tsx | 5 ++--- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/frontend/src/scenes/max/components/CapabilityBadges.tsx b/frontend/src/scenes/max/components/CapabilityBadges.tsx index b47512605257..48c24ee1f5c4 100644 --- a/frontend/src/scenes/max/components/CapabilityBadges.tsx +++ b/frontend/src/scenes/max/components/CapabilityBadges.tsx @@ -17,9 +17,8 @@ import { nextTypingDelayMs } from '../utils/typing' const COLORFUL_ICONS = 'group/colorful-product-icons colorful-product-icons-true' /** - * Height of the suggestion-cards / recents-grid swap area. Set once on the outer container by each - * surface; the cards and the homepage recents grid both fill it with `h-full`, so the two can never - * differ and swapping between them (or between capabilities) never shifts layout. + * Baseline height of the suggestion-cards / recents-grid swap area. Each surface can grow beyond it + * when its content needs more space. */ export const CAPABILITY_CARDS_HEIGHT_PX = 184 @@ -101,8 +100,7 @@ export interface CapabilitySuggestionsProps { } /** - * The 4 suggestion cards for a selected capability. Fills its parent's height (`h-full`) — the - * parent sets the fixed height (see `CAPABILITY_CARDS_HEIGHT_PX`), and the 4 cards split it evenly. + * The suggestion cards for a selected capability. Fills its parent's baseline height (`h-full`). */ export function CapabilitySuggestions({ capability, diff --git a/frontend/src/scenes/project-homepage/ai-first/HomepageInput.tsx b/frontend/src/scenes/project-homepage/ai-first/HomepageInput.tsx index 0b6e8e9c15fb..79e3c7e9855e 100644 --- a/frontend/src/scenes/project-homepage/ai-first/HomepageInput.tsx +++ b/frontend/src/scenes/project-homepage/ai-first/HomepageInput.tsx @@ -565,11 +565,10 @@ export function HomepageInput(): JSX.Element { selectedKey={selectedCapability} onSelect={setSelectedCapability} /> - {/* Single fixed-height swap area — the cards and the recents grid both fill - it (h-full), so switching never changes height. */} + {/* Keep the cards at their baseline height, but let the grid grow for longer lists. */}
{selectedCapabilityData ? ( Date: Thu, 13 Aug 2026 13:06:04 -0400 Subject: [PATCH 021/289] fix(homepage): preserve layout basis while expanding grid Generated-By: PostHog Desktop Task-Id: d9ca1400-717b-4f10-b10f-afc3e90af4c2 --- .../project-homepage/ai-first/HomepageInput.tsx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/frontend/src/scenes/project-homepage/ai-first/HomepageInput.tsx b/frontend/src/scenes/project-homepage/ai-first/HomepageInput.tsx index 79e3c7e9855e..d746ae2ea580 100644 --- a/frontend/src/scenes/project-homepage/ai-first/HomepageInput.tsx +++ b/frontend/src/scenes/project-homepage/ai-first/HomepageInput.tsx @@ -451,10 +451,9 @@ function IdleGrid(): JSX.Element { ref={gridRef} role="grid" data-attr="homepage-grid" - // Fills the fixed-height swap container (see HomepageInput) so the recents grid and the - // capability cards are always the same height. Only shown at @xl+ where columns sit in a row. + // Only shown at @xl+ where the homepage columns sit in a row. className={cn( - 'flex flex-col @xl/main-content:flex-row gap-8 @xl/main-content:gap-2 w-full px-3 outline-none h-full', + 'flex flex-col @xl/main-content:flex-row gap-8 @xl/main-content:gap-2 w-full px-3 outline-none', hasExtraMarginTop && 'mt-8' )} tabIndex={-1} @@ -565,10 +564,13 @@ export function HomepageInput(): JSX.Element { selectedKey={selectedCapability} onSelect={setSelectedCapability} /> - {/* Keep the cards at their baseline height, but let the grid grow for longer lists. */} + {/* Capability suggestions need a fixed basis; the idle grid uses its intrinsic height. */}
{selectedCapabilityData ? ( Date: Thu, 13 Aug 2026 18:04:46 -0600 Subject: [PATCH 022/289] feat(warehouse-sources): typed ingest for App Store Connect columns Report dates parse to dates, units and other counts to integers, monetary columns to floats, and JSON:API date-time attributes to UTC timestamps, all cast at row construction. Typing is column-name driven for the delimited report families and suffix driven for JSON:API, so it covers report variants added later. A value that does not parse becomes null with counted, aggregated warnings instead of failing the sync. Closes #80158 Generated-By: PostHog Desktop Task-Id: b53e8685-64ca-499d-9e72-7ff43c312bb7 --- .../app_store_connect/app_store_connect.py | 300 +++++++++++++++--- .../canonical_descriptions.py | 14 +- .../tests/test_app_store_connect.py | 225 +++++++++++-- 3 files changed, 459 insertions(+), 80 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/app_store_connect.py b/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/app_store_connect.py index 7fed0e8e0203..992e12fe2201 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/app_store_connect.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/app_store_connect.py @@ -2,11 +2,12 @@ import re import csv import gzip +import math import time import hashlib import tempfile import dataclasses -from collections.abc import Iterator +from collections.abc import Callable, Iterator from datetime import UTC, date, datetime, timedelta from typing import IO, Any, Optional from urllib.parse import urlsplit @@ -239,6 +240,196 @@ def _flatten_resource(resource: dict[str, Any]) -> dict[str, Any]: return row +class _ParseFailureCounter: + """Counts typed-column values that failed to parse and were stored as null. + + Null-on-unparseable is the deliberate failure policy for typed ingest: a malformed cell must + never fail the whole sync, and keeping the raw string instead would flip the column's Arrow + type between batches, which degrades the whole column back to text. Failures are logged as one + warning per column on its first occurrence (with a truncated sample) plus one aggregate summary + per run, never one line per value, so a systematically wrong file stays visible without + flooding the logs. The typed columns hold only dates, counts, and prices, so a sample value + can't carry personal data. + """ + + def __init__(self, logger: FilteringBoundLogger, endpoint: str) -> None: + self._logger = logger + self._endpoint = endpoint + self.counts: dict[str, int] = {} + + def record(self, column: str, value: Any) -> None: + self.counts[column] = self.counts.get(column, 0) + 1 + if self.counts[column] == 1: + self._logger.warning( + f"App Store Connect: unparseable value stored as null. " + f"endpoint={self._endpoint}, column={column}, value={str(value)[:40]!r}. " + f"Further failures in this column are counted and summarized when the run ends." + ) + + def flush(self) -> None: + if not self.counts: + return + total = sum(self.counts.values()) + self._logger.warning( + f"App Store Connect: {total} unparseable value(s) stored as null this run. " + f"endpoint={self._endpoint}, failures_by_column={self.counts}" + ) + + +# Name-driven typing for the delimited report families (sales/subscription reports and the +# analytics report streams), whose files are text with no type information. Columns are typed by +# NAME wherever they appear rather than per endpoint: Apple varies each report's column set by +# report type and version, and publishes Standard/Detailed variants of the analytics reports, so a +# name-driven mapping covers a column in every stream that carries it, including variants added +# later. Names not listed stay text; identifier-like numeric columns (apple_identifier, +# app_apple_id, subscription_apple_id, ...) deliberately stay text because they are join keys, not +# quantities, as do the Detailed-only attribution columns (campaign, page_title, source_info). +_REPORT_DATE_COLUMNS = frozenset( + { + # Sales and subscription-event reports carry month-first MM/DD/YYYY dates. + "begin_date", + "end_date", + "event_date", + "original_start_date", + # Analytics reports carry ISO YYYY-MM-DD dates. + "date", + "app_download_date", + "pre_order_start_date", + "pre_order_end_date", + } +) +_REPORT_INTEGER_COLUMNS = frozenset( + { + # Sales/subscription reports. Units can be negative: Apple books refunds as negative units. + "units", + "quantity", + "subscribers", + "consecutive_paid_periods", + "days_before_canceling", + "days_canceled", + # Analytics reports. + "sessions", + "unique_devices", + "counts", + "unique_counts", + "crashes", + "pre_orders_placed", + "pre_orders_canceled", + } +) +_REPORT_FLOAT_COLUMNS = frozenset( + { + # Monetary columns are amounts in the row's own currency column (customer_currency, + # currency_of_proceeds/proceeds_currency); the numeric type makes them filterable and + # summable WITHIN one currency, never across currencies. + "customer_price", + "developer_proceeds", + "total_session_duration", + } +) + +_REPORT_DATE_FORMATS = ("%m/%d/%Y", "%Y-%m-%d") + + +def _parse_report_date(text: str) -> date | None: + # Apple documents sales-report dates as month-first MM/DD/YYYY for every report type (layouts + # are fixed per report version, not localized per territory); analytics report files carry ISO + # YYYY-MM-DD. Both formats use strictly numeric strptime directives, which never consult the + # process locale, and a day-first reading is never attempted: a value like 13/01/2026 fails to + # parse rather than being silently guessed as January 13. + for fmt in _REPORT_DATE_FORMATS: + try: + return datetime.strptime(text, fmt).date() + except ValueError: + continue + return None + + +def _parse_report_int(text: str) -> int | None: + # Commas only ever appear as US-style thousands separators in Apple's reports; the decimal + # separator is always a point. + digits = text.replace(",", "") + try: + return int(digits) + except ValueError: + pass + try: + number = float(digits) + except ValueError: + return None + # A count that arrives as a whole-valued float ("3.0") still lands as an integer. A fractional + # or non-finite value nulls rather than silently truncating, and 2**53 bounds the conversion to + # where float holds integers exactly. + return int(number) if math.isfinite(number) and number.is_integer() and abs(number) <= 2**53 else None + + +def _parse_report_float(text: str) -> float | None: + try: + number = float(text.replace(",", "")) + except ValueError: + return None + # float() accepts "nan"/"inf", which no report legitimately contains. + return number if math.isfinite(number) else None + + +def _parse_iso_datetime(text: str) -> datetime | None: + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + +def _typed_report_value(column: str, value: Any, failures: _ParseFailureCounter) -> Any: + """Parse one delimited-report cell into its typed value, or null when it can't be parsed.""" + if not isinstance(value, str): + return value + if column in _REPORT_DATE_COLUMNS: + parse: Callable[[str], Any] = _parse_report_date + elif column in _REPORT_INTEGER_COLUMNS: + parse = _parse_report_int + elif column in _REPORT_FLOAT_COLUMNS: + parse = _parse_report_float + else: + return value + + text = value.strip() + if not text: + # Blank cells are routine (an empty price on a free row, an unset offer duration); they + # are nulls, not parse failures. + return None + parsed = parse(text) + if parsed is None: + failures.record(column, value) + return parsed + + +def _typed_json_api_row(row: dict[str, Any], failures: _ParseFailureCounter) -> dict[str, Any]: + """Convert a JSON:API row's ISO 8601 date-time attributes to UTC datetimes, in place. + + Apple's JSON:API resources carry every timestamp in an attribute named `...Date` (createdDate, + uploadedDate, expirationDate, earliestReleaseDate, lastModifiedDate), so the rule is + suffix-driven rather than a per-endpoint column list and covers attributes added later. Values + normalize to UTC because Apple emits varying local offsets and one column must stay in one + zone. Only table rows come through here; the resources the sync reads internally + (analyticsReportInstances and friends) keep their raw strings. + """ + for key, value in row.items(): + if not key.endswith("Date") or not isinstance(value, str): + continue + text = value.strip() + if not text: + row[key] = None + continue + parsed = _parse_iso_datetime(text) + if parsed is None: + failures.record(key, value) + row[key] = parsed + return row + + @dataclasses.dataclass(frozen=True, kw_only=True) class _Page: """One JSON:API page. ``resources`` and ``included`` share a type, so construction is @@ -291,12 +482,14 @@ def _iter_pages( page_params = None -def _page_rows(config: AppStoreConnectEndpointConfig, page: _Page) -> list[dict[str, Any]]: +def _page_rows( + config: AppStoreConnectEndpointConfig, page: _Page, failures: _ParseFailureCounter +) -> list[dict[str, Any]]: """Rows for one page: the flattened ``data`` resources, or, for endpoints configured to read a related resource off another collection's pages, the flattened ``included`` resources of that type. """ if config.rows_from_included_type is None: - return [_flatten_resource(resource) for resource in page.resources] + return [_typed_json_api_row(_flatten_resource(resource), failures) for resource in page.resources] # JSON:API full linkage guarantees every included resource is referenced from a primary # resource's relationship linkage; that linkage is where each row's parent id comes from. @@ -320,7 +513,7 @@ def _page_rows(config: AppStoreConnectEndpointConfig, page: _Page) -> list[dict[ continue row = _flatten_resource(resource) row[config.included_parent_column] = parent_ids.get(str(resource.get("id"))) - rows.append(row) + rows.append(_typed_json_api_row(row, failures)) return rows @@ -347,6 +540,7 @@ def _get_collection( token_provider: AppStoreConnectTokenProvider, logger: FilteringBoundLogger, manager: ResumableSourceManager[AppStoreConnectResumeConfig], + failures: _ParseFailureCounter, ) -> Iterator[list[dict[str, Any]]]: resume = _load_resume(manager) resumed_url = resume.next_url if resume is not None else None @@ -355,7 +549,7 @@ def _get_collection( params: dict[str, Any] | None = None if resumed_url else dict(config.params) for page in _iter_pages(session, token_provider, logger, url, params): - rows = _page_rows(config, page) + rows = _page_rows(config, page, failures) if rows: yield rows # Save AFTER yielding so a crash re-fetches the page we just emitted rather than skipping it; @@ -370,6 +564,7 @@ def _get_app_fanout( token_provider: AppStoreConnectTokenProvider, logger: FilteringBoundLogger, manager: ResumableSourceManager[AppStoreConnectResumeConfig], + failures: _ParseFailureCounter, ) -> Iterator[list[dict[str, Any]]]: app_ids = _list_app_ids(session, token_provider, logger) resume = _load_resume(manager) @@ -393,7 +588,7 @@ def _get_app_fanout( params = dict(config.params) for page in _iter_pages(session, token_provider, logger, url, params): - rows = _page_rows(config, page) + rows = _page_rows(config, page, failures) if rows: for row in rows: row["app_id"] = app_id @@ -434,7 +629,7 @@ def _decompress_report(payload: bytes) -> str: return raw.decode("utf-8-sig", errors="replace") -def _parse_report(payload: bytes, report_date: date) -> list[dict[str, Any]]: +def _parse_report(payload: bytes, report_date: date, failures: _ParseFailureCounter) -> list[dict[str, Any]]: reader = csv.reader(io.StringIO(_decompress_report(payload)), delimiter="\t") try: header = next(reader) @@ -442,16 +637,16 @@ def _parse_report(payload: bytes, report_date: date) -> list[dict[str, Any]]: return [] columns = [_normalize_report_column(column) for column in header] - report_date_str = report_date.isoformat() rows: list[dict[str, Any]] = [] for values in reader: if not any(value.strip() for value in values): continue row: dict[str, Any] = { - column: (values[index] if index < len(values) else None) for index, column in enumerate(columns) + column: _typed_report_value(column, values[index] if index < len(values) else None, failures) + for index, column in enumerate(columns) } - row["report_date"] = report_date_str + row["report_date"] = report_date # 1-based position in the file. A published day's report is immutable, so (report_date, _line) # is a stable unique key and re-reading a day merges instead of duplicating. row["_line"] = len(rows) + 1 @@ -467,6 +662,7 @@ def _fetch_report( logger: FilteringBoundLogger, vendor_number: str, report_date: date, + failures: _ParseFailureCounter, ) -> list[dict[str, Any]]: params: dict[str, str] = { "filter[frequency]": config.report_frequency, @@ -494,7 +690,7 @@ def _fetch_report( # same condition instead (see `missing_report_status_codes`). return [] - return _parse_report(response.content, report_date) + return _parse_report(response.content, report_date, failures) def _get_sales_report( @@ -503,6 +699,7 @@ def _get_sales_report( token_provider: AppStoreConnectTokenProvider, logger: FilteringBoundLogger, manager: ResumableSourceManager[AppStoreConnectResumeConfig], + failures: _ParseFailureCounter, vendor_number: str | None, should_use_incremental_field: bool, db_incremental_field_last_value: Any, @@ -533,7 +730,7 @@ def _get_sales_report( report_date = start days_fetched = 0 while report_date <= end and days_fetched < SALES_REPORT_MAX_DAYS_PER_RUN: - rows = _fetch_report(session, config, token_provider, logger, vendor_number, report_date) + rows = _fetch_report(session, config, token_provider, logger, vendor_number, report_date, failures) if rows: yield rows @@ -790,7 +987,9 @@ def _open_segment_text(spool: IO[bytes]) -> IO[str]: return io.TextIOWrapper(spool, encoding="utf-8-sig", errors="replace") -def _iter_segment_rows(text: IO[str], processing_date: date, line_start: int) -> Iterator[dict[str, Any]]: +def _iter_segment_rows( + text: IO[str], processing_date: date, line_start: int, failures: _ParseFailureCounter +) -> Iterator[dict[str, Any]]: header_line = text.readline() if not header_line.strip(): return @@ -799,16 +998,16 @@ def _iter_segment_rows(text: IO[str], processing_date: date, line_start: int) -> # delimited text, so sniff the delimiter from the header instead of assuming one. delimiter = "\t" if "\t" in header_line else "," columns = [_normalize_report_column(column) for column in next(csv.reader([header_line], delimiter=delimiter))] - processing_date_str = processing_date.isoformat() line = line_start for values in csv.reader(text, delimiter=delimiter): if not any(value.strip() for value in values): continue row: dict[str, Any] = { - column: (values[index] if index < len(values) else None) for index, column in enumerate(columns) + column: _typed_report_value(column, values[index] if index < len(values) else None, failures) + for index, column in enumerate(columns) } - row["processing_date"] = processing_date_str + row["processing_date"] = processing_date # 1-based position within the instance, continuing across its segments. A published # instance is immutable, so (app_id, processing_date, _line) stays a stable unique key # and re-reading an instance merges instead of duplicating. @@ -824,6 +1023,7 @@ def _get_analytics_report( token_provider: AppStoreConnectTokenProvider, logger: FilteringBoundLogger, manager: ResumableSourceManager[AppStoreConnectResumeConfig], + failures: _ParseFailureCounter, should_use_incremental_field: bool, db_incremental_field_last_value: Any, ) -> Iterator[list[dict[str, Any]]]: @@ -903,7 +1103,7 @@ def _get_analytics_report( spool = _download_segment(logger, segment) try: with _open_segment_text(spool) as text: - for row in _iter_segment_rows(text, processing_date, line): + for row in _iter_segment_rows(text, processing_date, line, failures): row["app_id"] = app_id line = row["_line"] batch.append(row) @@ -963,36 +1163,44 @@ def get_rows( config = APP_STORE_CONNECT_ENDPOINTS[endpoint] session = _make_session(private_key) token_provider = AppStoreConnectTokenProvider(issuer_id, key_id, private_key) + failures = _ParseFailureCounter(logger, endpoint) - if config.kind == "collection": - yield from _get_collection(session, config, token_provider, logger, resumable_source_manager) - elif config.kind == "app_fanout": - yield from _get_app_fanout(session, config, token_provider, logger, resumable_source_manager) - elif config.kind == "analytics_report": - yield from _get_analytics_report( - session, - # Segment listings ride a capture-disabled session: their bodies carry presigned - # URLs whose query strings are short-lived credentials the name-based scrubbers - # can't recognise. - _make_session(private_key, capture=False), - config, - token_provider, - logger, - resumable_source_manager, - should_use_incremental_field, - db_incremental_field_last_value, - ) - else: # "sales_report" - yield from _get_sales_report( - session, - config, - token_provider, - logger, - resumable_source_manager, - vendor_number, - should_use_incremental_field, - db_incremental_field_last_value, - ) + try: + if config.kind == "collection": + yield from _get_collection(session, config, token_provider, logger, resumable_source_manager, failures) + elif config.kind == "app_fanout": + yield from _get_app_fanout(session, config, token_provider, logger, resumable_source_manager, failures) + elif config.kind == "analytics_report": + yield from _get_analytics_report( + session, + # Segment listings ride a capture-disabled session: their bodies carry presigned + # URLs whose query strings are short-lived credentials the name-based scrubbers + # can't recognise. + _make_session(private_key, capture=False), + config, + token_provider, + logger, + resumable_source_manager, + failures, + should_use_incremental_field, + db_incremental_field_last_value, + ) + else: # "sales_report" + yield from _get_sales_report( + session, + config, + token_provider, + logger, + resumable_source_manager, + failures, + vendor_number, + should_use_incremental_field, + db_incremental_field_last_value, + ) + finally: + # The unparseable-value summary rides the generator's teardown so it also surfaces for a + # run that fails or is abandoned mid-walk. + failures.flush() # Walked to completion, so drop the checkpoint — leaving it would let a later attempt on this job # resume mid-stream instead of restarting cleanly. diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/canonical_descriptions.py index 70bb9cb09dc7..34ad8a6029db 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/canonical_descriptions.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/canonical_descriptions.py @@ -154,7 +154,7 @@ "description": "One row of Apple's daily Sales and Trends summary report: units and developer proceeds per SKU, territory and product type.", "docs_url": "https://developer.apple.com/documentation/appstoreconnectapi/download_sales_and_trends_reports", "columns": { - "report_date": "Date the report covers, as `YYYY-MM-DD`.", + "report_date": "Date the report covers.", "_line": "1-based line number within that date's report file, used with report_date as the row key.", "provider": "Provider of the content, normally `APPLE`.", "provider_country": "Country of the provider.", @@ -164,14 +164,14 @@ "version": "Version of the app the transaction applied to.", "product_type_identifier": "Code describing the transaction type, such as a first download, update or in-app purchase.", "units": "Number of units for this row; negative values are refunds.", - "developer_proceeds": "Amount paid to you per unit, in the currency of proceeds.", + "developer_proceeds": "Amount paid to you per unit, in the row's currency_of_proceeds. Sum it only within a single currency.", "begin_date": "First date covered by the row.", "end_date": "Last date covered by the row.", "customer_currency": "Currency the customer was charged in.", "country_code": "App Store territory the transaction happened in.", "currency_of_proceeds": "Currency your proceeds are reported in.", "apple_identifier": "Apple's numeric identifier for the app.", - "customer_price": "Price the customer paid, in customer currency.", + "customer_price": "Price the customer paid, in the row's customer_currency. Amounts in different currencies are not comparable, so sum this only within a single currency.", "promo_code": "Promotional or offer code applied to the transaction.", "parent_identifier": "SKU of the parent app for an in-app purchase row.", "subscription": "Whether the row relates to a subscription product.", @@ -190,7 +190,7 @@ "description": "One row of Apple's daily Subscription summary report: active, paid and trial subscription counts by state and territory.", "docs_url": "https://developer.apple.com/documentation/appstoreconnectapi/download_sales_and_trends_reports", "columns": { - "report_date": "Date the report covers, as `YYYY-MM-DD`.", + "report_date": "Date the report covers.", "_line": "1-based line number within that date's report file, used with report_date as the row key.", "app_name": "Name of the app the subscription belongs to.", "app_apple_id": "Apple's numeric identifier for the app.", @@ -200,9 +200,9 @@ "standard_subscription_duration": "Billing duration of the subscription, such as 1 Month.", "promotional_offer_name": "Name of the promotional offer applied, if any.", "promotional_offer_id": "Identifier of the promotional offer applied, if any.", - "customer_price": "Price the customer pays per period, in customer currency.", + "customer_price": "Price the customer pays per period, in the row's customer_currency. Amounts in different currencies are not comparable, so sum this only within a single currency.", "customer_currency": "Currency the customer is charged in.", - "developer_proceeds": "Proceeds paid to you per period.", + "developer_proceeds": "Proceeds paid to you per period, in the row's proceeds_currency. Sum it only within a single currency.", "proceeds_currency": "Currency your proceeds are reported in.", "preserved_pricing": "Whether legacy preserved pricing applies.", "proceeds_reason": "Reason the applied proceeds rate was used.", @@ -217,7 +217,7 @@ "description": "One row of Apple's daily Subscription Event report: counts of subscription lifecycle events such as renewals, cancellations and plan changes.", "docs_url": "https://developer.apple.com/documentation/appstoreconnectapi/download_sales_and_trends_reports", "columns": { - "report_date": "Date the report covers, as `YYYY-MM-DD`.", + "report_date": "Date the report covers.", "_line": "1-based line number within that date's report file, used with report_date as the row key.", "event_date": "Date the events happened.", "event": "Lifecycle event counted, such as Subscribe, Renew, Cancel or Reactivate.", diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/tests/test_app_store_connect.py b/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/tests/test_app_store_connect.py index 090b5349b20a..5f1d2b68eb56 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/tests/test_app_store_connect.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/app_store_connect/tests/test_app_store_connect.py @@ -1,6 +1,6 @@ import gzip import hashlib -from datetime import date, timedelta +from datetime import UTC, date, datetime, timedelta from typing import Any import pytest @@ -27,7 +27,9 @@ _normalize_report_column, _Page, _parse_report, + _ParseFailureCounter, _require_api_url, + _typed_report_value, app_store_connect_source, check_credentials, get_rows, @@ -168,6 +170,7 @@ def _collect( manager: _FakeManager, *, vendor_number: str | None = None, + logger: MagicMock | None = None, **kwargs: Any, ) -> list[dict[str, Any]]: session = MagicMock() @@ -180,7 +183,7 @@ def _collect( private_key=PRIVATE_KEY_PEM, vendor_number=vendor_number, endpoint=endpoint, - logger=MagicMock(), + logger=logger if logger is not None else MagicMock(), resumable_source_manager=manager, **kwargs, ): @@ -514,6 +517,39 @@ def test_source_response_is_unpartitioned(self) -> None: assert response.partition_keys is None +class TestJsonApiDateTimeColumns: + def test_iso_datetime_attributes_become_utc_datetimes(self) -> None: + api = _FakeApi( + { + f"{BASE_URL}/v1/betaGroups": _page( + [ + _resource("betaGroups", "1", name="Zulu", createdDate="2026-03-04T10:00:00Z"), + _resource("betaGroups", "2", name="Offset", createdDate="2026-03-04T12:30:00+02:00"), + _resource("betaGroups", "3", name="Fractional", createdDate="2026-03-04T10:00:00.123456-05:00"), + ] + ) + } + ) + + rows = _collect("beta_groups", api, _FakeManager()) + + # Apple emits varying local offsets; normalizing to UTC keeps one column in one zone. + assert [row["createdDate"] for row in rows] == [ + datetime(2026, 3, 4, 10, 0, tzinfo=UTC), + datetime(2026, 3, 4, 10, 30, tzinfo=UTC), + datetime(2026, 3, 4, 15, 0, 0, 123456, tzinfo=UTC), + ] + assert [row["name"] for row in rows] == ["Zulu", "Offset", "Fractional"] + + def test_unparseable_datetime_is_nulled_rather_than_failing_the_sync(self) -> None: + api = _FakeApi({f"{BASE_URL}/v1/betaGroups": _page([_resource("betaGroups", "1", createdDate="last Tuesday")])}) + + rows = _collect("beta_groups", api, _FakeManager()) + + assert rows[0]["createdDate"] is None + assert rows[0]["id"] == "1" + + APPS_URL = f"{BASE_URL}/v1/apps" REQUESTS_URL = f"{BASE_URL}/v1/apps/A1/analyticsReportRequests" CREATE_REQUEST_URL = f"{BASE_URL}/v1/analyticsReportRequests" @@ -653,9 +689,9 @@ def test_full_chain_parses_daily_instances_into_keyed_rows(self) -> None: # _line continues across an instance's segments; a restart per segment would give two # rows the same merge key and lose one of them. assert [(row["app_id"], row["processing_date"], row["_line"], row["sessions"]) for row in rows] == [ - ("A1", "2026-08-01", 1, "5"), - ("A1", "2026-08-01", 2, "7"), - ("A1", "2026-08-02", 1, "2"), + ("A1", date(2026, 8, 1), 1, 5), + ("A1", date(2026, 8, 1), 2, 7), + ("A1", date(2026, 8, 2), 1, 2), ] assert rows[0]["app_apple_identifier"] == "123" assert api.posts == [] @@ -709,7 +745,7 @@ def test_incremental_walk_reads_from_the_watermark_day_inclusive(self) -> None: db_incremental_field_last_value=date(2026, 8, 2), ) - assert [row["processing_date"] for row in rows] == ["2026-08-02"] + assert [row["processing_date"] for row in rows] == [date(2026, 8, 2)] assert _segments_url("I1") not in [url for url, _ in api.calls] def test_resume_bookmark_floors_the_walk(self) -> None: @@ -723,7 +759,7 @@ def test_resume_bookmark_floors_the_walk(self) -> None: rows = _collect_analytics(api, manager) - assert [row["processing_date"] for row in rows] == ["2026-08-02"] + assert [row["processing_date"] for row in rows] == [date(2026, 8, 2)] assert _segments_url("I1") not in [url for url, _ in api.calls] def test_unavailable_report_degrades_the_table_without_failing(self) -> None: @@ -793,7 +829,7 @@ def test_tab_delimited_segments_parse_too(self) -> None: rows = _collect_analytics(api, _FakeManager()) - assert [(row["date"], row["sessions"]) for row in rows] == [("2026-08-01", "5")] + assert [(row["date"], row["sessions"]) for row in rows] == [(date(2026, 8, 1), 5)] def test_per_run_instance_cap_saves_a_resumable_bookmark(self) -> None: payload = _gzip_csv("Date,Sessions\n2026-08-01,5\n") @@ -807,7 +843,7 @@ def test_per_run_instance_cap_saves_a_resumable_bookmark(self) -> None: with patch(f"{MODULE}.ANALYTICS_MAX_INSTANCES_PER_RUN", 1): rows = _collect_analytics(api, manager) - assert [row["processing_date"] for row in rows] == ["2026-08-01"] + assert [row["processing_date"] for row in rows] == [date(2026, 8, 1)] assert manager.saved[-1].processing_date == "2026-08-02" def test_dates_walk_in_order_across_apps(self) -> None: @@ -842,11 +878,42 @@ def test_dates_walk_in_order_across_apps(self) -> None: rows = _collect_analytics(api, _FakeManager()) assert [(row["app_id"], row["processing_date"]) for row in rows] == [ - ("A1", "2026-08-01"), - ("A2", "2026-08-02"), - ("A1", "2026-08-03"), + ("A1", date(2026, 8, 1)), + ("A2", date(2026, 8, 2)), + ("A1", date(2026, 8, 3)), ] + def test_columns_are_typed_by_name_and_attribution_columns_stay_text(self) -> None: + # Typing is column-name-driven rather than per-endpoint: Apple publishes Standard and + # Detailed variants of each report with differing column sets, so any stream carrying a + # known date or metric column gets it typed, while the Detailed-only attribution columns + # (campaign, page_title, source_info) stay text by omission from the mapping. + payload = _gzip_csv( + "Date,App Name,App Download Date,Campaign,Page Title,Source Info," + "Sessions,Total Session Duration,Unique Devices\n" + "2026-08-01,Example,2026-07-15,summer-launch,Alternate page,com.example.social,5,321.5,4\n" + ) + api = _analytics_api( + instances=[_instance("I1", "2026-08-01")], + segments_by_instance={"I1": [_segment("S1", "https://r.s3.amazonaws.com/1", payload)]}, + segment_payloads={"https://r.s3.amazonaws.com/1": payload}, + ) + + row = _collect_analytics(api, _FakeManager())[0] + + assert row["processing_date"] == date(2026, 8, 1) + assert row["date"] == date(2026, 8, 1) + assert row["app_download_date"] == date(2026, 7, 15) + assert row["sessions"] == 5 and isinstance(row["sessions"], int) + assert row["total_session_duration"] == 321.5 + assert row["unique_devices"] == 4 + assert (row["campaign"], row["page_title"], row["source_info"]) == ( + "summer-launch", + "Alternate page", + "com.example.social", + ) + assert row["app_name"] == "Example" + def test_analytics_source_response_checkpoints_ascending(self) -> None: response = app_store_connect_source( issuer_id="issuer", @@ -895,6 +962,68 @@ def test_match_tolerates_case_and_hyphen_drift(self) -> None: assert self._resolve("analytics_app_store_preorders", "App Store Pre-orders Standard") == "REP1" +def _failures() -> _ParseFailureCounter: + return _ParseFailureCounter(MagicMock(), "sales_reports") + + +class TestTypedReportValues: + @parameterized.expand( + [ + ("month_first_date", "begin_date", "03/04/2026", date(2026, 3, 4)), + ("single_digit_month_and_day", "begin_date", "3/4/2026", date(2026, 3, 4)), + ("iso_analytics_date", "date", "2026-03-04", date(2026, 3, 4)), + # 02/03/2026 must read as February 3, never March 2: the parse is month-first by + # Apple's report spec, independent of any locale or dayfirst heuristic. + ("ambiguous_date_reads_month_first", "event_date", "02/03/2026", date(2026, 2, 3)), + ("padded_date", "end_date", " 03/04/2026 ", date(2026, 3, 4)), + ("count", "units", "3", 3), + ("negative_refund_count", "units", "-2", -2), + ("count_with_thousands_separator", "units", "1,234", 1234), + ("whole_valued_float_count", "units", "3.0", 3), + ("price", "customer_price", "0.99", 0.99), + ("price_with_thousands_separator", "customer_price", "1,234.56", 1234.56), + ("unmapped_column_untouched", "promo_code", "0099", "0099"), + ("identifier_stays_text", "apple_identifier", "123456789", "123456789"), + ] + ) + def test_mapped_columns_parse_and_unmapped_stay_text( + self, _name: str, column: str, value: str, expected: Any + ) -> None: + failures = _failures() + + parsed = _typed_report_value(column, value, failures) + + assert parsed == expected + assert type(parsed) is type(expected) + assert failures.counts == {} + + @parameterized.expand([("empty", "begin_date", ""), ("whitespace", "units", " ")]) + def test_blank_cells_are_null_but_not_counted_as_failures(self, _name: str, column: str, value: str) -> None: + failures = _failures() + + assert _typed_report_value(column, value, failures) is None + assert failures.counts == {} + + @parameterized.expand( + [ + # A heuristic parser would read 13/01/2026 as January 13 once the month overflows; + # rejecting it keeps a mis-formatted file loud instead of silently day-first. + ("day_first_date", "begin_date", "13/01/2026"), + ("nonsense_date", "begin_date", "garbage"), + ("out_of_range_date", "begin_date", "04/31/2026"), + ("non_numeric_count", "units", "N/A"), + ("fractional_count", "units", "2.5"), + ("currency_prefixed_price", "customer_price", "USD 0.99"), + ("non_finite_price", "customer_price", "inf"), + ] + ) + def test_unparseable_values_are_null_and_counted(self, _name: str, column: str, value: str) -> None: + failures = _failures() + + assert _typed_report_value(column, value, failures) is None + assert failures.counts == {column: 1} + + class TestReportColumnNames: @parameterized.expand( [ @@ -912,26 +1041,40 @@ def test_header_is_normalized_to_snake_case(self, _name: str, header: str, expec class TestParseReport: - def test_gzipped_tsv_becomes_keyed_rows(self) -> None: - tsv = "Provider\tSKU\tUnits\tDeveloper Proceeds\nAPPLE\tacme-pro\t3\t2.10\nAPPLE\tacme-lite\t1\t0.70\n" + def test_gzipped_tsv_becomes_keyed_and_typed_rows(self) -> None: + tsv = ( + "Provider\tSKU\tUnits\tCustomer Price\tDeveloper Proceeds\tBegin Date\tEnd Date\tApple Identifier\n" + "APPLE\tacme-pro\t3\t2.99\t2.10\t03/04/2026\t03/04/2026\t123456789\n" + "APPLE\tacme-lite\t1\t0.99\t0.70\t03/04/2026\t03/04/2026\t123456789\n" + ) - rows = _parse_report(gzip.compress(tsv.encode()), date(2026, 3, 4)) + rows = _parse_report(gzip.compress(tsv.encode()), date(2026, 3, 4), _failures()) + # Dates and quantities arrive typed; identifier-like numeric columns stay text because + # they are join keys, not quantities. assert rows == [ { "provider": "APPLE", "sku": "acme-pro", - "units": "3", - "developer_proceeds": "2.10", - "report_date": "2026-03-04", + "units": 3, + "customer_price": 2.99, + "developer_proceeds": 2.10, + "begin_date": date(2026, 3, 4), + "end_date": date(2026, 3, 4), + "apple_identifier": "123456789", + "report_date": date(2026, 3, 4), "_line": 1, }, { "provider": "APPLE", "sku": "acme-lite", - "units": "1", - "developer_proceeds": "0.70", - "report_date": "2026-03-04", + "units": 1, + "customer_price": 0.99, + "developer_proceeds": 0.70, + "begin_date": date(2026, 3, 4), + "end_date": date(2026, 3, 4), + "apple_identifier": "123456789", + "report_date": date(2026, 3, 4), "_line": 2, }, ] @@ -939,26 +1082,26 @@ def test_gzipped_tsv_becomes_keyed_rows(self) -> None: def test_blank_lines_are_skipped_so_line_numbers_stay_dense(self) -> None: tsv = "SKU\tUnits\nacme-pro\t3\n\n \nacme-lite\t1\n" - rows = _parse_report(gzip.compress(tsv.encode()), date(2026, 3, 4)) + rows = _parse_report(gzip.compress(tsv.encode()), date(2026, 3, 4), _failures()) assert [(row["sku"], row["_line"]) for row in rows] == [("acme-pro", 1), ("acme-lite", 2)] def test_short_rows_are_padded_with_none(self) -> None: tsv = "SKU\tUnits\tDevice\nacme-pro\t3\n" - rows = _parse_report(gzip.compress(tsv.encode()), date(2026, 3, 4)) + rows = _parse_report(gzip.compress(tsv.encode()), date(2026, 3, 4), _failures()) assert rows[0]["device"] is None def test_uncompressed_payload_is_parsed_too(self) -> None: # urllib3 unwraps a `Content-Encoding: gzip` body before we see it. - rows = _parse_report(b"SKU\tUnits\nacme-pro\t3\n", date(2026, 3, 4)) + rows = _parse_report(b"SKU\tUnits\nacme-pro\t3\n", date(2026, 3, 4), _failures()) assert rows[0]["sku"] == "acme-pro" @parameterized.expand([("empty", b""), ("header_only", b"SKU\tUnits\n")]) def test_reports_without_data_rows_yield_nothing(self, _name: str, payload: bytes) -> None: - assert _parse_report(payload, date(2026, 3, 4)) == [] + assert _parse_report(payload, date(2026, 3, 4), _failures()) == [] class TestSalesReports: @@ -983,7 +1126,10 @@ def test_walks_dates_forward_from_the_watermark_and_skips_empty_days(self) -> No ) # Yesterday (2026-03-04) is the newest date Apple has published; 03-03 404s and is skipped. - assert [(row["report_date"], row["units"]) for row in rows] == [("2026-03-02", "1"), ("2026-03-04", "2")] + assert [(row["report_date"], row["units"]) for row in rows] == [ + (date(2026, 3, 2), 1), + (date(2026, 3, 4), 2), + ] assert [params["filter[reportDate]"] for _, params in api.calls] == ["2026-03-02", "2026-03-03", "2026-03-04"] @freeze_time("2026-03-05 09:00:00") @@ -1002,7 +1148,7 @@ def test_subscription_report_tolerates_apples_misleading_400(self) -> None: db_incremental_field_last_value=date(2026, 3, 2), ) - assert [(row["report_date"], row["units"]) for row in rows] == [("2026-03-04", "1")] + assert [(row["report_date"], row["units"]) for row in rows] == [(date(2026, 3, 4), 1)] assert [params["filter[reportDate]"] for _, params in api.calls] == ["2026-03-02", "2026-03-03", "2026-03-04"] @freeze_time("2026-03-05 09:00:00") @@ -1028,6 +1174,31 @@ def test_sales_report_400_is_not_tolerated(self) -> None: ) ) + @freeze_time("2026-03-05 09:00:00") + def test_unparseable_values_are_nulled_with_counted_warnings(self) -> None: + # Three bad units and one bad date must produce one first-occurrence warning per column + # plus one end-of-run summary, never one log line per value. + tsv = "SKU\tUnits\tBegin Date\nsku-1\tN/A\t03/04/2026\nsku-2\tN/A\t04/31/2026\nsku-3\tN/A\t03/04/2026\n" + api = self._api({"2026-03-04": tsv}) + logger = MagicMock() + + rows = _collect( + "sales_reports", + api, + _FakeManager(), + vendor_number="85234567", + logger=logger, + should_use_incremental_field=True, + db_incremental_field_last_value=date(2026, 3, 4), + ) + + assert [row["units"] for row in rows] == [None, None, None] + assert [row["begin_date"] for row in rows] == [date(2026, 3, 4), None, date(2026, 3, 4)] + warning_messages = [call.args[0] for call in logger.warning.call_args_list] + assert len(warning_messages) == 3 + assert "'units': 3" in warning_messages[-1] + assert "'begin_date': 1" in warning_messages[-1] + @freeze_time("2026-03-05 09:00:00") def test_sends_the_report_type_filters_from_settings(self) -> None: api = self._api({"2026-03-04": "SKU\tUnits\nacme\t1\n"}) From 3520d6545f4cef7e58de6c2fa0275454fee9f751 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 06:21:14 +0000 Subject: [PATCH 023/289] fix(warehouse-sources): surface Shopify's real token error Read the 4xx body from Shopify's OAuth token endpoint and put its error / error_description into the raised message and a structured log. Split invalid_client and unsupported_grant_type into their own user-facing messages, mirroring the 404 store-not-found handling. Fix the wizard copy: drop "reconnect your Shopify integration" (there is no reconnect step), replace the misleading shpss_... secret placeholder, and point the caption at the Dev Dashboard app setup the docs describe. Generated-By: PostHog Desktop Task-Id: 1f2b2a31-107d-4c69-a045-bfa3e0a91ae2 --- .../data_imports/sources/shopify/shopify.py | 83 +++++++++++++++++-- .../data_imports/sources/shopify/source.py | 17 +++- .../shopify/tests/test_access_token.py | 27 ++++++ 3 files changed, 116 insertions(+), 11 deletions(-) diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py index 66d7c15e478b..85f4fbb8513d 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py @@ -5,6 +5,7 @@ from typing import Any import requests +import structlog from requests.exceptions import ChunkedEncodingError from structlog.types import FilteringBoundLogger from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter @@ -32,18 +33,37 @@ SHOPIFY_PAGE_SIZE_OVERRIDES, ) +logger = structlog.get_logger(__name__) + # Resume phases for the shopify source. "all" is the non-incremental branch; # "earliest" and "latest" are the two incremental sweeps in shopify_source.get_rows. PHASE_ALL = "all" PHASE_EARLIEST = "earliest" PHASE_LATEST = "latest" -# Raised when Shopify's OAuth token endpoint returns a 4xx — the app credentials are -# invalid or the app was uninstalled, so re-auth is the only fix. `ShopifySource. -# get_non_retryable_errors` matches on this exact text to fail the job fast. +# Raised when Shopify's OAuth token endpoint returns a 4xx that we can't attribute to a more +# specific cause below. The app credentials are invalid or revoked, so re-entering them is the +# only fix. `ShopifySource.get_non_retryable_errors` matches on this exact text to fail the job +# fast. The raised message also carries Shopify's own `error`/`error_description` (see +# `_oauth_error_detail`) so support can see what Shopify objected to. SHOPIFY_ACCESS_TOKEN_AUTH_ERROR = ( - "Failed to retrieve Shopify access token: the app credentials are invalid or the " - "app was uninstalled. Please reconnect your Shopify integration." + "Shopify rejected your app credentials. Check the client ID and secret in your Shopify app and re-enter them here." +) + +# Raised on a 4xx whose body reports `error: invalid_client` — the client ID or secret does not +# match a Shopify app. Surfaced separately so the message names the field to fix. +SHOPIFY_ACCESS_TOKEN_INVALID_CLIENT_ERROR = ( + "Shopify rejected your app credentials (invalid_client). Check that the client ID and " + "secret both come from the same Shopify app, then re-enter them here." +) + +# Raised on a 4xx whose body reports `error: unsupported_grant_type` — the app can't use the +# client_credentials grant PostHog mints tokens with. This is the legacy store-admin custom app +# type; PostHog needs a Dev Dashboard app. Surfaced separately so the message points at the fix. +SHOPIFY_ACCESS_TOKEN_UNSUPPORTED_GRANT_ERROR = ( + "This Shopify app does not support the sign-in method PostHog uses " + "(unsupported_grant_type). Create a Dev Dashboard app by following the PostHog Shopify " + "docs, then enter its client ID and secret." ) # Raised when the OAuth token endpoint returns 404 — there is no store at @@ -325,6 +345,42 @@ def normalize_store_id(raw: str) -> str: return store_id +def _parse_oauth_error(response: requests.Response) -> tuple[str | None, str | None]: + """Shopify's OAuth token endpoint returns `{"error": ..., "error_description": ...}` on a 4xx. + An edge or proxy can return non-JSON (e.g. an HTML error page) instead, so parse defensively + and return `(None, None)` when the body has no usable error code.""" + try: + body = response.json() + except ValueError: + return None, None + if not isinstance(body, dict): + return None, None + error = body.get("error") + description = body.get("error_description") + return ( + error if isinstance(error, str) else None, + description if isinstance(description, str) else None, + ) + + +def _access_token_auth_error_message(error_code: str | None) -> str: + """The user-facing message for a token-endpoint 4xx, chosen from Shopify's `error` code.""" + if error_code == "invalid_client": + return SHOPIFY_ACCESS_TOKEN_INVALID_CLIENT_ERROR + if error_code == "unsupported_grant_type": + return SHOPIFY_ACCESS_TOKEN_UNSUPPORTED_GRANT_ERROR + return SHOPIFY_ACCESS_TOKEN_AUTH_ERROR + + +def _oauth_error_detail(error_code: str | None, error_description: str | None, status_code: int) -> str: + """Shopify's raw error appended to the raised message so support can see what Shopify said.""" + if error_code and error_description: + return f"Shopify {error_code}: {error_description}, HTTP {status_code}" + if error_code: + return f"Shopify {error_code}, HTTP {status_code}" + return f"HTTP {status_code}" + + @retry( # A transient TLS/connection drop on the token endpoint (e.g. SSL EOF, proxy/egress hiccup, # connect/read timeout) surfaces from `post` as requests ConnectionError/Timeout — SSLError @@ -362,10 +418,21 @@ def _get_shopify_access_token(shopify_store_id: str, shopify_client_id: str, sho # instead of telling them their credentials are bad. if access_res.status_code == 404: raise Exception(f"{SHOPIFY_STORE_NOT_FOUND_ERROR} (HTTP 404)") - # Any other 4xx means the app credentials are invalid/revoked (e.g. the app was - # uninstalled) — re-auth is the only fix, so surface a non-retryable message. + # Any other 4xx means the app credentials are invalid/revoked — re-auth is the only fix, + # so surface a non-retryable message. Read Shopify's own `error`/`error_description` so + # the user gets the specific cause and support can see what Shopify rejected. if 400 <= access_res.status_code < 500 and access_res.status_code != 429: - raise Exception(f"{SHOPIFY_ACCESS_TOKEN_AUTH_ERROR} (HTTP {access_res.status_code})") + error_code, error_description = _parse_oauth_error(access_res) + logger.warning( + "Shopify OAuth token request failed", + store_id=shopify_store_id, + status_code=access_res.status_code, + shopify_error=error_code, + shopify_error_description=error_description, + ) + message = _access_token_auth_error_message(error_code) + detail = _oauth_error_detail(error_code, error_description, access_res.status_code) + raise Exception(f"{message} ({detail})") # 429 (rate limit) and 5xx (e.g. a 502 Bad Gateway from Shopify's edge) are transient — # retry locally with backoff instead of failing the import, mirroring the GraphQL path. raise ShopifyRetryableError( diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.py index 3b7a25e9a2d8..8c22da93435c 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.py @@ -28,6 +28,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.settings import ENDPOINT_CONFIGS from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.shopify import ( SHOPIFY_ACCESS_TOKEN_AUTH_ERROR, + SHOPIFY_ACCESS_TOKEN_INVALID_CLIENT_ERROR, + SHOPIFY_ACCESS_TOKEN_UNSUPPORTED_GRANT_ERROR, SHOPIFY_GRAPHQL_ACCESS_DENIED_ERROR, SHOPIFY_GRAPHQL_UNAUTHORIZED_ERROR_MATCH, SHOPIFY_GRAPHQL_UNAUTHORIZED_ERROR_MESSAGE, @@ -66,8 +68,13 @@ def get_canonical_descriptions(self) -> CanonicalDescriptions: def get_non_retryable_errors(self) -> dict[str, str | None]: return { # 4xx from Shopify's OAuth token endpoint — invalid/revoked app credentials. - # Retrying cannot recover; the user must reconnect the integration. + # Retrying cannot recover; the user must re-enter valid credentials. SHOPIFY_ACCESS_TOKEN_AUTH_ERROR: SHOPIFY_ACCESS_TOKEN_AUTH_ERROR, + # 4xx `invalid_client` — the client ID or secret does not match a Shopify app. + SHOPIFY_ACCESS_TOKEN_INVALID_CLIENT_ERROR: SHOPIFY_ACCESS_TOKEN_INVALID_CLIENT_ERROR, + # 4xx `unsupported_grant_type` — the app can't use the client_credentials grant, so + # the user needs a Dev Dashboard app instead of a legacy custom app. + SHOPIFY_ACCESS_TOKEN_UNSUPPORTED_GRANT_ERROR: SHOPIFY_ACCESS_TOKEN_UNSUPPORTED_GRANT_ERROR, # 404 from the same endpoint — no store at this subdomain. Retrying cannot # recover; the user must correct the store id. SHOPIFY_STORE_NOT_FOUND_ERROR: SHOPIFY_STORE_NOT_FOUND_ERROR, @@ -106,7 +113,11 @@ def get_source_config(self) -> SourceConfig: name=SchemaExternalDataSourceType.SHOPIFY, category=DataWarehouseSourceCategory.E_COMMERCE, iconPath="/static/services/shopify.png", - caption="""Enter your Shopify credentials to automatically pull your Shopify data into the PostHog Data warehouse.""", + caption=( + "Create a Shopify Dev Dashboard app, then enter its client ID and secret here to " + "pull your Shopify data into the PostHog Data warehouse. The docs walk through the " + "app setup steps." + ), docsUrl="https://posthog.com/docs/data-warehouse/sources/shopify", fields=cast( list[FieldType], @@ -136,7 +147,7 @@ def get_source_config(self) -> SourceConfig: label="Secret", type=SourceFieldInputConfigType.PASSWORD, required=True, - placeholder="shpss_...", + placeholder="client-secret", secret=True, ), ], diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_access_token.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_access_token.py index 67f4bc2cfc31..9b3ed4106113 100644 --- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_access_token.py +++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_access_token.py @@ -5,6 +5,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.shopify import ( SHOPIFY_ACCESS_TOKEN_AUTH_ERROR, + SHOPIFY_ACCESS_TOKEN_INVALID_CLIENT_ERROR, + SHOPIFY_ACCESS_TOKEN_UNSUPPORTED_GRANT_ERROR, SHOPIFY_STORE_NOT_FOUND_ERROR, ShopifyRetryableError, _get_shopify_access_token, @@ -46,6 +48,31 @@ def test_get_access_token_4xx_is_non_retryable(status_code): ) +@pytest.mark.parametrize( + "error_code,expected_message", + [ + ("invalid_client", SHOPIFY_ACCESS_TOKEN_INVALID_CLIENT_ERROR), + ("unsupported_grant_type", SHOPIFY_ACCESS_TOKEN_UNSUPPORTED_GRANT_ERROR), + ], +) +def test_get_access_token_4xx_maps_shopify_error_code(error_code, expected_message): + # Shopify names the cause in the 4xx body. Each recognized code must surface its own + # message and carry Shopify's raw error/description so support can read what Shopify said. + body = {"error": error_code, "error_description": "Shopify says so"} + with _patched_token_call(_mock_response(400, ok=False, json_data=body)): + with pytest.raises(Exception) as exc_info: + _get_shopify_access_token("store", "client-id", "client-secret") + + error_message = str(exc_info.value) + assert expected_message in error_message + assert error_code in error_message + assert "Shopify says so" in error_message + patterns = ShopifySource().get_non_retryable_errors() + assert any(pattern in error_message for pattern in patterns), ( + f"4xx token error '{error_message}' should match a non-retryable pattern" + ) + + def test_get_access_token_404_reports_missing_store(): # A 404 means the store subdomain doesn't resolve to a real store — a distinct, # non-retryable failure that reconnecting the app can't fix, so it must surface the From 04d22c66a08a52899d5aa67bffc179a82e44a92d Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Fri, 14 Aug 2026 10:55:47 -0400 Subject: [PATCH 024/289] fix(capture): bound global rate limiter redis work to enforceable keys The limiter keys on token:distinct_id, an unbounded key space, and queued a Redis sync on every cache miss. Sync volume tracked total traffic rather than the keys that could be limited, and the per-tick MGET grew past a hardcoded 100ms timeout, leaving cached counts stale and enforcement degraded. Add an absolute sync floor so keys that cannot be limited skip the round trip, bound and chunk the per-tick pipeline, and expose the command timeouts and local cache lifetimes as settings. Co-Authored-By: Claude Opus 5 --- rust/capture/src/config.rs | 51 ++ rust/capture/src/global_rate_limiter.rs | 10 + rust/capture/src/v1/quota_limiter_shim.rs | 8 + rust/capture/tests/common/utils.rs | 10 + .../limiters/benches/global_rate_limiter.rs | 6 + .../limiters/src/global_rate_limiter.rs | 574 +++++++++++++----- .../global_rate_limiter_integration_tests.rs | 6 + 7 files changed, 527 insertions(+), 138 deletions(-) diff --git a/rust/capture/src/config.rs b/rust/capture/src/config.rs index a03a2a8817ec..cbc0feca8c9c 100644 --- a/rust/capture/src/config.rs +++ b/rust/capture/src/config.rs @@ -149,6 +149,57 @@ pub struct Config { #[envconfig(default = "5000000")] pub global_rate_limit_token_distinctid_local_cache_max_entries: u64, + /// Minimum effective event count before a key earns a Redis sync. Keys below + /// this cannot be limited whatever other nodes report, so syncing them costs + /// two Redis keys per tick for no enforcement value. With an unbounded key + /// space this is what keeps the pipeline sized to enforceable keys rather + /// than to total traffic. 0 syncs every key. + /// + /// The level is per-pod, so this must stay well under + /// `threshold / pod_count` or a key sitting at the threshold but spread + /// evenly across the fleet would never sync and could never be limited. + #[envconfig(default = "10")] + pub global_rate_limit_min_sync_floor: u64, + + /// Max keys drained from the pending-sync set per tick. Excess stays queued, + /// so a backlog shows up as sync staleness rather than a tick that overruns + /// its interval. + #[envconfig(default = "20000")] + pub global_rate_limit_max_sync_keys_per_tick: usize, + + /// Max Redis keys per individual command. Reads cost two keys per entity, so + /// an entity chunk is half this. Bounds how long any single command can take, + /// which is what the per-command timeouts below are budgeting for. + #[envconfig(default = "2000")] + pub global_rate_limit_max_keys_per_command: usize, + + /// How many chunked commands may be in flight at once per Redis instance. + #[envconfig(default = "4")] + pub global_rate_limit_max_concurrent_commands: usize, + + /// How long a local cache entry survives regardless of access (seconds). + /// Bounds how stale a key's cached count can be before it is rebuilt. + #[envconfig(default = "600")] + pub global_rate_limit_local_cache_ttl_secs: u64, + + /// Evict local cache entries not accessed within this window (seconds). + /// This is the main lever on cache cardinality: with a key space dominated + /// by one-shot identities, most entries are pure churn and hold a slot for + /// the full idle window. Must stay at or above the rate-limit window, or + /// entries expire inside the enforcement window and the limiter loses the + /// counts it is supposed to be accumulating -- values below the window are + /// clamped up, with a warning. + #[envconfig(default = "300")] + pub global_rate_limit_local_cache_idle_timeout_secs: u64, + + /// Timeout for a single global rate limiter Redis read command (milliseconds). + #[envconfig(default = "250")] + pub global_rate_limit_read_timeout_ms: u64, + + /// Timeout for a single global rate limiter Redis write command (milliseconds). + #[envconfig(default = "250")] + pub global_rate_limit_write_timeout_ms: u64, + // --- Token-only limiter config (not currently used in production, retained for new_token()) --- /// Per-token rate limit threshold per window interval /// Note: default is too high to trigger limiting in production diff --git a/rust/capture/src/global_rate_limiter.rs b/rust/capture/src/global_rate_limiter.rs index 6676d0582c4b..4b8e1abf3a1d 100644 --- a/rust/capture/src/global_rate_limiter.rs +++ b/rust/capture/src/global_rate_limiter.rs @@ -197,6 +197,16 @@ impl GlobalRateLimiter { ), local_cache_max_entries, metrics_scope: metrics_scope.to_string(), + min_sync_floor: config.global_rate_limit_min_sync_floor, + max_sync_keys_per_tick: config.global_rate_limit_max_sync_keys_per_tick, + max_keys_per_command: config.global_rate_limit_max_keys_per_command, + max_concurrent_commands: config.global_rate_limit_max_concurrent_commands, + global_read_timeout: Duration::from_millis(config.global_rate_limit_read_timeout_ms), + global_write_timeout: Duration::from_millis(config.global_rate_limit_write_timeout_ms), + local_cache_ttl: Duration::from_secs(config.global_rate_limit_local_cache_ttl_secs), + local_cache_idle_timeout: Duration::from_secs( + config.global_rate_limit_local_cache_idle_timeout_secs, + ), ..Default::default() }; diff --git a/rust/capture/src/v1/quota_limiter_shim.rs b/rust/capture/src/v1/quota_limiter_shim.rs index 61ae959aa6b5..454b59eb1148 100644 --- a/rust/capture/src/v1/quota_limiter_shim.rs +++ b/rust/capture/src/v1/quota_limiter_shim.rs @@ -131,6 +131,14 @@ mod tests { global_rate_limit_token_distinctid_threshold: 10_000, global_rate_limit_token_distinctid_overrides_csv: None, global_rate_limit_token_distinctid_local_cache_max_entries: 300_000, + global_rate_limit_min_sync_floor: 0, + global_rate_limit_max_sync_keys_per_tick: 20_000, + global_rate_limit_max_keys_per_command: 2_000, + global_rate_limit_max_concurrent_commands: 4, + global_rate_limit_local_cache_ttl_secs: 600, + global_rate_limit_local_cache_idle_timeout_secs: 300, + global_rate_limit_read_timeout_ms: 250, + global_rate_limit_write_timeout_ms: 250, global_rate_limit_token_threshold: 300_000, global_rate_limit_token_overrides_csv: None, global_rate_limit_token_local_cache_max_entries: 300_000, diff --git a/rust/capture/tests/common/utils.rs b/rust/capture/tests/common/utils.rs index b759184c49f7..47b785a6afca 100644 --- a/rust/capture/tests/common/utils.rs +++ b/rust/capture/tests/common/utils.rs @@ -47,6 +47,16 @@ pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { global_rate_limit_token_distinctid_threshold: 10_000, global_rate_limit_token_distinctid_overrides_csv: None, global_rate_limit_token_distinctid_local_cache_max_entries: 300_000, + // Integration tests assert on exact limiter behavior at a threshold of + // 10_000, so every key syncs and every tick drains fully. + global_rate_limit_min_sync_floor: 0, + global_rate_limit_max_sync_keys_per_tick: 20_000, + global_rate_limit_max_keys_per_command: 2_000, + global_rate_limit_max_concurrent_commands: 4, + global_rate_limit_local_cache_ttl_secs: 600, + global_rate_limit_local_cache_idle_timeout_secs: 300, + global_rate_limit_read_timeout_ms: 250, + global_rate_limit_write_timeout_ms: 250, global_rate_limit_token_threshold: 300_000, global_rate_limit_token_overrides_csv: None, global_rate_limit_token_local_cache_max_entries: 300_000, diff --git a/rust/common/limiters/benches/global_rate_limiter.rs b/rust/common/limiters/benches/global_rate_limiter.rs index d32834345758..9214ef1ee43e 100644 --- a/rust/common/limiters/benches/global_rate_limiter.rs +++ b/rust/common/limiters/benches/global_rate_limiter.rs @@ -87,6 +87,12 @@ fn bench_config() -> GlobalRateLimiterConfig { global_read_timeout: Duration::from_millis(50), global_write_timeout: Duration::from_millis(50), metrics_scope: "bench".to_string(), + // The benchmark measures the hot path against a fully-syncing limiter, so + // the floor and the per-tick bound are both left wide open. + min_sync_floor: 0, + max_sync_keys_per_tick: 100_000, + max_keys_per_command: 2_000, + max_concurrent_commands: 4, } } diff --git a/rust/common/limiters/src/global_rate_limiter.rs b/rust/common/limiters/src/global_rate_limiter.rs index 625f93d866e2..3209702fae28 100644 --- a/rust/common/limiters/src/global_rate_limiter.rs +++ b/rust/common/limiters/src/global_rate_limiter.rs @@ -42,6 +42,12 @@ const GLOBAL_RATE_LIMITER_ESTIMATE_DRIFT_HISTOGRAM: &str = "global_rate_limiter_ const GLOBAL_RATE_LIMITER_SYNC_STALENESS_HISTOGRAM: &str = "global_rate_limiter_sync_staleness_ms"; const GLOBAL_RATE_LIMITER_CACHE_SIZE_GAUGE: &str = "global_rate_limiter_cache_size"; const GLOBAL_RATE_LIMITER_EVICTION_COUNTER: &str = "global_rate_limiter_eviction_total"; +/// Keys still queued for sync after a tick took its bounded slice. +const GLOBAL_RATE_LIMITER_SYNC_DEFERRED_GAUGE: &str = "global_rate_limiter_sync_deferred_size"; +/// Syncs not queued because the key's level is below `min_sync_floor`. +const GLOBAL_RATE_LIMITER_SYNC_SKIPPED_COUNTER: &str = "global_rate_limiter_sync_skipped_total"; +/// Redis commands issued per tick, after chunking. +const GLOBAL_RATE_LIMITER_COMMANDS_HISTOGRAM: &str = "global_rate_limiter_commands_per_tick"; /// Number of custom-key thresholds applied at the last successful refresh. const CUSTOM_THRESHOLDS_LOADED_GAUGE: &str = "global_rate_limiter_custom_thresholds_loaded"; /// Unix timestamp of the last successful custom-key threshold refresh. @@ -181,6 +187,34 @@ pub struct GlobalRateLimiterConfig { pub local_cache_max_entries: u64, /// Capacity of the mpsc channel for async global cache updates pub channel_capacity: usize, + /// Minimum effective level before a key is worth a Redis round trip. + /// + /// A key far below its threshold cannot be limited no matter what the other + /// nodes report, so syncing it buys nothing and costs two Redis keys per + /// tick. With an unbounded key space (e.g. keyed on distinct_id) the + /// one-shot keys dominate, so this floor is what keeps the pipeline sized to + /// the keys that can actually be enforced rather than to total traffic. + /// + /// The level is per-node, so the ceiling on a safe value is + /// `global_threshold / node_count` -- above that, a key sitting exactly at + /// the threshold but spread evenly across the fleet would never sync and so + /// could never be limited. Keep well under that: the saving is dominated by + /// the single-event keys, so a small floor captures nearly all of it. + /// + /// Set to 0 to sync every key, restoring the pre-floor behavior. + pub min_sync_floor: u64, + /// Maximum keys drained from `pending_sync` per tick. The remainder stays + /// queued for the next tick, so a backlog degrades into staleness instead of + /// a tick loop that overruns its own interval. + pub max_sync_keys_per_tick: usize, + /// Maximum Redis keys per individual command. Reads cost two keys per entity + /// (current + previous epoch), so an entity chunk is half this. Bounds how + /// long any single command can take, which is what the per-command timeout + /// is actually budgeting for. + pub max_keys_per_command: usize, + /// How many chunked commands may be in flight at once against one instance. + /// Trades tick wall-clock against instantaneous Redis load. + pub max_concurrent_commands: usize, /// Per-key custom limits. Overrides the default limit for specific *more granular* keys. /// /// Wrapped in `Arc>` so the map can be atomically replaced at @@ -245,10 +279,14 @@ impl Default for GlobalRateLimiterConfig { local_cache_ttl: Duration::from_secs(600), local_cache_idle_timeout: Duration::from_secs(300), global_cache_ttl: window_interval.mul_f64(2.0), - global_read_timeout: Duration::from_millis(100), - global_write_timeout: Duration::from_millis(100), + global_read_timeout: Duration::from_millis(250), + global_write_timeout: Duration::from_millis(250), local_cache_max_entries: 300_000, channel_capacity: 1_000_000, + min_sync_floor: 10, + max_sync_keys_per_tick: 20_000, + max_keys_per_command: 2_000, + max_concurrent_commands: 4, custom_keys: Arc::new(ArcSwap::from_pointee(HashMap::new())), custom_key_resolver: None, custom_key_source: None, @@ -460,6 +498,23 @@ impl GlobalRateLimiterImpl { let scope: &'static str = Box::leak(config.metrics_scope.clone().into_boxed_str()); + // An idle timeout shorter than the window would expire entries inside the + // very window they are accumulating counts for, silently under-enforcing. + // Clamp rather than error: this is deploy-time config, and taking capture + // down over a tuning value is worse than running with a corrected one. + let mut config = config; + if config.local_cache_idle_timeout < config.window_interval { + warn!( + scope, + idle_timeout = ?config.local_cache_idle_timeout, + window_interval = ?config.window_interval, + "local_cache_idle_timeout below window_interval would drop counts \ + inside the enforcement window; clamping to window_interval" + ); + config.local_cache_idle_timeout = config.window_interval; + } + let config = config; + let cache = Cache::builder() .max_capacity(config.local_cache_max_entries) .time_to_live(config.local_cache_ttl) @@ -553,23 +608,24 @@ impl GlobalRateLimiterImpl { let staleness_ms = now_instant.duration_since(entry.synced_at).as_millis() as f64; metrics::histogram!(GLOBAL_RATE_LIMITER_SYNC_STALENESS_HISTOGRAM, "scope" => self.scope).record(staleness_ms); - // Check if sync is needed based on pressure tier - let current_pressure = level / threshold as f64; - let effective_pressure = current_pressure.max(entry.pressure); - if let Some(tier_interval) = - tier_sync_interval(effective_pressure, self.config.sync_interval) - { - if now_instant.duration_since(entry.synced_at) > tier_interval { - self.pending_sync.insert(key.to_string()); - metrics::counter!(GLOBAL_RATE_LIMITER_CACHE_COUNTER, "scope" => self.scope, "result" => "sync_queued") - .increment(1); - } else { - metrics::counter!(GLOBAL_RATE_LIMITER_CACHE_COUNTER, "scope" => self.scope, "result" => "hit") - .increment(1); - } + // Sync decision. The absolute floor is checked first: a key this far + // under its threshold cannot be limited whatever the other nodes + // report, so the round trip buys nothing and the key space is large + // enough that those round trips are the dominant cost. Above the + // floor the pressure tier sets the cadence, and a key that clears the + // floor while still idle-tier syncs on the Low cadence rather than + // never -- otherwise a key that is hot across the fleet but cold on + // any single node would never be discovered. + if self.sync_floor_blocks(level) { + metrics::counter!(GLOBAL_RATE_LIMITER_CACHE_COUNTER, "scope" => self.scope, "result" => "hit") + .increment(1); } else { - // Idle tier: only queue sync if local traffic has pushed us above idle threshold - if current_pressure >= 0.1 { + let effective_pressure = (level / threshold as f64).max(entry.pressure); + let tier_interval = + tier_sync_interval(effective_pressure, self.config.sync_interval) + .unwrap_or_else(|| self.config.sync_interval.mul_f64(4.0)); + + if now_instant.duration_since(entry.synced_at) > tier_interval { self.pending_sync.insert(key.to_string()); metrics::counter!(GLOBAL_RATE_LIMITER_CACHE_COUNTER, "scope" => self.scope, "result" => "sync_queued") .increment(1); @@ -597,7 +653,9 @@ impl GlobalRateLimiterImpl { pressure: 0.0, }; self.cache.insert(key.to_string(), entry); - self.pending_sync.insert(key.to_string()); + if !self.sync_floor_blocks(count as f64) { + self.pending_sync.insert(key.to_string()); + } (count as f64, false) }; @@ -622,6 +680,24 @@ impl GlobalRateLimiterImpl { } } + /// True when `level` sits below the configured sync floor, meaning a Redis + /// round trip for this key cannot change any enforcement decision. Records + /// the skip so the saving is visible next to `cache_counts_total`. + /// + /// A floor of 0 disables the check, restoring sync-every-key behavior. + fn sync_floor_blocks(&self, level: f64) -> bool { + if self.config.min_sync_floor == 0 || level >= self.config.min_sync_floor as f64 { + return false; + } + metrics::counter!( + GLOBAL_RATE_LIMITER_SYNC_SKIPPED_COUNTER, + "scope" => self.scope, + "reason" => "below_floor", + ) + .increment(1); + true + } + /// Queue an update to be batched and sent to Redis fn enqueue_update(&self, key: &str, count: u64, timestamp: DateTime) { let update = UpdateRequest { @@ -797,9 +873,21 @@ impl GlobalRateLimiterImpl { // throttled full scan (slow-moving, see TIER_SCAN_INTERVAL_TICKS). Self::emit_cache_gauges(cache, scope, tick_n); - // Drain pending sync set (lock-free: iterate then clear) - let sync_keys: Vec = pending_sync.iter().map(|r| r.key().clone()).collect(); - pending_sync.clear(); + // Take a bounded slice of the pending set rather than all of it. The + // remainder stays queued, so a backlog surfaces as sync staleness instead + // of a tick that overruns its own interval and starves every other key. + // `collect` drops the iterator before the removals, which keeps us off + // dashmap's held-shard-lock path. + let sync_keys: Vec = pending_sync + .iter() + .take(config.max_sync_keys_per_tick) + .map(|r| r.key().clone()) + .collect(); + for key in &sync_keys { + pending_sync.remove(key); + } + metrics::gauge!(GLOBAL_RATE_LIMITER_SYNC_DEFERRED_GAUGE, "scope" => scope) + .set(pending_sync.len() as f64); // Take ownership of write batch let writes = std::mem::take(write_batch); @@ -851,131 +939,188 @@ impl GlobalRateLimiterImpl { writes: &HashMap<(String, i64), u64>, scope: &'static str, ) { - let redis_idx_str = redis_idx.to_string(); + let redis_idx_str: Arc = Arc::from(redis_idx.to_string().as_str()); let now = Utc::now(); let ttl = config.global_cache_ttl.as_secs() as usize; - // --- WRITES --- - if !writes.is_empty() { - let write_items: Vec<(String, i64)> = writes - .iter() - .map(|((key, epoch), count)| { - let redis_key = epoch_key(&config.redis_key_prefix, key, *epoch); - (redis_key, *count as i64) - }) - .collect(); - - let write_count = write_items.len(); - let pipeline_start = Instant::now(); - - match tokio::time::timeout( - config.global_write_timeout, - redis.batch_incr_by_expire(write_items, ttl), - ) - .await - { - Ok(Ok(_)) => { - metrics::counter!( - GLOBAL_RATE_LIMITER_RECORDS_COUNTER, - "scope" => scope, - "op" => "redis_write", - "redis_idx" => redis_idx_str.clone(), - ) - .increment(write_count as u64); - metrics::histogram!( - GLOBAL_RATE_LIMITER_PIPELINE_HISTOGRAM, - "scope" => scope, - "redis_idx" => redis_idx_str.clone(), - ) - .record(pipeline_start.elapsed().as_micros() as f64 / 1000.0); - } - Ok(Err(e)) => { - metrics::counter!( - GLOBAL_RATE_LIMITER_ERROR_COUNTER, - "scope" => scope, - "step" => "pipeline", - "cause" => "redis_write", - "redis_idx" => redis_idx_str.clone(), - ) - .increment(1); - warn!(error = %e, records = write_count, redis_idx = redis_idx, "Failed to write rate limit batch to Redis"); - } - Err(_) => { - metrics::counter!( - GLOBAL_RATE_LIMITER_ERROR_COUNTER, - "scope" => scope, - "step" => "pipeline", - "cause" => "timeout", - "redis_idx" => redis_idx_str.clone(), + let writes_issued = + Self::run_writes(config, redis, &redis_idx_str, writes, ttl, scope).await; + let reads_issued = + Self::run_reads(config, redis, &redis_idx_str, cache, sync_keys, now, scope).await; + + metrics::histogram!(GLOBAL_RATE_LIMITER_COMMANDS_HISTOGRAM, "scope" => scope, "op" => "write") + .record(writes_issued as f64); + metrics::histogram!(GLOBAL_RATE_LIMITER_COMMANDS_HISTOGRAM, "scope" => scope, "op" => "read") + .record(reads_issued as f64); + } + + /// Issue the write half of a tick as size-bounded, concurrently-executed + /// commands. Returns how many commands were issued. + /// + /// One oversized command is the failure mode this exists to prevent: the + /// per-command timeout can only be a meaningful budget if the command's size + /// is bounded, otherwise a growing key space silently converts a working + /// timeout into a guaranteed one. + async fn run_writes( + config: &GlobalRateLimiterConfig, + redis: &Arc, + redis_idx_str: &Arc, + writes: &HashMap<(String, i64), u64>, + ttl: usize, + scope: &'static str, + ) -> usize { + if writes.is_empty() { + return 0; + } + + let write_items: Vec<(String, i64)> = writes + .iter() + .map(|((key, epoch), count)| { + let redis_key = epoch_key(&config.redis_key_prefix, key, *epoch); + (redis_key, *count as i64) + }) + .collect(); + + let chunks: Vec> = write_items + .chunks(config.max_keys_per_command.max(1)) + .map(|chunk| chunk.to_vec()) + .collect(); + let issued = chunks.len(); + + // Waves of `max_concurrent_commands` rather than a `buffer_unordered` + // stream: the stream combinator forces a higher-ranked `Send` bound the + // spawned tick task cannot satisfy, and this keeps the same bound on + // in-flight commands. + for wave in chunks.chunks(config.max_concurrent_commands.max(1)) { + let futures = wave.iter().map(|chunk| { + let redis_idx_str = redis_idx_str.clone(); + async move { + let chunk_len = chunk.len(); + let started = Instant::now(); + match tokio::time::timeout( + config.global_write_timeout, + redis.batch_incr_by_expire(chunk.clone(), ttl), ) - .increment(1); - warn!( - records = write_count, - redis_idx = redis_idx, - "Redis write timeout in pipeline" - ); + .await + { + Ok(Ok(_)) => { + metrics::counter!( + GLOBAL_RATE_LIMITER_RECORDS_COUNTER, + "scope" => scope, + "op" => "redis_write", + "redis_idx" => redis_idx_str.clone(), + ) + .increment(chunk_len as u64); + metrics::histogram!( + GLOBAL_RATE_LIMITER_PIPELINE_HISTOGRAM, + "scope" => scope, + "redis_idx" => redis_idx_str.clone(), + ) + .record(started.elapsed().as_micros() as f64 / 1000.0); + } + Ok(Err(e)) => { + Self::record_pipeline_error(scope, &redis_idx_str, "redis_write"); + warn!(error = %e, records = chunk_len, "Failed to write rate limit batch to Redis"); + } + Err(_) => { + Self::record_pipeline_error(scope, &redis_idx_str, "write_timeout"); + warn!(records = chunk_len, "Redis write timeout in pipeline"); + } + } } - } + }); + futures::future::join_all(futures).await; } - // --- READS --- - if !sync_keys.is_empty() { - // Build MGET key list: for each entity, we need current + prev epoch key - let mut mget_keys: Vec = Vec::with_capacity(sync_keys.len() * 2); - for key in sync_keys { - let (curr, prev) = - epoch_keys(&config.redis_key_prefix, key, now, config.window_interval); - mget_keys.push(curr); - mget_keys.push(prev); - } + issued + } - let pipeline_start = Instant::now(); - match tokio::time::timeout(config.global_read_timeout, redis.mget(mget_keys)).await { - Ok(Ok(results)) => { - metrics::counter!( - GLOBAL_RATE_LIMITER_RECORDS_COUNTER, - "scope" => scope, - "op" => "redis_read", - "redis_idx" => redis_idx_str.clone(), - ) - .increment(results.len() as u64); - metrics::histogram!( - GLOBAL_RATE_LIMITER_PIPELINE_HISTOGRAM, - "scope" => scope, - "redis_idx" => redis_idx_str.clone(), - ) - .record(pipeline_start.elapsed().as_micros() as f64 / 1000.0); + /// Issue the read half of a tick as size-bounded, concurrently-executed + /// commands, applying each chunk's results as it lands. Returns how many + /// commands were issued. + #[allow(clippy::too_many_arguments)] + async fn run_reads( + config: &GlobalRateLimiterConfig, + redis: &Arc, + redis_idx_str: &Arc, + cache: &Cache, + sync_keys: &[String], + now: DateTime, + scope: &'static str, + ) -> usize { + if sync_keys.is_empty() { + return 0; + } - Self::process_read_results(config, cache, sync_keys, &results, now, scope); - } - Ok(Err(e)) => { - metrics::counter!( - GLOBAL_RATE_LIMITER_ERROR_COUNTER, - "scope" => scope, - "step" => "pipeline", - "cause" => "redis_error", - "redis_idx" => redis_idx_str.clone(), - ) - .increment(1); - warn!(keys = sync_keys.len(), redis_idx = redis_idx, error = %e, "Failed to read rate limits from Redis"); - } - Err(_) => { - metrics::counter!( - GLOBAL_RATE_LIMITER_ERROR_COUNTER, - "scope" => scope, - "step" => "pipeline", - "cause" => "timeout", - "redis_idx" => redis_idx_str.clone(), - ) - .increment(1); - warn!( - keys = sync_keys.len(), - redis_idx = redis_idx, - "Redis read timeout in pipeline" - ); + // Each entity costs two Redis keys (current + previous epoch), so the + // entity chunk is half the per-command key budget. + let entities_per_chunk = (config.max_keys_per_command / 2).max(1); + let chunks: Vec<&[String]> = sync_keys.chunks(entities_per_chunk).collect(); + let issued = chunks.len(); + + // See `run_writes` for why this is waves of `join_all` rather than a + // `buffer_unordered` stream. + for wave in chunks.chunks(config.max_concurrent_commands.max(1)) { + let futures = wave.iter().map(|chunk| { + let redis_idx_str = redis_idx_str.clone(); + async move { + let mut mget_keys: Vec = Vec::with_capacity(chunk.len() * 2); + for key in chunk.iter() { + let (curr, prev) = + epoch_keys(&config.redis_key_prefix, key, now, config.window_interval); + mget_keys.push(curr); + mget_keys.push(prev); + } + + let started = Instant::now(); + match tokio::time::timeout(config.global_read_timeout, redis.mget(mget_keys)) + .await + { + Ok(Ok(results)) => { + metrics::counter!( + GLOBAL_RATE_LIMITER_RECORDS_COUNTER, + "scope" => scope, + "op" => "redis_read", + "redis_idx" => redis_idx_str.clone(), + ) + .increment(results.len() as u64); + metrics::histogram!( + GLOBAL_RATE_LIMITER_PIPELINE_HISTOGRAM, + "scope" => scope, + "redis_idx" => redis_idx_str.clone(), + ) + .record(started.elapsed().as_micros() as f64 / 1000.0); + + Self::process_read_results(config, cache, chunk, &results, now, scope); + } + Ok(Err(e)) => { + Self::record_pipeline_error(scope, &redis_idx_str, "redis_error"); + warn!(keys = chunk.len(), error = %e, "Failed to read rate limits from Redis"); + } + Err(_) => { + Self::record_pipeline_error(scope, &redis_idx_str, "read_timeout"); + warn!(keys = chunk.len(), "Redis read timeout in pipeline"); + } + } } - } + }); + futures::future::join_all(futures).await; } + + issued + } + + /// Record a pipeline-step failure. `cause` distinguishes read from write so + /// a saturating side is identifiable from the metric alone. + fn record_pipeline_error(scope: &'static str, redis_idx_str: &Arc, cause: &'static str) { + metrics::counter!( + GLOBAL_RATE_LIMITER_ERROR_COUNTER, + "scope" => scope, + "step" => "pipeline", + "cause" => cause, + "redis_idx" => redis_idx_str.clone(), + ) + .increment(1); } /// Execute a tick partitioned across multiple Redis instances. @@ -1177,6 +1322,13 @@ mod tests { global_read_timeout: Duration::from_millis(5), global_write_timeout: Duration::from_millis(10), metrics_scope: "test".to_string(), + // Tests drive a threshold of 10, so a production-sized floor would + // suppress every sync. 0 keeps the pre-floor behavior; the floor's + // own behavior is covered by the dedicated tests below. + min_sync_floor: 0, + max_sync_keys_per_tick: 20_000, + max_keys_per_command: 2_000, + max_concurrent_commands: 4, } } @@ -1345,10 +1497,14 @@ mod tests { assert_eq!(config.global_cache_ttl, Duration::from_secs(120)); assert_eq!(config.local_cache_ttl, Duration::from_secs(600)); assert_eq!(config.local_cache_idle_timeout, Duration::from_secs(300)); - assert_eq!(config.global_read_timeout, Duration::from_millis(100)); - assert_eq!(config.global_write_timeout, Duration::from_millis(100)); + assert_eq!(config.global_read_timeout, Duration::from_millis(250)); + assert_eq!(config.global_write_timeout, Duration::from_millis(250)); assert_eq!(config.local_cache_max_entries, 300_000); assert_eq!(config.channel_capacity, 1_000_000); + assert_eq!(config.min_sync_floor, 10); + assert_eq!(config.max_sync_keys_per_tick, 20_000); + assert_eq!(config.max_keys_per_command, 2_000); + assert_eq!(config.max_concurrent_commands, 4); assert!(config.custom_keys.load().is_empty()); assert!(config.custom_key_resolver.is_none()); assert_eq!(config.metrics_scope, "default"); @@ -1724,6 +1880,148 @@ mod tests { ); } + /// `test_config` with the sync floor set and the background drain parked, so + /// `pending_sync` assertions observe only what `check_limit` queued. + fn config_with_floor(floor: u64) -> GlobalRateLimiterConfig { + GlobalRateLimiterConfig { + min_sync_floor: floor, + tick_interval: Duration::from_secs(3600), + ..test_config() + } + } + + #[tokio::test] + async fn test_min_sync_floor_gates_cold_miss_sync() { + // (floor, count, expect_queued) + let cases = vec![ + (0, 1, true), // floor disabled: every miss syncs (pre-floor behavior) + (5, 1, false), // below floor: no round trip for a key that cannot be limited + (5, 5, true), // exactly at the floor + (5, 9, true), // above the floor + ]; + + for (floor, count, expect_queued) in cases { + let client = Arc::new(MockRedisClient::new()); + let limiter = + GlobalRateLimiterImpl::new(config_with_floor(floor), vec![client]).unwrap(); + let key = format!("cold_{floor}_{count}"); + + limiter.check_limit(&key, count, None).await; + + assert_eq!( + limiter.pending_sync.contains(&key), + expect_queued, + "floor={floor} count={count} should queue sync = {expect_queued}" + ); + } + } + + #[tokio::test] + async fn test_idle_timeout_clamped_up_to_window_interval() { + // (idle_timeout_secs, expected_secs) + let cases = vec![ + (10, 60), // below the 60s window: clamped up + (59, 60), // just below: clamped up + (60, 60), // exactly at the window: untouched + (300, 300), // above: untouched + ]; + + for (idle_secs, expected_secs) in cases { + let client = Arc::new(MockRedisClient::new()); + let config = GlobalRateLimiterConfig { + local_cache_idle_timeout: Duration::from_secs(idle_secs), + ..test_config() + }; + let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap(); + + assert_eq!( + limiter.config.local_cache_idle_timeout, + Duration::from_secs(expected_secs), + "idle_timeout={idle_secs}s against a 60s window should resolve to {expected_secs}s -- \ + an idle timeout inside the window expires entries mid-window and silently under-enforces" + ); + } + } + + #[tokio::test] + async fn test_idle_tier_key_above_floor_still_syncs() { + let client = Arc::new(MockRedisClient::new()); + let config = GlobalRateLimiterConfig { + global_threshold: 1000, + ..config_with_floor(10) + }; + let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap(); + + // Locally accumulated events don't decay, so this entry sits at level 50: + // idle by pressure (0.05 < 0.1) but well above the absolute floor, and + // last synced longer ago than the Low cadence (4 * 15s). + limiter.cache.insert( + "fleet_hot".to_string(), + CacheEntry { + estimated_count: 0.0, + synced_at: Instant::now() - Duration::from_secs(120), + local_pending: 50, + pressure: 0.05, + }, + ); + + limiter.check_limit("fleet_hot", 1, None).await; + + assert!( + limiter.pending_sync.contains("fleet_hot"), + "an idle-tier key above the floor must still sync, else a key hot across \ + the fleet but cold on any single node is never discovered and never limited" + ); + } + + #[tokio::test] + async fn test_tick_bounds_drain_and_chunks_reads() { + let mock = Arc::new(MockRedisClient::new()); + let client: Arc = mock.clone(); + let config = GlobalRateLimiterConfig { + max_sync_keys_per_tick: 10, + // 2 entities per read command (two epoch keys each). + max_keys_per_command: 4, + ..config_with_floor(0) + }; + let cache: Cache = Cache::builder().max_capacity(1000).build(); + let pending: Arc> = Arc::new(DashSet::new()); + for i in 0..25 { + pending.insert(format!("k{i}")); + } + let mut writes: HashMap<(String, i64), u64> = HashMap::new(); + + GlobalRateLimiterImpl::tick( + &config, + std::slice::from_ref(&client), + &cache, + &pending, + &mut writes, + "test", + 1, + ) + .await; + + assert_eq!( + pending.len(), + 15, + "tick must take at most max_sync_keys_per_tick and leave the remainder \ + queued -- deferring keeps the tick inside its interval, dropping them \ + would silently lose syncs" + ); + + let mget_calls = mock + .get_calls() + .into_iter() + .filter(|c| c.op == "mget") + .count(); + assert_eq!( + mget_calls, 5, + "10 drained keys at 2 entities per command must issue 5 bounded MGETs, \ + not one oversized command that cannot fit the per-command timeout" + ); + } + #[tokio::test] async fn test_sync_dedup() { let client = Arc::new(MockRedisClient::new()); diff --git a/rust/common/limiters/tests/global_rate_limiter_integration_tests.rs b/rust/common/limiters/tests/global_rate_limiter_integration_tests.rs index 7149812a5069..ac9623614e28 100644 --- a/rust/common/limiters/tests/global_rate_limiter_integration_tests.rs +++ b/rust/common/limiters/tests/global_rate_limiter_integration_tests.rs @@ -55,6 +55,12 @@ fn test_config(test_name: &str) -> GlobalRateLimiterConfig { global_read_timeout: Duration::from_millis(500), global_write_timeout: Duration::from_millis(500), metrics_scope: "integration_test".to_string(), + // These tests assert exact Redis counter values against a threshold of + // 1000, so every key must sync and every tick must drain fully. + min_sync_floor: 0, + max_sync_keys_per_tick: 10_000, + max_keys_per_command: 2_000, + max_concurrent_commands: 4, } } From d51c4e5c26e91294fadd219603275a0b39ba46b6 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Fri, 14 Aug 2026 11:26:47 -0400 Subject: [PATCH 025/289] fix(capture): harden sync floor and tick ordering from review findings Cap the sync floor at 1% of each key's threshold so a low custom override cannot sit entirely below a floor tuned for the global threshold and become unenforceable. Run reads before writes in the tick and bound the write drain with carry-over, so a high-cardinality write burst cannot starve the syncs enforcement depends on. Clamp local_cache_ttl to the window interval, matching the idle clamp. Co-Authored-By: Claude Fable 5 --- .../limiters/src/global_rate_limiter.rs | 206 ++++++++++++++++-- 1 file changed, 192 insertions(+), 14 deletions(-) diff --git a/rust/common/limiters/src/global_rate_limiter.rs b/rust/common/limiters/src/global_rate_limiter.rs index 3209702fae28..5d3388edbfe3 100644 --- a/rust/common/limiters/src/global_rate_limiter.rs +++ b/rust/common/limiters/src/global_rate_limiter.rs @@ -44,6 +44,8 @@ const GLOBAL_RATE_LIMITER_CACHE_SIZE_GAUGE: &str = "global_rate_limiter_cache_si const GLOBAL_RATE_LIMITER_EVICTION_COUNTER: &str = "global_rate_limiter_eviction_total"; /// Keys still queued for sync after a tick took its bounded slice. const GLOBAL_RATE_LIMITER_SYNC_DEFERRED_GAUGE: &str = "global_rate_limiter_sync_deferred_size"; +/// (key, epoch) write entries still batched after a tick took its bounded slice. +const GLOBAL_RATE_LIMITER_WRITE_DEFERRED_GAUGE: &str = "global_rate_limiter_write_deferred_size"; /// Syncs not queued because the key's level is below `min_sync_floor`. const GLOBAL_RATE_LIMITER_SYNC_SKIPPED_COUNTER: &str = "global_rate_limiter_sync_skipped_total"; /// Redis commands issued per tick, after chunking. @@ -513,6 +515,19 @@ impl GlobalRateLimiterImpl { ); config.local_cache_idle_timeout = config.window_interval; } + // Same hazard for the hard TTL: an entry evicted mid-window discards the + // counts it was accumulating, and the next request follows the always- + // allowed miss path. + if config.local_cache_ttl < config.window_interval { + warn!( + scope, + ttl = ?config.local_cache_ttl, + window_interval = ?config.window_interval, + "local_cache_ttl below window_interval would drop counts inside \ + the enforcement window; clamping to window_interval" + ); + config.local_cache_ttl = config.window_interval; + } let config = config; let cache = Cache::builder() @@ -616,7 +631,7 @@ impl GlobalRateLimiterImpl { // floor while still idle-tier syncs on the Low cadence rather than // never -- otherwise a key that is hot across the fleet but cold on // any single node would never be discovered. - if self.sync_floor_blocks(level) { + if self.sync_floor_blocks(level, threshold) { metrics::counter!(GLOBAL_RATE_LIMITER_CACHE_COUNTER, "scope" => self.scope, "result" => "hit") .increment(1); } else { @@ -653,7 +668,7 @@ impl GlobalRateLimiterImpl { pressure: 0.0, }; self.cache.insert(key.to_string(), entry); - if !self.sync_floor_blocks(count as f64) { + if !self.sync_floor_blocks(count as f64, threshold) { self.pending_sync.insert(key.to_string()); } @@ -680,13 +695,25 @@ impl GlobalRateLimiterImpl { } } - /// True when `level` sits below the configured sync floor, meaning a Redis - /// round trip for this key cannot change any enforcement decision. Records - /// the skip so the saving is visible next to `cache_counts_total`. + /// True when `level` sits below the sync floor for this key's threshold, + /// meaning a Redis round trip cannot change any enforcement decision. + /// Records the skip so the saving is visible next to `cache_counts_total`. + /// + /// The configured floor is capped at 1% of the key's own threshold. The + /// floor is a per-node level, so a fleet of N nodes can hide at most + /// N * floor events from Redis; the cap keeps that bypass under N% of the + /// threshold regardless of configuration. Without it, a custom threshold + /// far below the global one (the exact keys overrides exist to clamp) could + /// sit entirely below a floor tuned for the global threshold and never + /// sync, making the override unenforceable. /// - /// A floor of 0 disables the check, restoring sync-every-key behavior. - fn sync_floor_blocks(&self, level: f64) -> bool { - if self.config.min_sync_floor == 0 || level >= self.config.min_sync_floor as f64 { + /// A configured floor of 0 disables the check entirely. + fn sync_floor_blocks(&self, level: f64, threshold: u64) -> bool { + if self.config.min_sync_floor == 0 { + return false; + } + let effective_floor = self.config.min_sync_floor.min((threshold / 100).max(1)); + if level >= effective_floor as f64 { return false; } metrics::counter!( @@ -889,8 +916,27 @@ impl GlobalRateLimiterImpl { metrics::gauge!(GLOBAL_RATE_LIMITER_SYNC_DEFERRED_GAUGE, "scope" => scope) .set(pending_sync.len() as f64); - // Take ownership of write batch - let writes = std::mem::take(write_batch); + // Bound the write drain the same way. The deferred remainder stays in + // `write_batch`, where new arrivals merge into it by (key, epoch), so no + // count is lost -- it lands in the same epoch key up to a few ticks late. + // Without the bound, a high-cardinality burst produces a write batch + // whose waves consume the whole tick before reads run. + let writes: HashMap<(String, i64), u64> = + if write_batch.len() <= config.max_sync_keys_per_tick { + std::mem::take(write_batch) + } else { + let drain_keys: Vec<(String, i64)> = write_batch + .keys() + .take(config.max_sync_keys_per_tick) + .cloned() + .collect(); + drain_keys + .into_iter() + .filter_map(|k| write_batch.remove_entry(&k)) + .collect() + }; + metrics::gauge!(GLOBAL_RATE_LIMITER_WRITE_DEFERRED_GAUGE, "scope" => scope) + .set(write_batch.len() as f64); let read_count = sync_keys.len(); let write_count = writes.len(); @@ -943,10 +989,14 @@ impl GlobalRateLimiterImpl { let now = Utc::now(); let ttl = config.global_cache_ttl.as_secs() as usize; - let writes_issued = - Self::run_writes(config, redis, &redis_idx_str, writes, ttl, scope).await; + // Reads first: enforcement accuracy depends on fresh global counts, while + // a write is additive and lands correctly a tick late. The old + // writes-first order let a large write batch consume the tick before any + // read ran, starving the very syncs the burst made necessary. let reads_issued = Self::run_reads(config, redis, &redis_idx_str, cache, sync_keys, now, scope).await; + let writes_issued = + Self::run_writes(config, redis, &redis_idx_str, writes, ttl, scope).await; metrics::histogram!(GLOBAL_RATE_LIMITER_COMMANDS_HISTOGRAM, "scope" => scope, "op" => "write") .record(writes_issued as f64); @@ -1902,8 +1952,13 @@ mod tests { for (floor, count, expect_queued) in cases { let client = Arc::new(MockRedisClient::new()); - let limiter = - GlobalRateLimiterImpl::new(config_with_floor(floor), vec![client]).unwrap(); + // Threshold large enough (floor * 100 or more) that the 1% cap does + // not reduce the configured floor; the cap has its own test below. + let config = GlobalRateLimiterConfig { + global_threshold: 1000, + ..config_with_floor(floor) + }; + let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap(); let key = format!("cold_{floor}_{count}"); limiter.check_limit(&key, count, None).await; @@ -1943,6 +1998,129 @@ mod tests { } } + #[tokio::test] + async fn test_sync_floor_capped_at_one_percent_of_threshold() { + // (configured_floor, threshold, count, expect_queued) + let cases = vec![ + // Custom-style low threshold: cap = max(1, 100/100) = 1, so any + // counted event syncs. A floor tuned for the global threshold must + // not make a low custom override unenforceable. + (10, 100, 1, true), + // Threshold 500: cap = 5. The configured 10 is reduced to 5. + (10, 500, 4, false), + (10, 500, 5, true), + // Large threshold: cap = 150 leaves the configured 10 in charge. + (10, 15_000, 9, false), + (10, 15_000, 10, true), + ]; + + for (floor, threshold, count, expect_queued) in cases { + let client = Arc::new(MockRedisClient::new()); + let config = GlobalRateLimiterConfig { + global_threshold: threshold, + ..config_with_floor(floor) + }; + let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap(); + let key = format!("cap_{floor}_{threshold}_{count}"); + + limiter.check_limit(&key, count, None).await; + + assert_eq!( + limiter.pending_sync.contains(&key), + expect_queued, + "floor={floor} threshold={threshold} count={count} should queue sync = {expect_queued}" + ); + } + } + + #[tokio::test] + async fn test_ttl_clamped_up_to_window_interval() { + let client = Arc::new(MockRedisClient::new()); + let config = GlobalRateLimiterConfig { + local_cache_ttl: Duration::from_secs(1), + ..test_config() // 60s window + }; + let limiter = GlobalRateLimiterImpl::new(config, vec![client]).unwrap(); + + assert_eq!( + limiter.config.local_cache_ttl, + Duration::from_secs(60), + "a TTL below the window evicts entries mid-window; the next request \ + takes the always-allowed miss path and the limiter under-enforces" + ); + } + + #[tokio::test] + async fn test_tick_runs_reads_before_writes() { + let mock = Arc::new(MockRedisClient::new()); + let client: Arc = mock.clone(); + let config = config_with_floor(0); + let cache: Cache = Cache::builder().max_capacity(100).build(); + let pending: Arc> = Arc::new(DashSet::new()); + pending.insert("read_key".to_string()); + let mut writes: HashMap<(String, i64), u64> = HashMap::new(); + writes.insert(("write_key".to_string(), 1), 5); + + GlobalRateLimiterImpl::tick( + &config, + std::slice::from_ref(&client), + &cache, + &pending, + &mut writes, + "test", + 1, + ) + .await; + + let calls = mock.get_calls(); + let first_read = calls.iter().position(|c| c.op == "mget"); + let first_write = calls.iter().position(|c| c.op.starts_with("batch_incr")); + assert!( + first_read.is_some() && first_write.is_some(), + "tick should issue both a read and a write" + ); + assert!( + first_read < first_write, + "reads must run before writes: enforcement depends on fresh global \ + counts, and a large write batch running first starves the reads a \ + burst makes necessary. calls={calls:?}" + ); + } + + #[tokio::test] + async fn test_tick_bounds_write_drain_and_carries_remainder() { + let client: Arc = Arc::new(MockRedisClient::new()); + let config = GlobalRateLimiterConfig { + max_sync_keys_per_tick: 10, + ..config_with_floor(0) + }; + let cache: Cache = Cache::builder().max_capacity(100).build(); + let pending: Arc> = Arc::new(DashSet::new()); + let mut writes: HashMap<(String, i64), u64> = HashMap::new(); + for i in 0..25 { + writes.insert((format!("w{i}"), 1), 1); + } + + GlobalRateLimiterImpl::tick( + &config, + std::slice::from_ref(&client), + &cache, + &pending, + &mut writes, + "test", + 1, + ) + .await; + + assert_eq!( + writes.len(), + 15, + "tick must drain at most max_sync_keys_per_tick write entries and \ + leave the remainder batched -- deferring keeps counts (they merge by \ + key+epoch and land a tick late), dropping them would lose counts" + ); + } + #[tokio::test] async fn test_idle_tier_key_above_floor_still_syncs() { let client = Arc::new(MockRedisClient::new()); From 32760c5ee79926a3a0341bbc1c5d275828098200 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Fri, 14 Aug 2026 12:15:36 -0400 Subject: [PATCH 026/289] feat(capture): token-level rate limiter, redis self-heal, bounded write batch Wire the token-level aggregate limiter (off by default): identity-rotation floods spread volume so no per-(token, distinct_id) key ever approaches a threshold, and only a whole-token aggregate can see them. Token-limited events keep result Ok so per-key limiter counts stay identical across v0/v1. Give the GRL Redis client a heal path: MultiplexedConnection never reconnects after its TCP connection dies, so a Redis failover left every pod's limiter failing open until restart. The tick now heals on unrecoverable errors only (never on timeouts), with a cooldown. Cap the deferred write batch and purge unreadable epochs, closing the unbounded-memory vector the bounded write drain introduced. Add a management command to bootstrap the custom-threshold Redis blob, which no environment has ever written. Co-Authored-By: Claude Fable 5 --- .../sync_global_rate_limit_thresholds.py | 26 +++ ...test_global_rate_limit_threshold_config.py | 11 ++ rust/capture/src/config.rs | 33 +++- rust/capture/src/events/analytics.rs | 32 ++++ rust/capture/src/global_rate_limiter.rs | 27 +++- rust/capture/src/overflow_parity.rs | 1 + rust/capture/src/router.rs | 3 + rust/capture/src/setup.rs | 14 ++ rust/capture/src/v0_endpoint.rs | 1 + rust/capture/src/v1/analytics/process.rs | 148 +++++++++++++++++ rust/capture/src/v1/quota_limiter_shim.rs | 3 + rust/capture/src/v1/test_utils.rs | 8 + .../capture/tests/common/integration_utils.rs | 1 + rust/capture/tests/common/utils.rs | 3 + rust/capture/tests/integration_ai_endpoint.rs | 7 + .../tests/integration_ai_restrictions.rs | 3 + .../tests/integration_analytics_ai_routing.rs | 1 + .../integration_analytics_restrictions.rs | 2 + .../tests/integration_otel_endpoint.rs | 1 + .../tests/integration_replay_restrictions.rs | 2 + rust/capture/tests/quota_limiters.rs | 5 + .../limiters/benches/global_rate_limiter.rs | 1 + .../limiters/src/global_rate_limiter.rs | 153 +++++++++++++++++- .../global_rate_limiter_integration_tests.rs | 1 + rust/common/redis/Cargo.toml | 2 + rust/common/redis/src/client.rs | 122 +++++++++++--- rust/common/redis/src/lib.rs | 8 + rust/common/redis/src/read_write.rs | 5 + 28 files changed, 586 insertions(+), 38 deletions(-) create mode 100644 posthog/management/commands/sync_global_rate_limit_thresholds.py diff --git a/posthog/management/commands/sync_global_rate_limit_thresholds.py b/posthog/management/commands/sync_global_rate_limit_thresholds.py new file mode 100644 index 000000000000..aff1fd016b8a --- /dev/null +++ b/posthog/management/commands/sync_global_rate_limit_thresholds.py @@ -0,0 +1,26 @@ +from django.core.management.base import BaseCommand + +from posthog.models.global_rate_limit_threshold_config import ( + CUSTOM_THRESHOLDS_REDIS_KEY, + GlobalRateLimitThresholdConfig, + regenerate_redis_thresholds, +) + + +class Command(BaseCommand): + help = ( + "Write the capture global rate limiter's custom-threshold blob to Redis from the " + "current GlobalRateLimitThresholdConfig rows. Normally the post_save/post_delete " + "signals keep the blob current, but they only fire on row changes: an environment " + "with zero rows has never written the key at all, and capture treats the absent key " + "as fail-static (it polls forever without ever loading a map). Run this once per " + "environment to bootstrap the key (an explicit empty blob is a valid, loadable state), " + "or to force a resync if the key is lost." + ) + + def handle(self, *args, **options) -> None: + row_count = GlobalRateLimitThresholdConfig.objects.count() + regenerate_redis_thresholds() + self.stdout.write( + self.style.SUCCESS(f"Wrote {row_count} threshold override(s) to Redis key {CUSTOM_THRESHOLDS_REDIS_KEY!r}") + ) diff --git a/posthog/models/test/test_global_rate_limit_threshold_config.py b/posthog/models/test/test_global_rate_limit_threshold_config.py index 052f7388550d..25e343ffd78d 100644 --- a/posthog/models/test/test_global_rate_limit_threshold_config.py +++ b/posthog/models/test/test_global_rate_limit_threshold_config.py @@ -2,6 +2,7 @@ from posthog.test.base import BaseTest +from django.core.management import call_command from django.db import transaction from parameterized import parameterized @@ -40,6 +41,16 @@ def test_resolved_key_truncates_long_distinct_id(self): config = GlobalRateLimitThresholdConfig(token="phc_abc", distinct_id=long_distinct_id, threshold=10) self.assertEqual(config.resolved_key, f"phc_abc:{'d' * MAX_DISTINCT_ID_CHARS}") + def test_sync_command_bootstraps_empty_blob(self): + # An environment with zero rows never fires the signals, so the Redis + # key stays absent and capture polls not_found forever. The command + # writes the explicit empty blob, which capture loads as a valid map. + self.assertIsNone(self.redis_client.get(CUSTOM_THRESHOLDS_REDIS_KEY)) + + call_command("sync_global_rate_limit_thresholds") + + self.assertEqual(self._blob(), {}) + def test_post_save_writes_blob(self): with self.captureOnCommitCallbacks(execute=True): GlobalRateLimitThresholdConfig.objects.create(token="phc_abc", threshold=1000) diff --git a/rust/capture/src/config.rs b/rust/capture/src/config.rs index cbc0feca8c9c..22128c2e9a93 100644 --- a/rust/capture/src/config.rs +++ b/rust/capture/src/config.rs @@ -177,6 +177,13 @@ pub struct Config { #[envconfig(default = "4")] pub global_rate_limit_max_concurrent_commands: usize, + /// Max distinct (key, epoch) entries held in the deferred write batch per + /// limiter. Merges are always accepted; at the cap, updates for new keys + /// are dropped and counted (fail-open). Bounds limiter memory under + /// unique-key floods that outrun the per-tick write drain. + #[envconfig(default = "200000")] + pub global_rate_limit_max_write_batch_entries: usize, + /// How long a local cache entry survives regardless of access (seconds). /// Bounds how stale a key's cached count can be before it is rebuilt. #[envconfig(default = "600")] @@ -200,12 +207,30 @@ pub struct Config { #[envconfig(default = "250")] pub global_rate_limit_write_timeout_ms: u64, - // --- Token-only limiter config (not currently used in production, retained for new_token()) --- - /// Per-token rate limit threshold per window interval - /// Note: default is too high to trigger limiting in production - #[envconfig(default = "5000000")] + // --- Token-level (whole-token aggregate) limiter config --- + /// Enable the token-level limiter. Requires global_rate_limit_enabled. + /// Catches single-token floods that per-(token, distinct_id) keying cannot + /// see (identity rotation spreads volume so no per-identity key ever + /// approaches a threshold), and makes token-level dynamic overrides + /// enforceable as a whole-token aggregate. + #[envconfig(default = "false")] + pub global_rate_limit_token_enabled: bool, + + /// Per-token rate limit threshold per window interval. The default is a + /// flood backstop sized several times above the largest legitimate + /// per-token window volume observed in production; per-token clamps below + /// it belong in the dynamic override blob, not here. + #[envconfig(default = "40000000")] pub global_rate_limit_token_threshold: u64, + /// Sync floor for the token limiter (see global_rate_limit_min_sync_floor + /// for semantics). Token keys are few but individually high-volume, so a + /// much higher floor keeps Redis sync traffic to the handful of tokens + /// that could ever matter. The 1%-of-threshold cap in the limiter still + /// applies per key, so dynamic overrides below this stay enforceable. + #[envconfig(default = "10000")] + pub global_rate_limit_token_min_sync_floor: u64, + /// CSV list of key=value pairs for custom per-token thresholds pub global_rate_limit_token_overrides_csv: Option, diff --git a/rust/capture/src/events/analytics.rs b/rust/capture/src/events/analytics.rs index be2d111a9921..506c551d3669 100644 --- a/rust/capture/src/events/analytics.rs +++ b/rust/capture/src/events/analytics.rs @@ -249,6 +249,7 @@ pub async fn process_events( restriction_service: Option, historical_cfg: router::HistoricalConfig, global_rate_limiter: Option>, + global_rate_limiter_token: Option>, overflow_limiter: Option>, ai_events_overflow_limiter: Option>, ingestion_warning_emitter: Option>, @@ -269,6 +270,7 @@ pub async fn process_events( restriction_service, historical_cfg, global_rate_limiter, + global_rate_limiter_token, overflow_limiter, ai_events_overflow_limiter, ingestion_warning_emitter, @@ -290,6 +292,7 @@ async fn process_events_inner( restriction_service: Option, historical_cfg: router::HistoricalConfig, global_rate_limiter: Option>, + global_rate_limiter_token: Option>, overflow_limiter: Option>, ai_events_overflow_limiter: Option>, ingestion_warning_emitter: Option>, @@ -476,6 +479,32 @@ async fn process_events_inner( // Import is unaffected by both: the GRL never runs (guard below) and no // overflowable lane is reachable, so behavior is identical across paths. if context.capture_mode.applies_global_rate_limit() { + // Token-level aggregate first. Stamps only person processing and the + // overflow reroute -- events stay live, so the per-key loop below still + // consults the shared per-key limiter for every one of them, keeping + // v0/v1 per-key counts identical (see the invariant note above). + if let Some(ref limiter) = global_rate_limiter_token { + let event_count = events.len() as u64; + let cache_key = GlobalRateLimitKey::Token(&context.token).to_cache_key(); + if event_count > 0 && limiter.is_limited(&cache_key, event_count).await.is_some() { + for event in events.iter_mut() { + event.metadata.skip_person_processing = true; + if event.metadata.data_type == DataType::AnalyticsMain { + event.metadata.overflow_reason = Some(OverflowReason::ForceLimited); + } + } + counter!( + "capture_events_rate_limited_token", + "reason" => "global_rate_limit_token", + ) + .increment(event_count); + warn!( + token = context.token, + limited_event_count = event_count, + "events rate limited by token -- person processing disabled" + ); + } + } if let Some(ref limiter) = global_rate_limiter { let mut limited_distinct_ids: HashSet<&str> = HashSet::new(); let mut limited_event_count: u64 = 0; @@ -677,6 +706,7 @@ mod tests { restriction_service: Option, historical_cfg: router::HistoricalConfig, global_rate_limiter: Option>, + global_rate_limiter_token: Option>, overflow_limiter: Option>, ai_events_overflow_limiter: Option>, ingestion_warning_emitter: Option>, @@ -689,6 +719,7 @@ mod tests { restriction_service: None, historical_cfg: router::HistoricalConfig::new(false, 1), global_rate_limiter: None, + global_rate_limiter_token: None, overflow_limiter: None, ai_events_overflow_limiter: None, ingestion_warning_emitter: None, @@ -708,6 +739,7 @@ mod tests { options.restriction_service, options.historical_cfg, options.global_rate_limiter, + options.global_rate_limiter_token, options.overflow_limiter, options.ai_events_overflow_limiter, options.ingestion_warning_emitter, diff --git a/rust/capture/src/global_rate_limiter.rs b/rust/capture/src/global_rate_limiter.rs index 4b8e1abf3a1d..4ecccf0a6fc6 100644 --- a/rust/capture/src/global_rate_limiter.rs +++ b/rust/capture/src/global_rate_limiter.rs @@ -82,14 +82,26 @@ impl GlobalRateLimiter { .global_rate_limit_token_distinctid_overrides_csv .as_ref(), config.global_rate_limit_token_distinctid_local_cache_max_entries, + config.global_rate_limit_min_sync_floor, &prefix, &metrics_scope, config.global_rate_limit_custom_threshold_key.is_some(), ) } + /// Build the token-level rate limiter from the capture config, mirroring + /// `try_from_config`. Uses its own Redis connection off the same + /// configuration, so the two limiters' pipelines cannot head-of-line block + /// each other. + pub async fn try_token_from_config( + config: &Config, + shared_redis: Arc, + ) -> anyhow::Result { + let redis_client = Self::build_redis_client(config, shared_redis).await?; + Self::new_token(config, vec![redis_client]) + } + /// Create a per-token rate limiter sharing the given Redis instances. - /// Not currently wired into production call sites -- retained for future use. pub fn new_token( config: &Config, redis_instances: Vec>, @@ -102,12 +114,13 @@ impl GlobalRateLimiter { config.global_rate_limit_token_threshold, config.global_rate_limit_token_overrides_csv.as_ref(), config.global_rate_limit_token_local_cache_max_entries, + config.global_rate_limit_token_min_sync_floor, &prefix, &metrics_scope, - // The token-only limiter is not wired to the dynamic refresh source. - // (The hierarchical resolver is still set but is a no-op for bare - // token keys, which have no `:distinct_id` suffix.) - false, + // Same dynamic source as the tok_distid limiter: lookup keys here + // are bare tokens, so only the blob's token-level entries resolve + // (the hierarchical resolver's `:`-split fallback never fires). + config.global_rate_limit_custom_threshold_key.is_some(), ) } @@ -136,6 +149,7 @@ impl GlobalRateLimiter { threshold: u64, custom_keys_csv: Option<&String>, local_cache_max_entries: u64, + min_sync_floor: u64, redis_key_prefix: &str, metrics_scope: &str, enable_dynamic_source: bool, @@ -197,10 +211,11 @@ impl GlobalRateLimiter { ), local_cache_max_entries, metrics_scope: metrics_scope.to_string(), - min_sync_floor: config.global_rate_limit_min_sync_floor, + min_sync_floor, max_sync_keys_per_tick: config.global_rate_limit_max_sync_keys_per_tick, max_keys_per_command: config.global_rate_limit_max_keys_per_command, max_concurrent_commands: config.global_rate_limit_max_concurrent_commands, + max_write_batch_entries: config.global_rate_limit_max_write_batch_entries, global_read_timeout: Duration::from_millis(config.global_rate_limit_read_timeout_ms), global_write_timeout: Duration::from_millis(config.global_rate_limit_write_timeout_ms), local_cache_ttl: Duration::from_secs(config.global_rate_limit_local_cache_ttl_secs), diff --git a/rust/capture/src/overflow_parity.rs b/rust/capture/src/overflow_parity.rs index 9c9afcd9f868..6f0e5a622dfa 100644 --- a/rust/capture/src/overflow_parity.rs +++ b/rust/capture/src/overflow_parity.rs @@ -141,6 +141,7 @@ async fn run_v0(limits: Limits, batch_size: usize, observe: usize) -> Observed { None, HistoricalConfig::new(false, 1), global, + None, v0_overflow_limiter(limits), None, None, diff --git a/rust/capture/src/router.rs b/rust/capture/src/router.rs index f5da6cb0b9e5..3031a128b0ed 100644 --- a/rust/capture/src/router.rs +++ b/rust/capture/src/router.rs @@ -41,6 +41,7 @@ pub struct State { pub timesource: Arc, pub redis: Arc, pub global_rate_limiter_token_distinctid: Option>, + pub global_rate_limiter_token: Option>, pub quota_limiter: Arc, pub token_dropper: Arc, /// Restriction service scoped to all pipelines this capture deployment @@ -151,6 +152,7 @@ pub fn router, redis: Arc, global_rate_limiter_token_distinctid: Option>, + global_rate_limiter_token: Option>, quota_limiter: CaptureQuotaLimiter, token_dropper: TokenDropper, event_restriction_service: Option, @@ -181,6 +183,7 @@ pub fn router "token", + "outcome" => "allowed", + ) + .increment(ok_count); + return; + } + + let mut limited_count: u64 = 0; + let mut already_disabled_count: u64 = 0; + for event in events.iter_mut() { + if event.result != EventResult::Ok { + continue; + } + if event.force_disable_person_processing { + already_disabled_count += 1; + continue; + } + event.force_disable_person_processing = true; + if event.destination == Destination::AnalyticsMain { + event.destination = Destination::Overflow; + } + limited_count += 1; + } + + if limited_count > 0 { + metrics::counter!( + CAPTURE_V1_RATE_LIMITER, + "limiter" => "token", + "outcome" => "limited", + ) + .increment(limited_count); + crate::ctx_log!( + Level::WARN, + context, + limited_event_count = limited_count, + "events rate limited by token -- person processing disabled" + ); + } + if already_disabled_count > 0 { + metrics::counter!( + CAPTURE_V1_RATE_LIMITER, + "limiter" => "token", + "outcome" => "already_disabled", + ) + .increment(already_disabled_count); + } +} + async fn apply_token_distinct_id_limits( limiter: &GlobalRateLimiter, context: &RequestContext, @@ -2377,6 +2459,72 @@ mod tests { ctx } + #[tokio::test] + async fn token_limits_stamp_batch_and_never_drop() { + // Limited on the bare token key: every Ok event loses person + // processing, AnalyticsMain reroutes to overflow, nothing is dropped + // and `result` stays Ok so downstream stages still see live events. + let limiter = mock_limiter(vec!["phc_tok"]); + let ctx = td_context(); + let mut events = vec![ + wrapped_event("$pageview", "user-1"), + wrapped_event("$identify", "user-2"), + ]; + events[1].destination = Destination::AnalyticsHistorical; + + apply_token_limits(&limiter, &ctx, &mut events).await; + + for ev in &events { + assert_eq!(ev.result, EventResult::Ok); + assert!(ev.force_disable_person_processing); + } + assert_eq!(events[0].destination, Destination::Overflow); + assert_eq!( + events[1].destination, + Destination::AnalyticsHistorical, + "only AnalyticsMain may reroute to overflow" + ); + } + + #[tokio::test] + async fn token_allowed_batch_untouched() { + let limiter = mock_limiter(vec![]); + let ctx = td_context(); + let mut events = vec![wrapped_event("$pageview", "user-1")]; + + apply_token_limits(&limiter, &ctx, &mut events).await; + + assert_eq!(events[0].result, EventResult::Ok); + assert!(!events[0].force_disable_person_processing); + assert_eq!(events[0].destination, Destination::AnalyticsMain); + } + + #[tokio::test] + async fn token_limited_events_still_feed_per_key_limiter() { + // The v0/v1 invariant: both pipelines consult the shared per-key + // limiter for every non-dropped event. A token-limited batch must not + // vanish from the per-key counts -- the per-key loop still evaluates + // each event and reports it as already_disabled. + let token_limiter = mock_limiter(vec!["phc_tok"]); + let (per_key_limiter, calls) = mock_limiter_with_log(vec![]); + let ctx = td_context(); + let mut events = vec![ + wrapped_event("$pageview", "user-1"), + wrapped_event("$pageview", "user-2"), + ]; + + apply_token_limits(&token_limiter, &ctx, &mut events).await; + let tally = apply_token_distinct_id_limits(&per_key_limiter, &ctx, None, &mut events).await; + + assert_eq!( + calls.lock().unwrap().len(), + 2, + "per-key limiter must still count token-limited events" + ); + assert_eq!(tally.already_disabled, 2); + assert_eq!(tally.limited, 0); + } + #[tokio::test] async fn td_limits_under_limit_all_pass() { let limiter = mock_limiter(vec![]); diff --git a/rust/capture/src/v1/quota_limiter_shim.rs b/rust/capture/src/v1/quota_limiter_shim.rs index 454b59eb1148..091fa5092f91 100644 --- a/rust/capture/src/v1/quota_limiter_shim.rs +++ b/rust/capture/src/v1/quota_limiter_shim.rs @@ -135,11 +135,14 @@ mod tests { global_rate_limit_max_sync_keys_per_tick: 20_000, global_rate_limit_max_keys_per_command: 2_000, global_rate_limit_max_concurrent_commands: 4, + global_rate_limit_max_write_batch_entries: 200_000, global_rate_limit_local_cache_ttl_secs: 600, global_rate_limit_local_cache_idle_timeout_secs: 300, global_rate_limit_read_timeout_ms: 250, global_rate_limit_write_timeout_ms: 250, + global_rate_limit_token_enabled: false, global_rate_limit_token_threshold: 300_000, + global_rate_limit_token_min_sync_floor: 0, global_rate_limit_token_overrides_csv: None, global_rate_limit_token_local_cache_max_entries: 300_000, global_rate_limit_redis_url: None, diff --git a/rust/capture/src/v1/test_utils.rs b/rust/capture/src/v1/test_utils.rs index 012b89c58da8..6addc1c3b32c 100644 --- a/rust/capture/src/v1/test_utils.rs +++ b/rust/capture/src/v1/test_utils.rs @@ -701,6 +701,7 @@ pub struct TestStateBuilder { historical_threshold_days: Option, restriction_service: Option, global_rate_limiter: Option>, + global_rate_limiter_token: Option>, mock_producer: Option>, ai_gateway_signing_secret: Option, ingestion_warning_emitter: Option>, @@ -724,6 +725,7 @@ impl TestStateBuilder { historical_threshold_days: None, restriction_service: None, global_rate_limiter: None, + global_rate_limiter_token: None, mock_producer: None, ai_gateway_signing_secret: None, ingestion_warning_emitter: None, @@ -790,6 +792,11 @@ impl TestStateBuilder { } /// Add a global rate limiter. + pub fn with_global_rate_limiter_token(mut self, limiter: Arc) -> Self { + self.global_rate_limiter_token = Some(limiter); + self + } + pub fn with_global_rate_limiter(mut self, limiter: Arc) -> Self { self.global_rate_limiter = Some(limiter); self @@ -917,6 +924,7 @@ impl TestStateBuilder { timesource, redis, global_rate_limiter_token_distinctid: self.global_rate_limiter, + global_rate_limiter_token: self.global_rate_limiter_token, quota_limiter: Arc::new(quota_limiter), token_dropper: Arc::new(TokenDropper::default()), event_restriction_service: self.restriction_service, diff --git a/rust/capture/tests/common/integration_utils.rs b/rust/capture/tests/common/integration_utils.rs index 1a8164f210f2..4e80384a5e46 100644 --- a/rust/capture/tests/common/integration_utils.rs +++ b/rust/capture/tests/common/integration_utils.rs @@ -1098,6 +1098,7 @@ fn build_router_for_mode_at(mode: CaptureMode, fixed_time: &str) -> (Router, Mem Arc::new(sink.clone()), redis, None, // global_rate_limiter_token_distinctid + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service diff --git a/rust/capture/tests/common/utils.rs b/rust/capture/tests/common/utils.rs index 47b785a6afca..758eb37a76e8 100644 --- a/rust/capture/tests/common/utils.rs +++ b/rust/capture/tests/common/utils.rs @@ -53,11 +53,14 @@ pub static DEFAULT_CONFIG: Lazy = Lazy::new(|| Config { global_rate_limit_max_sync_keys_per_tick: 20_000, global_rate_limit_max_keys_per_command: 2_000, global_rate_limit_max_concurrent_commands: 4, + global_rate_limit_max_write_batch_entries: 200_000, global_rate_limit_local_cache_ttl_secs: 600, global_rate_limit_local_cache_idle_timeout_secs: 300, global_rate_limit_read_timeout_ms: 250, global_rate_limit_write_timeout_ms: 250, + global_rate_limit_token_enabled: false, global_rate_limit_token_threshold: 300_000, + global_rate_limit_token_min_sync_floor: 0, global_rate_limit_token_overrides_csv: None, global_rate_limit_token_local_cache_max_entries: 300_000, global_rate_limit_redis_url: None, diff --git a/rust/capture/tests/integration_ai_endpoint.rs b/rust/capture/tests/integration_ai_endpoint.rs index 465c96f1103f..856db24efb25 100644 --- a/rust/capture/tests/integration_ai_endpoint.rs +++ b/rust/capture/tests/integration_ai_endpoint.rs @@ -164,6 +164,7 @@ fn setup_ai_test_router() -> Router { Arc::new(sink), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service @@ -219,6 +220,7 @@ fn setup_ai_router_collecting_warnings() -> (Router, Arc) { Arc::new(TestSink), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service @@ -1287,6 +1289,7 @@ fn setup_ai_test_router_with_capturing_sink() -> (Router, CapturingSink) { Arc::new(sink), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service @@ -1955,6 +1958,7 @@ fn setup_ai_test_router_with_token_dropper(token_dropper: TokenDropper) -> (Rout Arc::new(sink), redis, None, + None, // global_rate_limiter_token quota_limiter, token_dropper, None, // event_restriction_service @@ -2166,6 +2170,7 @@ fn setup_ai_test_router_with_llm_quota_limited(token: &str) -> (Router, Capturin Arc::new(sink), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service @@ -2322,6 +2327,7 @@ fn setup_ai_test_router_with_overflow_limiter( Arc::new(sink), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service @@ -2462,6 +2468,7 @@ fn ai_router( Arc::new(sink), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, diff --git a/rust/capture/tests/integration_ai_restrictions.rs b/rust/capture/tests/integration_ai_restrictions.rs index 7af68e047b5f..9cfeb9919c68 100644 --- a/rust/capture/tests/integration_ai_restrictions.rs +++ b/rust/capture/tests/integration_ai_restrictions.rs @@ -161,6 +161,7 @@ async fn setup_ai_router_with_restriction( Arc::new(sink), redis, None, // global_rate_limiter_token_distinctid + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), Some(service), @@ -481,6 +482,7 @@ async fn setup_ai_router_with_redirect_to_topic( Arc::new(sink), redis, None, // global_rate_limiter_token_distinctid + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), Some(service), @@ -559,6 +561,7 @@ async fn setup_ai_router_with_force_overflow_and_limiter( Arc::new(sink), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), Some(service), diff --git a/rust/capture/tests/integration_analytics_ai_routing.rs b/rust/capture/tests/integration_analytics_ai_routing.rs index d09734773b0a..70c85d60d944 100644 --- a/rust/capture/tests/integration_analytics_ai_routing.rs +++ b/rust/capture/tests/integration_analytics_ai_routing.rs @@ -116,6 +116,7 @@ fn setup_router_for_mode( Arc::new(sink), redis, None, // global_rate_limiter_token_distinctid + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service diff --git a/rust/capture/tests/integration_analytics_restrictions.rs b/rust/capture/tests/integration_analytics_restrictions.rs index 71a0c3c1d38a..92d3ea5850ae 100644 --- a/rust/capture/tests/integration_analytics_restrictions.rs +++ b/rust/capture/tests/integration_analytics_restrictions.rs @@ -111,6 +111,7 @@ async fn setup_analytics_router_with_restriction( Arc::new(sink), redis, None, // global_rate_limiter_token_distinctid + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), Some(service), @@ -536,6 +537,7 @@ async fn setup_analytics_router_with_redirect_to_topic( Arc::new(sink), redis, None, // global_rate_limiter_token_distinctid + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), Some(service), diff --git a/rust/capture/tests/integration_otel_endpoint.rs b/rust/capture/tests/integration_otel_endpoint.rs index d5b5c8217693..31b0c330ec67 100644 --- a/rust/capture/tests/integration_otel_endpoint.rs +++ b/rust/capture/tests/integration_otel_endpoint.rs @@ -172,6 +172,7 @@ fn make_test_client_with_options(sink: &CapturingSink, options: TestClientOption Arc::new(sink.clone()), redis, None, // global_rate_limiter_token_distinctid + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), options.event_restriction_service, diff --git a/rust/capture/tests/integration_replay_restrictions.rs b/rust/capture/tests/integration_replay_restrictions.rs index 8a919d6b9089..200492ad8bfa 100644 --- a/rust/capture/tests/integration_replay_restrictions.rs +++ b/rust/capture/tests/integration_replay_restrictions.rs @@ -108,6 +108,7 @@ async fn setup_recordings_router_with_restriction( Arc::new(sink), redis, None, // global_rate_limiter_token_distinctid + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), Some(service), @@ -479,6 +480,7 @@ async fn setup_recordings_router_with_redirect_to_topic( Arc::new(sink), redis, None, // global_rate_limiter_token_distinctid + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), Some(service), diff --git a/rust/capture/tests/quota_limiters.rs b/rust/capture/tests/quota_limiters.rs index fe45243a07db..7a59614f7ec2 100644 --- a/rust/capture/tests/quota_limiters.rs +++ b/rust/capture/tests/quota_limiters.rs @@ -130,6 +130,7 @@ async fn setup_router_with_limits( Arc::new(sink.clone()), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service @@ -1183,6 +1184,7 @@ async fn test_survey_quota_cross_batch_first_submission_allowed() { Arc::new(sink.clone()), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service @@ -1274,6 +1276,7 @@ async fn test_survey_quota_cross_batch_duplicate_submission_dropped() { Arc::new(sink.clone()), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service @@ -1369,6 +1372,7 @@ async fn test_survey_quota_cross_batch_redis_error_fail_open() { Arc::new(sink.clone()), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service @@ -1801,6 +1805,7 @@ async fn test_ai_quota_cross_batch_redis_error_fail_open() { Arc::new(sink.clone()), redis, None, + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), None, // event_restriction_service diff --git a/rust/common/limiters/benches/global_rate_limiter.rs b/rust/common/limiters/benches/global_rate_limiter.rs index 9214ef1ee43e..deeac8ad0515 100644 --- a/rust/common/limiters/benches/global_rate_limiter.rs +++ b/rust/common/limiters/benches/global_rate_limiter.rs @@ -93,6 +93,7 @@ fn bench_config() -> GlobalRateLimiterConfig { max_sync_keys_per_tick: 100_000, max_keys_per_command: 2_000, max_concurrent_commands: 4, + max_write_batch_entries: 200_000, } } diff --git a/rust/common/limiters/src/global_rate_limiter.rs b/rust/common/limiters/src/global_rate_limiter.rs index 5d3388edbfe3..7db647fbe583 100644 --- a/rust/common/limiters/src/global_rate_limiter.rs +++ b/rust/common/limiters/src/global_rate_limiter.rs @@ -217,6 +217,13 @@ pub struct GlobalRateLimiterConfig { /// How many chunked commands may be in flight at once against one instance. /// Trades tick wall-clock against instantaneous Redis load. pub max_concurrent_commands: usize, + /// Maximum distinct (key, epoch) entries held in the deferred write batch. + /// Merges into existing entries are always accepted (they add no memory); + /// at the cap, updates for new keys are dropped and counted. Without this, + /// unique-key inflow faster than the per-tick drain grows the batch without + /// bound -- the update channel's capacity does not help, because the + /// receiver moves entries into this map as fast as they arrive. + pub max_write_batch_entries: usize, /// Per-key custom limits. Overrides the default limit for specific *more granular* keys. /// /// Wrapped in `Arc>` so the map can be atomically replaced at @@ -289,6 +296,7 @@ impl Default for GlobalRateLimiterConfig { max_sync_keys_per_tick: 20_000, max_keys_per_command: 2_000, max_concurrent_commands: 4, + max_write_batch_entries: 200_000, custom_keys: Arc::new(ArcSwap::from_pointee(HashMap::new())), custom_key_resolver: None, custom_key_source: None, @@ -855,7 +863,14 @@ impl GlobalRateLimiterImpl { match result { Some(req) => { let epoch = epoch_from_timestamp(req.timestamp, config.window_interval); - *write_batch.entry((req.key, epoch)).or_insert(0) += req.count; + Self::absorb_update( + &mut write_batch, + req.key, + epoch, + req.count, + config.max_write_batch_entries, + scope, + ); } None => { // Channel closed, do final flush and exit @@ -881,6 +896,43 @@ impl GlobalRateLimiterImpl { }); } + /// Merge one update into the deferred write batch, enforcing the entry cap. + /// + /// Merges never grow the map, so they are always accepted; only a brand-new + /// (key, epoch) entry can be refused. A refused update undercounts the + /// global tally for that key -- under-enforcement, consistent with every + /// other overload path here failing open -- and is counted so the loss is + /// visible. No log line: at the inflow rates that reach the cap, per-drop + /// logging would itself be a problem. + fn absorb_update( + write_batch: &mut HashMap<(String, i64), u64>, + key: String, + epoch: i64, + count: u64, + max_entries: usize, + scope: &'static str, + ) { + let at_cap = write_batch.len() >= max_entries; + match write_batch.entry((key, epoch)) { + std::collections::hash_map::Entry::Occupied(mut entry) => { + *entry.get_mut() += count; + } + std::collections::hash_map::Entry::Vacant(slot) => { + if at_cap { + metrics::counter!( + GLOBAL_RATE_LIMITER_ERROR_COUNTER, + "scope" => scope, + "step" => "enqueue_update", + "cause" => "write_batch_full", + ) + .increment(1); + } else { + slot.insert(count); + } + } + } + } + /// Execute one tick of the background pipeline. /// /// Drains pending reads + writes, builds a single pipeline, executes it, @@ -916,6 +968,24 @@ impl GlobalRateLimiterImpl { metrics::gauge!(GLOBAL_RATE_LIMITER_SYNC_DEFERRED_GAUGE, "scope" => scope) .set(pending_sync.len() as f64); + // Deferred entries whose epoch has aged out of the readable window can + // no longer affect any decision: reads consult only the current and + // previous epochs. Purge them instead of spending write commands (and + // deferral slots) on counts nothing will ever read. + let min_live_epoch = epoch_from_timestamp(Utc::now(), config.window_interval) - 1; + let before_purge = write_batch.len(); + write_batch.retain(|(_, epoch), _| *epoch >= min_live_epoch); + let purged = before_purge - write_batch.len(); + if purged > 0 { + metrics::counter!( + GLOBAL_RATE_LIMITER_ERROR_COUNTER, + "scope" => scope, + "step" => "pipeline", + "cause" => "stale_epoch_purged", + ) + .increment(purged as u64); + } + // Bound the write drain the same way. The deferred remainder stays in // `write_batch`, where new arrivals merge into it by (key, epoch), so no // count is lost -- it lands in the same epoch key up to a few ticks late. @@ -1071,6 +1141,12 @@ impl GlobalRateLimiterImpl { Ok(Err(e)) => { Self::record_pipeline_error(scope, &redis_idx_str, "redis_write"); warn!(error = %e, records = chunk_len, "Failed to write rate limit batch to Redis"); + // A dead MultiplexedConnection never recovers on its + // own; ask the client to rebuild. Timeouts are + // transient and never route here. + if e.is_unrecoverable_error() { + redis.heal().await; + } } Err(_) => { Self::record_pipeline_error(scope, &redis_idx_str, "write_timeout"); @@ -1146,6 +1222,9 @@ impl GlobalRateLimiterImpl { Ok(Err(e)) => { Self::record_pipeline_error(scope, &redis_idx_str, "redis_error"); warn!(keys = chunk.len(), error = %e, "Failed to read rate limits from Redis"); + if e.is_unrecoverable_error() { + redis.heal().await; + } } Err(_) => { Self::record_pipeline_error(scope, &redis_idx_str, "read_timeout"); @@ -1379,6 +1458,7 @@ mod tests { max_sync_keys_per_tick: 20_000, max_keys_per_command: 2_000, max_concurrent_commands: 4, + max_write_batch_entries: 200_000, } } @@ -1555,6 +1635,7 @@ mod tests { assert_eq!(config.max_sync_keys_per_tick, 20_000); assert_eq!(config.max_keys_per_command, 2_000); assert_eq!(config.max_concurrent_commands, 4); + assert_eq!(config.max_write_batch_entries, 200_000); assert!(config.custom_keys.load().is_empty()); assert!(config.custom_key_resolver.is_none()); assert_eq!(config.metrics_scope, "default"); @@ -2059,7 +2140,9 @@ mod tests { let pending: Arc> = Arc::new(DashSet::new()); pending.insert("read_key".to_string()); let mut writes: HashMap<(String, i64), u64> = HashMap::new(); - writes.insert(("write_key".to_string(), 1), 5); + // Current epoch: a stale epoch would be purged before the write runs. + let epoch = epoch_from_timestamp(Utc::now(), config.window_interval); + writes.insert(("write_key".to_string(), epoch), 5); GlobalRateLimiterImpl::tick( &config, @@ -2087,6 +2170,69 @@ mod tests { ); } + #[tokio::test] + async fn test_write_batch_cap_drops_new_keys_but_merges_existing() { + // At the cap, an update for a brand-new key is dropped (bounded memory + // beats an unbounded map under unique-key floods), while an update for + // a key already in the batch still merges -- merging costs no memory + // and dropping it would silently undercount a key we are tracking. + let mut batch: HashMap<(String, i64), u64> = HashMap::new(); + batch.insert(("k1".to_string(), 1), 5); + batch.insert(("k2".to_string(), 1), 5); + + GlobalRateLimiterImpl::absorb_update(&mut batch, "k3".to_string(), 1, 7, 2, "test"); + assert_eq!(batch.len(), 2, "new key at cap must be dropped"); + assert!(!batch.contains_key(&("k3".to_string(), 1))); + + GlobalRateLimiterImpl::absorb_update(&mut batch, "k1".to_string(), 1, 7, 2, "test"); + assert_eq!( + batch.get(&("k1".to_string(), 1)), + Some(&12), + "existing key at cap must still merge" + ); + } + + #[tokio::test] + async fn test_tick_purges_stale_epochs_instead_of_writing_them() { + let mock = Arc::new(MockRedisClient::new()); + let client: Arc = mock.clone(); + let config = config_with_floor(0); // 60s window + let cache: Cache = Cache::builder().max_capacity(100).build(); + let pending: Arc> = Arc::new(DashSet::new()); + + let current_epoch = epoch_from_timestamp(Utc::now(), config.window_interval); + let mut writes: HashMap<(String, i64), u64> = HashMap::new(); + writes.insert(("live".to_string(), current_epoch), 1); + writes.insert(("stale".to_string(), current_epoch - 5), 1); + + GlobalRateLimiterImpl::tick( + &config, + std::slice::from_ref(&client), + &cache, + &pending, + &mut writes, + "test", + 1, + ) + .await; + + let write_calls: Vec = mock + .get_calls() + .into_iter() + .filter(|c| c.op == "batch_incr_by_expire") + .map(|c| c.key) + .collect(); + assert_eq!( + write_calls, + vec![format!("items=1;ttl={}", config.global_cache_ttl.as_secs())], + "only the readable-epoch entry may be written; a stale epoch can never be read (reads consult current + previous only) and must not spend write commands" + ); + assert!( + writes.is_empty(), + "stale entry must be purged, not deferred" + ); + } + #[tokio::test] async fn test_tick_bounds_write_drain_and_carries_remainder() { let client: Arc = Arc::new(MockRedisClient::new()); @@ -2097,8 +2243,9 @@ mod tests { let cache: Cache = Cache::builder().max_capacity(100).build(); let pending: Arc> = Arc::new(DashSet::new()); let mut writes: HashMap<(String, i64), u64> = HashMap::new(); + let epoch = epoch_from_timestamp(Utc::now(), config.window_interval); for i in 0..25 { - writes.insert((format!("w{i}"), 1), 1); + writes.insert((format!("w{i}"), epoch), 1); } GlobalRateLimiterImpl::tick( diff --git a/rust/common/limiters/tests/global_rate_limiter_integration_tests.rs b/rust/common/limiters/tests/global_rate_limiter_integration_tests.rs index ac9623614e28..002c8c799170 100644 --- a/rust/common/limiters/tests/global_rate_limiter_integration_tests.rs +++ b/rust/common/limiters/tests/global_rate_limiter_integration_tests.rs @@ -61,6 +61,7 @@ fn test_config(test_name: &str) -> GlobalRateLimiterConfig { max_sync_keys_per_tick: 10_000, max_keys_per_command: 2_000, max_concurrent_commands: 4, + max_write_batch_entries: 200_000, } } diff --git a/rust/common/redis/Cargo.toml b/rust/common/redis/Cargo.toml index 1499ef6cd2fb..0312492ca7aa 100644 --- a/rust/common/redis/Cargo.toml +++ b/rust/common/redis/Cargo.toml @@ -7,6 +7,8 @@ edition = "2021" workspace = true [dependencies] +arc-swap = { workspace = true } +tokio = { workspace = true } async-trait = { workspace = true } # `futures-timer` (not `tokio::time::sleep`) lets the mock module simulate # blocking pipeline calls without making `tokio` a runtime dependency of every diff --git a/rust/common/redis/src/client.rs b/rust/common/redis/src/client.rs index 7dbfcef3814b..6f7e4d5d74a5 100644 --- a/rust/common/redis/src/client.rs +++ b/rust/common/redis/src/client.rs @@ -1,8 +1,11 @@ +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use arc_swap::ArcSwap; use async_trait::async_trait; use redis::aio::MultiplexedConnection; use redis::{AsyncCommands, RedisError}; -use std::time::Duration; -use tracing::warn; +use tracing::{info, warn}; use crate::pipeline::{PipelineCommand, PipelineResult}; use crate::{Client, CompressionConfig, CustomRedisError, RedisValueFormat}; @@ -14,11 +17,72 @@ const ERR_RAWBYTES_SET: &str = #[derive(Clone)] pub struct RedisClient { - connection: MultiplexedConnection, + /// Shared across clones so a `heal()` on any handle repairs all of them. + /// `MultiplexedConnection` does not reconnect after its TCP connection + /// dies; `heal()` swaps in a rebuilt one. + connection: Arc>, + /// Connection info retained so `heal()` can rebuild. + client: redis::Client, + response_timeout: Option, + connection_timeout: Option, + /// Serializes heal attempts and carries the last-attempt time for the + /// cooldown, so an error burst cannot stampede reconnects. + heal_state: Arc>, compression: CompressionConfig, format: RedisValueFormat, } +/// Minimum time between reconnect attempts (see `RedisClient::heal_connection`). +const HEAL_COOLDOWN: Duration = Duration::from_secs(5); + +impl RedisClient { + /// Current connection handle. Cheap: one atomic load plus a + /// `MultiplexedConnection` clone (an mpsc sender clone). + fn conn(&self) -> MultiplexedConnection { + self.connection.load().as_ref().clone() + } + + /// Rebuild the underlying connection after it has died. + /// + /// `MultiplexedConnection` never reconnects on its own: once its TCP + /// connection drops (Redis failover, node replacement), every command + /// errors forever. Callers that detect an unrecoverable error + /// (`CustomRedisError::is_unrecoverable_error`) call this to swap in a + /// fresh connection; all clones of this client share the swap. Attempts + /// are serialized and rate-limited by `HEAL_COOLDOWN`, and a failed + /// attempt just waits for the next caller -- the client keeps failing + /// open in the meantime, exactly as it would without healing. + pub async fn heal_connection(&self) { + let mut last_attempt = self.heal_state.lock().await; + if last_attempt.elapsed() < HEAL_COOLDOWN { + return; + } + *last_attempt = Instant::now(); + + let mut config = redis::AsyncConnectionConfig::new(); + if let Some(timeout) = self.response_timeout { + config = config.set_response_timeout(timeout); + } + if let Some(timeout) = self.connection_timeout { + config = config.set_connection_timeout(timeout); + } + + match self + .client + .get_multiplexed_async_connection_with_config(&config) + .await + { + Ok(connection) => { + self.connection.store(Arc::new(connection)); + info!("Redis connection healed after unrecoverable error"); + } + Err(e) => { + warn!(error = %e, "Redis heal attempt failed; will retry after cooldown"); + } + } + } +} + impl RedisClient { /// Create a new RedisClient with default settings /// @@ -147,7 +211,11 @@ impl RedisClient { .await?; Ok(RedisClient { - connection, + connection: Arc::new(ArcSwap::from_pointee(connection)), + client, + response_timeout, + connection_timeout, + heal_state: Arc::new(tokio::sync::Mutex::new(Instant::now() - HEAL_COOLDOWN)), compression, format, }) @@ -239,7 +307,7 @@ impl RedisClient { for arg in args { invocation.arg(arg); } - let mut conn = self.connection.clone(); + let mut conn = self.conn(); let result: Vec = invocation.invoke_async(&mut conn).await?; Ok(result) } @@ -247,25 +315,29 @@ impl RedisClient { #[async_trait] impl Client for RedisClient { + async fn heal(&self) { + self.heal_connection().await; + } + async fn zrangebyscore( &self, k: String, min: String, max: String, ) -> Result, CustomRedisError> { - let mut conn = self.connection.clone(); + let mut conn = self.conn(); let results = conn.zrangebyscore(k, min, max).await?; Ok(results) } async fn zadd(&self, k: String, member: String, score: i64) -> Result<(), CustomRedisError> { - let mut conn = self.connection.clone(); + let mut conn = self.conn(); conn.zadd::<_, _, _, ()>(k, member, score).await?; Ok(()) } async fn hincrby(&self, k: String, v: String, count: i64) -> Result<(), CustomRedisError> { - let mut conn = self.connection.clone(); + let mut conn = self.conn(); conn.hincr::<_, _, _, ()>(k, v, count).await?; Ok(()) } @@ -279,7 +351,7 @@ impl Client for RedisClient { k: String, format: RedisValueFormat, ) -> Result { - let mut conn = self.connection.clone(); + let mut conn = self.conn(); let raw_bytes: Vec = conn.get(k).await?; // return NotFound error when empty @@ -308,7 +380,7 @@ impl Client for RedisClient { } async fn get_raw_bytes(&self, k: String) -> Result, CustomRedisError> { - let mut conn = self.connection.clone(); + let mut conn = self.conn(); let raw_bytes: Vec = conn.get(k).await?; // return NotFound error when empty @@ -327,7 +399,7 @@ impl Client for RedisClient { v: Vec, ttl_seconds: Option, ) -> Result<(), CustomRedisError> { - let mut conn = self.connection.clone(); + let mut conn = self.conn(); match ttl_seconds { Some(ttl) => conn.set_ex::<_, _, ()>(k, v, ttl).await?, None => conn.set::<_, _, ()>(k, v).await?, @@ -347,7 +419,7 @@ impl Client for RedisClient { ) -> Result<(), CustomRedisError> { let final_bytes = self.serialize_and_compress(v, format)?; - let mut conn = self.connection.clone(); + let mut conn = self.conn(); conn.set::<_, _, ()>(k, final_bytes).await?; Ok(()) } @@ -365,7 +437,7 @@ impl Client for RedisClient { ) -> Result<(), CustomRedisError> { let final_bytes = self.serialize_and_compress(v, format)?; - let mut conn = self.connection.clone(); + let mut conn = self.conn(); conn.set_ex::<_, _, ()>(k, final_bytes, seconds).await?; Ok(()) } @@ -388,7 +460,7 @@ impl Client for RedisClient { ) -> Result { let final_bytes = self.serialize_and_compress(v, format)?; - let mut conn = self.connection.clone(); + let mut conn = self.conn(); let seconds_usize = seconds as usize; // Use SET with both NX and EX options @@ -423,7 +495,7 @@ impl Client for RedisClient { .ignore(); } - let mut conn = self.connection.clone(); + let mut conn = self.conn(); pipe.query_async::<()>(&mut conn).await?; Ok(()) } @@ -439,19 +511,19 @@ impl Client for RedisClient { pipe.cmd("EXPIRE").arg(&k).arg(ttl_seconds).ignore(); } - let mut conn = self.connection.clone(); + let mut conn = self.conn(); pipe.query_async::<()>(&mut conn).await?; Ok(()) } async fn del(&self, k: String) -> Result<(), CustomRedisError> { - let mut conn = self.connection.clone(); + let mut conn = self.conn(); conn.del::<_, ()>(k).await?; Ok(()) } async fn hget(&self, k: String, field: String) -> Result { - let mut conn = self.connection.clone(); + let mut conn = self.conn(); let result: Option = conn.hget(k, field).await?; match result { @@ -461,7 +533,7 @@ impl Client for RedisClient { } async fn scard(&self, k: String) -> Result { - let mut conn = self.connection.clone(); + let mut conn = self.conn(); let result = conn.scard(k).await?; Ok(result) } @@ -470,7 +542,7 @@ impl Client for RedisClient { if keys.is_empty() { return Ok(vec![]); } - let mut conn = self.connection.clone(); + let mut conn = self.conn(); let results: Vec>> = conn.mget(&keys).await?; Ok(results) } @@ -483,7 +555,7 @@ impl Client for RedisClient { for k in &keys { pipe.scard(k); } - let mut conn = self.connection.clone(); + let mut conn = self.conn(); let results: Vec = pipe.query_async(&mut conn).await?; Ok(results) } @@ -505,7 +577,7 @@ impl Client for RedisClient { .arg("NX") .ignore(); } - let mut conn = self.connection.clone(); + let mut conn = self.conn(); pipe.query_async::<()>(&mut conn).await?; Ok(()) } @@ -521,7 +593,7 @@ impl Client for RedisClient { for (k, v, ttl) in &items { pipe.cmd("SET").arg(k).arg(v).arg("NX").arg("EX").arg(ttl); } - let mut conn = self.connection.clone(); + let mut conn = self.conn(); let results: Vec> = pipe.query_async(&mut conn).await?; Ok(results.into_iter().map(|r| r.is_some()).collect()) } @@ -530,7 +602,7 @@ impl Client for RedisClient { if keys.is_empty() { return Ok(()); } - let mut conn = self.connection.clone(); + let mut conn = self.conn(); redis::cmd("DEL") .arg(&keys) .query_async::<()>(&mut conn) @@ -613,7 +685,7 @@ impl Client for RedisClient { } // Execute the pipeline - let mut conn = self.connection.clone(); + let mut conn = self.conn(); let raw_results: Vec = pipe.query_async(&mut conn).await?; // Process results diff --git a/rust/common/redis/src/lib.rs b/rust/common/redis/src/lib.rs index be2e60e17893..a1867aec9748 100644 --- a/rust/common/redis/src/lib.rs +++ b/rust/common/redis/src/lib.rs @@ -290,6 +290,14 @@ pub trait Client: Send + Sync { &self, commands: Vec, ) -> Result>, CustomRedisError>; + + /// Attempt to repair a dead underlying connection. + /// + /// Callers that see `CustomRedisError::is_unrecoverable_error()` may call + /// this; implementations that self-heal (or have nothing to heal) keep the + /// default no-op. Must be cheap to call repeatedly -- implementations own + /// their own cooldown. + async fn heal(&self) {} } /// Extension trait providing the `.pipeline()` builder method. diff --git a/rust/common/redis/src/read_write.rs b/rust/common/redis/src/read_write.rs index 6c0c709a8c72..17db25225a6f 100644 --- a/rust/common/redis/src/read_write.rs +++ b/rust/common/redis/src/read_write.rs @@ -253,6 +253,11 @@ impl ReadWriteClient { #[async_trait] impl Client for ReadWriteClient { + async fn heal(&self) { + self.reader.heal().await; + self.writer.heal().await; + } + async fn get(&self, k: String) -> Result { match self.reader.get(k.clone()).await { Ok(value) => Ok(value), From 9d04ed538602c44e47f3579cb699487e936d7569 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Fri, 14 Aug 2026 12:16:07 -0400 Subject: [PATCH 027/289] chore(capture): commit lockfile for common-redis deps Co-Authored-By: Claude Fable 5 --- rust/Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index a194f4063932..665878afbfb6 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2913,6 +2913,7 @@ dependencies = [ name = "common-redis" version = "0.1.0" dependencies = [ + "arc-swap", "async-trait", "futures-timer", "redis", From 6eb33520956463c05f3bfd3d025478f5d11fc701 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 14 Aug 2026 17:58:33 +0100 Subject: [PATCH 028/289] docs(ci): add hobby smoke test skill Generated-By: PostHog Desktop Task-Id: 3b2cd40e-fad1-41a2-bf15-419b22c0f84a --- .../extending-hobby-smoke-tests/SKILL.md | 102 ++++++++++++++++++ .../agents/openai.yaml | 4 + 2 files changed, 106 insertions(+) create mode 100644 products/engineering_analytics/skills/extending-hobby-smoke-tests/SKILL.md create mode 100644 products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml diff --git a/products/engineering_analytics/skills/extending-hobby-smoke-tests/SKILL.md b/products/engineering_analytics/skills/extending-hobby-smoke-tests/SKILL.md new file mode 100644 index 000000000000..f1edef0fd458 --- /dev/null +++ b/products/engineering_analytics/skills/extending-hobby-smoke-tests/SKILL.md @@ -0,0 +1,102 @@ +--- +name: extending-hobby-smoke-tests +description: Design, extend, review, or debug PostHog Hobby end-to-end smoke tests in bin/hobby-ci.py and .github/workflows/ci-hobby.yml. Use when adding an ingestion round trip, deciding whether a product belongs in Hobby CI, changing the CI Hobby service topology or API-key scopes, or diagnosing a smoke test that captures data but cannot query it. +--- + +# Extending Hobby smoke tests + +Treat Hobby CI as proof that a supported Hobby install works across real process boundaries. Keep each check small, strong, and limited to a stable product surface. + +## Decide whether the check belongs + +Add a check only when all of these are true: + +- The product is supported for Hobby deployments and is no longer alpha. +- A break can leave the install apparently healthy while the product is unusable. +- The check crosses boundaries that unit or service integration tests cannot cover, such as capture, queue, consumer, storage, and query API. +- The released Hobby images and default compose topology contain every required service. +- A deterministic request and an exact read-back assertion are available. + +Do not add the check when it requires a private feature flag, a CI-only service topology, or a different image registry only to make an alpha path available. Test that path at a lower layer until it becomes part of the supported Hobby install. + +If the check exposes a missing service or configuration that every supported Hobby install needs, fix the install and add the check together. If the missing plumbing exists only for the proposed test, stop and reconsider the check. + +## Map the round trip before editing + +Write down this chain from repository evidence: + +```text +public ingest endpoint -> request contract -> service/consumer -> storage -> read API -> required scope +``` + +Verify each link: + +1. Find an existing end-to-end or receiver fixture for the ingest payload. Reuse its envelope and minimum valid data instead of inventing a plausible payload. +2. Locate the consumer command and confirm it exists in the released image used by `docker-compose.hobby.yml`. +3. Confirm the consumer is already started by the default Hobby compose files. +4. Confirm no unreleased or private feature flag is required. +5. Find the supported read API and its personal API-key scope. +6. Identify a unique value that can select only the captured object. + +Do this before starting a full Hobby run. A successful HTTP capture response proves receipt, not ingestion. + +## Build the smallest strong check + +Change `bin/hobby-ci.py` for the round trip and `bin/hobby-ci-setup-user.py` only for the least read scope needed. + +- Generate collision-resistant identifiers with UUIDs or nanosecond timestamps. +- Record the query window before capture. +- Send the smallest payload known to reach storage. +- Fail immediately on a non-success capture response and include a short response body. +- Poll the supported read API with the exact identifier. +- Require the expected stored object or value. An HTTP 200 with an empty result is not success. +- Use the existing bounded timeout and polling style. +- Preserve earlier smoke checks and return a result that names every completed round trip. +- Keep one product idea per PR when the checks can fail independently. + +Avoid adding general abstractions for a single payload. Extract a helper only when it removes real repetition or gives a concept a useful name. + +## Validate from cheap to expensive + +Run focused checks before asking Hobby CI to create a server: + +```bash +python3 -m py_compile bin/hobby-ci.py bin/hobby-ci-setup-user.py +ruff check bin/hobby-ci.py bin/hobby-ci-setup-user.py +ruff format --check bin/hobby-ci.py bin/hobby-ci-setup-user.py +git diff --check +``` + +When compose or installer files change, also render the final compose configuration and run the focused installer tests. Inspect the rendered image and command for every added service. + +Use a PR-specific image only when the PR changes code that must be built into that image. Do not change workflow path filters or registries merely because a smoke-test-only PR needs an unreleased service. + +Then run Hobby CI once and follow the exact run through image build, cloud setup, health, and ingestion. Do not restart a healthy migration phase just because it is slow. + +## Read failures by boundary + +Use the first failed boundary to choose the next investigation: + +| Evidence | Likely boundary | +| --- | --- | +| Capture returns 4xx | Endpoint, token, or payload envelope | +| Read API returns 401 or 403 | Personal key scope or feature access | +| Capture succeeds, exact query stays empty | Payload semantics, missing consumer, routing, or storage | +| Added container is absent or unhealthy | Released image or default compose topology | +| Query returns 200 with empty series | Not success; keep polling or strengthen the assertion | +| Earlier product checks fail too | Shared install or trunk failure, not the new assertion alone | + +Pull the failed job log before editing. Confirm the hypothesis against the receiver fixture, consumer registration, compose rendering, and API implementation. Do not run another full deployment on a guessed payload. + +## Review the final diff + +Before publishing, prove the PR contains only what the supported round trip requires: + +- No alpha-only feature flags. +- No registry or image-build changes unless product code in the PR requires a new image. +- No new service unless that service belongs in every supported Hobby install. +- No broad API scope. +- No assertion that accepts an empty response. +- No synthetic payload that lacks a known accepted fixture. + +Update the PR description with the exact ingest and read-back proof. State any product intentionally excluded because it is not yet stable. diff --git a/products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml b/products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml new file mode 100644 index 000000000000..28d9ae2b9a3b --- /dev/null +++ b/products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Extend Hobby Smoke Tests" + short_description: "Design reliable Hobby ingestion smoke tests" + default_prompt: "Use $extending-hobby-smoke-tests to decide whether and how to add a Hobby ingestion smoke test." From 911c799c34a10df564c3f62daff7073c3244abc4 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 14 Aug 2026 18:14:19 +0100 Subject: [PATCH 029/289] fix(ci): format hobby smoke test skill metadata Generated-By: PostHog Desktop Task-Id: 3b2cd40e-fad1-41a2-bf15-419b22c0f84a --- .../skills/extending-hobby-smoke-tests/agents/openai.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml b/products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml index 28d9ae2b9a3b..7f9c1d2aafcd 100644 --- a/products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml +++ b/products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml @@ -1,4 +1,4 @@ interface: - display_name: "Extend Hobby Smoke Tests" - short_description: "Design reliable Hobby ingestion smoke tests" - default_prompt: "Use $extending-hobby-smoke-tests to decide whether and how to add a Hobby ingestion smoke test." + display_name: 'Extend Hobby Smoke Tests' + short_description: 'Design reliable Hobby ingestion smoke tests' + default_prompt: 'Use $extending-hobby-smoke-tests to decide whether and how to add a Hobby ingestion smoke test.' From dcf27320efb2fe821088507b246a619cb1652565 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 14 Aug 2026 18:27:33 +0100 Subject: [PATCH 030/289] fix(ci): format hobby smoke test skill Generated-By: PostHog Desktop Task-Id: 3b2cd40e-fad1-41a2-bf15-419b22c0f84a --- .../skills/extending-hobby-smoke-tests/SKILL.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/products/engineering_analytics/skills/extending-hobby-smoke-tests/SKILL.md b/products/engineering_analytics/skills/extending-hobby-smoke-tests/SKILL.md index f1edef0fd458..f975cee746b3 100644 --- a/products/engineering_analytics/skills/extending-hobby-smoke-tests/SKILL.md +++ b/products/engineering_analytics/skills/extending-hobby-smoke-tests/SKILL.md @@ -77,14 +77,14 @@ Then run Hobby CI once and follow the exact run through image build, cloud setup Use the first failed boundary to choose the next investigation: -| Evidence | Likely boundary | -| --- | --- | -| Capture returns 4xx | Endpoint, token, or payload envelope | -| Read API returns 401 or 403 | Personal key scope or feature access | -| Capture succeeds, exact query stays empty | Payload semantics, missing consumer, routing, or storage | -| Added container is absent or unhealthy | Released image or default compose topology | -| Query returns 200 with empty series | Not success; keep polling or strengthen the assertion | -| Earlier product checks fail too | Shared install or trunk failure, not the new assertion alone | +| Evidence | Likely boundary | +| ----------------------------------------- | ------------------------------------------------------------ | +| Capture returns 4xx | Endpoint, token, or payload envelope | +| Read API returns 401 or 403 | Personal key scope or feature access | +| Capture succeeds, exact query stays empty | Payload semantics, missing consumer, routing, or storage | +| Added container is absent or unhealthy | Released image or default compose topology | +| Query returns 200 with empty series | Not success; keep polling or strengthen the assertion | +| Earlier product checks fail too | Shared install or trunk failure, not the new assertion alone | Pull the failed job log before editing. Confirm the hypothesis against the receiver fixture, consumer registration, compose rendering, and API implementation. Do not run another full deployment on a guessed payload. From 13551357a7b7f5500bce246001328fc0ad532af3 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 14 Aug 2026 18:48:03 +0100 Subject: [PATCH 031/289] fix(ci): address hobby smoke test skill review Generated-By: PostHog Desktop Task-Id: 3b2cd40e-fad1-41a2-bf15-419b22c0f84a --- .../skills/extending-hobby-smoke-tests/SKILL.md | 0 .github/workflows/ci-hobby.yml | 1 + .github/workflows/container-images-ci.yml | 1 + .../skills/extending-hobby-smoke-tests/agents/openai.yaml | 4 ---- 4 files changed, 2 insertions(+), 4 deletions(-) rename {products/engineering_analytics => .agents}/skills/extending-hobby-smoke-tests/SKILL.md (100%) delete mode 100644 products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml diff --git a/products/engineering_analytics/skills/extending-hobby-smoke-tests/SKILL.md b/.agents/skills/extending-hobby-smoke-tests/SKILL.md similarity index 100% rename from products/engineering_analytics/skills/extending-hobby-smoke-tests/SKILL.md rename to .agents/skills/extending-hobby-smoke-tests/SKILL.md diff --git a/.github/workflows/ci-hobby.yml b/.github/workflows/ci-hobby.yml index 6d42f0a38f01..54e082191cfa 100644 --- a/.github/workflows/ci-hobby.yml +++ b/.github/workflows/ci-hobby.yml @@ -60,6 +60,7 @@ jobs: - docker-compose.hobby.yml # Hobby-specific scripts - 'bin/deploy-hobby' + - 'bin/hobby-ci-setup-user.py' - 'bin/hobby-ci.py' - 'bin/upgrade-hobby' - 'bin/migrate-*-hobby' diff --git a/.github/workflows/container-images-ci.yml b/.github/workflows/container-images-ci.yml index dc614d61ff0d..a91aa5ab8ce9 100644 --- a/.github/workflows/container-images-ci.yml +++ b/.github/workflows/container-images-ci.yml @@ -59,6 +59,7 @@ jobs: - docker-compose.base.yml - docker-compose.hobby.yml - 'bin/deploy-hobby' + - 'bin/hobby-ci-setup-user.py' - 'bin/hobby-ci.py' - 'bin/upgrade-hobby' - 'bin/migrate-*-hobby' diff --git a/products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml b/products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml deleted file mode 100644 index 7f9c1d2aafcd..000000000000 --- a/products/engineering_analytics/skills/extending-hobby-smoke-tests/agents/openai.yaml +++ /dev/null @@ -1,4 +0,0 @@ -interface: - display_name: 'Extend Hobby Smoke Tests' - short_description: 'Design reliable Hobby ingestion smoke tests' - default_prompt: 'Use $extending-hobby-smoke-tests to decide whether and how to add a Hobby ingestion smoke test.' From 4e66fbb03ca92396b28cefbca52d977be1545100 Mon Sep 17 00:00:00 2001 From: Paul D'Ambra Date: Fri, 14 Aug 2026 20:01:39 +0100 Subject: [PATCH 032/289] fix(ci): enable replay for hobby smoke tests Generated-By: PostHog Desktop Task-Id: 3b2cd40e-fad1-41a2-bf15-419b22c0f84a --- bin/hobby-ci-setup-user.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/hobby-ci-setup-user.py b/bin/hobby-ci-setup-user.py index 48308d297390..48aec9bf9d12 100644 --- a/bin/hobby-ci-setup-user.py +++ b/bin/hobby-ci-setup-user.py @@ -24,6 +24,8 @@ team = Team.objects.filter(organization=org).first() if not team: team = Team.objects.create(organization=org, name="Default project") +team.session_recording_opt_in = True +team.save(update_fields=["session_recording_opt_in"]) user = User.objects.filter(email="ci@posthog.com").first() if not user: From 9a7982d085ad6848ce48ecac38fd16046c4551df Mon Sep 17 00:00:00 2001 From: Phil Haack Date: Fri, 14 Aug 2026 13:22:28 -0700 Subject: [PATCH 033/289] feat(flags): switch flag_evaluations typed columns to DEFAULT MATERIALIZED columns reject ALTER UPDATE, which is what the property- removal deletion path uses to reset extracted values. DEFAULT computes the same values on insert but stays assignable, matching the kind materialize() mints on sharded_events. Migration 0299 drop-and-recreates the empty table family; measured behaviors are recorded in docs/internal/clickhouse-deletion-coverage.md. Generated-By: PostHog Desktop Task-Id: a81b6b1e-b91d-42af-9ccc-108b819f65ea --- docs/internal/clickhouse-deletion-coverage.md | 36 +++++++------ .../hcl/golden/local-multi/data.hcl | 54 +++++++++---------- .../hcl/roles/data/local/tables.hcl | 25 ++++----- .../clickhouse/hcl/sql/local-multi/data.sql | 18 +++---- .../0299_flag_evaluations_default_columns.py | 49 +++++++++++++++++ .../clickhouse/migrations/max_migration.txt | 2 +- .../test/__snapshots__/test_schema.ambr | 36 ++++++------- posthog/clickhouse/test/test_schema.py | 11 ++-- posthog/dags/data_deletion_requests.py | 7 +-- .../dags/tests/test_data_deletion_requests.py | 7 +-- posthog/models/deletion_targets.py | 7 ++- posthog/models/flag_evaluations/sql.py | 51 +++++++++++------- 12 files changed, 188 insertions(+), 115 deletions(-) create mode 100644 posthog/clickhouse/migrations/0299_flag_evaluations_default_columns.py diff --git a/docs/internal/clickhouse-deletion-coverage.md b/docs/internal/clickhouse-deletion-coverage.md index 5d255d8f25ab..e3854f03235b 100644 --- a/docs/internal/clickhouse-deletion-coverage.md +++ b/docs/internal/clickhouse-deletion-coverage.md @@ -47,27 +47,31 @@ That decision predates this document; the older `posthog/models/async_deletion/d The events property-removal path rewrites rows in a staging table and resets each affected materialized column with `ALTER TABLE … UPDATE = ''`. That works because `materialize()` creates columns as `DEFAULT `, which is assignable. -`flag_evaluations` declares its nine typed columns as true ClickHouse `MATERIALIZED`. Measured against ClickHouse 26.6: +All of that machinery (column discovery, staging rewrite, shard walk) is scoped to `events`; none of it reaches `flag_evaluations`. +Until it does, `get_property_removal_shards` refuses to start when the table holds rows matching the request, so a request cannot complete while data it named survives. +The check costs nothing while the table is empty. -- Assigning to one is rejected: `Cannot UPDATE materialized column 'session_id'`. -- Updating `properties` is rejected too, because `flag_key` is materialized from it and sits in the sort key: `Updated column 'properties' affects MATERIALIZED column 'flag_key', which is a key column`. -- `CREATE TABLE tmp AS sharded_flag_evaluations ENGINE = MergeTree()` inherits the sort key and the column kinds, so the staging table hits the same rejection. +The schema stopped being a second obstacle with migration `0299_flag_evaluations_default_columns`, which recreated the nine typed columns as `DEFAULT `, the kind `materialize()` mints on events; they were true ClickHouse `MATERIALIZED` before, which is not assignable at all. +Measured against ClickHouse 26.6.2 on the `DEFAULT` shape: -The planned fix is two follow-ups: +- `CREATE TABLE` accepts a `DEFAULT`-from-`properties` column (`flag_key`) in the sort key, and an insert that omits the typed columns computes them from `properties`. +- Assigning to a non-key typed column is accepted: `ALTER TABLE … UPDATE session_id = ''` completes. Under `MATERIALIZED` it was rejected with `Cannot UPDATE materialized column 'session_id'`. +- Updating `properties` is accepted, alone and in the events-path form that resets affected typed columns in the same mutation. + The `MATERIALIZED`-era rejection (`Updated column 'properties' affects MATERIALIZED column 'flag_key', which is a key column`) does not fire for `DEFAULT` dependents. +- An `UPDATE` of `properties` does not recompute the typed columns; rows keep their stored values. + The rewrite must reset each affected column explicitly, exactly as the events path already does. +- `flag_key` itself can never be reset: `ALTER TABLE … UPDATE flag_key = ''` is rejected with `Cannot UPDATE key column 'flag_key'` (`CANNOT_UPDATE_COLUMN`), whatever the column kind. + A request naming `$feature_flag` therefore still cannot be honored by mutation; that one property needs a refusal, or the heavier rewrite: `INSERT … SELECT` the cleaned rows omitting the typed columns so the shard recomputes them, then lightweight-delete the originals. -1. Recreate the table with the typed columns as `DEFAULT `, the kind `materialize()` mints on events. - Recreating is only free while the table is empty, so this must land before the producer ships. -2. Point the events rewrite machinery (column discovery, staging rewrite, shard walk) at the table; today all of it is scoped to `events`. +The switch also changed two behaviors, measured on the same shape: -One open question for the schema follow-up: `flag_key` sits in the sort key, and ClickHouse never accepts `UPDATE` on a key column, whatever its kind. -Whether `UPDATE properties` is accepted once a `DEFAULT` key column depends on it needs measuring. -If it is still rejected, the fallback is the heavier rewrite: `INSERT … SELECT` the cleaned rows, let the shard recompute the typed columns on write, then lightweight-delete the originals. +- `SELECT *` on the shard now includes the nine typed columns, where `MATERIALIZED` hid them. Nothing in the repo depended on the hidden shape. +- An insert that names a typed column stores the given value even when it contradicts `properties`, where `MATERIALIZED` rejected such inserts. + Producers must omit the columns; the Kafka path enforces that because `writable_flag_evaluations` does not declare them. -Until the fix exists, `get_property_removal_shards` refuses to start when the table holds rows matching the request, so a request cannot complete while data it named survives. -The check costs nothing while the table is empty. +The remaining fix is pointing the events rewrite machinery at this table, with the `$feature_flag` limitation above built into whatever it does here. -This is also why `flag_evaluations` is deliberately absent from `MATERIALIZATION_VALID_TABLES`. -Adding it would let `materialize()` mint `DEFAULT`-kind columns on a table whose property-removal path cannot reset them. +`flag_evaluations` stays deliberately absent from `MATERIALIZATION_VALID_TABLES` until that lands: new `materialize()`-minted columns would only widen what the unfixed path silently leaves behind. #### If a request arrives before the fix lands @@ -101,7 +105,7 @@ Keeping the fork downstream of person resolution is the contract, tracked on #81 ## Adding a table -Register it in `PERSONAL_DATA_TARGETS`, with capability flags reflecting what its schema can actually take. +Register it in `PERSONAL_DATA_TARGETS`, with capability flags reflecting what its schema can actually take and what the sweep code actually implements: `accepts_property_rewrite` needs the rewrite machinery to reach the table, not just assignable columns. If it is not going to be swept, add it to `TTL_ONLY_TABLES` with the window you are accepting. `posthog/clickhouse/test/test_deletion_coverage.py` fails on any storage table that declares `person_properties` and appears in neither list, so the decision has to be made rather than skipped. diff --git a/posthog/clickhouse/hcl/golden/local-multi/data.hcl b/posthog/clickhouse/hcl/golden/local-multi/data.hcl index 9347ba413bdf..c658a215804f 100644 --- a/posthog/clickhouse/hcl/golden/local-multi/data.hcl +++ b/posthog/clickhouse/hcl/golden/local-multi/data.hcl @@ -6159,49 +6159,49 @@ database "posthog" { default = "timestamp" } column "$group_0" { - type = "String" - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$group_0'), '^\"|\"$', '')" - comment = "column_materializer::$group_0" + type = "String" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$group_0'), '^\"|\"$', '')" + comment = "column_materializer::$group_0" } column "$group_1" { - type = "String" - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$group_1'), '^\"|\"$', '')" - comment = "column_materializer::$group_1" + type = "String" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$group_1'), '^\"|\"$', '')" + comment = "column_materializer::$group_1" } column "$group_2" { - type = "String" - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$group_2'), '^\"|\"$', '')" - comment = "column_materializer::$group_2" + type = "String" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$group_2'), '^\"|\"$', '')" + comment = "column_materializer::$group_2" } column "$group_3" { - type = "String" - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$group_3'), '^\"|\"$', '')" - comment = "column_materializer::$group_3" + type = "String" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$group_3'), '^\"|\"$', '')" + comment = "column_materializer::$group_3" } column "$group_4" { - type = "String" - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$group_4'), '^\"|\"$', '')" - comment = "column_materializer::$group_4" + type = "String" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$group_4'), '^\"|\"$', '')" + comment = "column_materializer::$group_4" } column "flag_key" { - type = "String" - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag'), '^\"|\"$', '')" - comment = "column_materializer::properties::$feature_flag" + type = "String" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag'), '^\"|\"$', '')" + comment = "column_materializer::properties::$feature_flag" } column "response" { - type = "LowCardinality(String)" - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^\"|\"$', '')" - comment = "column_materializer::properties::$feature_flag_response" + type = "LowCardinality(String)" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^\"|\"$', '')" + comment = "column_materializer::properties::$feature_flag_response" } column "session_id" { - type = "String" - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$session_id'), '^\"|\"$', '')" - comment = "column_materializer::properties::$session_id" + type = "String" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$session_id'), '^\"|\"$', '')" + comment = "column_materializer::properties::$session_id" } column "request_id" { - type = "String" - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_request_id'), '^\"|\"$', '')" - comment = "column_materializer::properties::$feature_flag_request_id" + type = "String" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_request_id'), '^\"|\"$', '')" + comment = "column_materializer::properties::$feature_flag_request_id" } column "_timestamp" { type = "DateTime" diff --git a/posthog/clickhouse/hcl/roles/data/local/tables.hcl b/posthog/clickhouse/hcl/roles/data/local/tables.hcl index 0288b9be12e3..09795ae15f04 100644 --- a/posthog/clickhouse/hcl/roles/data/local/tables.hcl +++ b/posthog/clickhouse/hcl/roles/data/local/tables.hcl @@ -1,8 +1,9 @@ database "posthog" { # Mirrors the events column set minus events' materialized property columns, with - # the full properties JSON kept as the source of truth. The materialized columns - # below carry their expression only on sharded_flag_evaluations; the Distributed - # proxy repeats them plain, because a Distributed engine computes nothing. + # the full properties JSON kept as the source of truth. The typed property columns + # below carry their DEFAULT expression only on sharded_flag_evaluations; the + # Distributed proxy repeats them plain, because a Distributed engine computes + # nothing. table "_flag_evaluations_columns" { abstract = true column "uuid" { @@ -3578,31 +3579,31 @@ database "posthog" { } extend = "_flag_evaluations_columns" patch_column "$group_0" { - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$group_0'), '^\"|\"$', '')" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$group_0'), '^\"|\"$', '')" } patch_column "$group_1" { - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$group_1'), '^\"|\"$', '')" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$group_1'), '^\"|\"$', '')" } patch_column "$group_2" { - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$group_2'), '^\"|\"$', '')" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$group_2'), '^\"|\"$', '')" } patch_column "$group_3" { - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$group_3'), '^\"|\"$', '')" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$group_3'), '^\"|\"$', '')" } patch_column "$group_4" { - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$group_4'), '^\"|\"$', '')" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$group_4'), '^\"|\"$', '')" } patch_column "flag_key" { - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag'), '^\"|\"$', '')" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag'), '^\"|\"$', '')" } patch_column "response" { - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^\"|\"$', '')" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^\"|\"$', '')" } patch_column "session_id" { - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$session_id'), '^\"|\"$', '')" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$session_id'), '^\"|\"$', '')" } patch_column "request_id" { - materialized = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_request_id'), '^\"|\"$', '')" + default = "replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_request_id'), '^\"|\"$', '')" } index "distinct_id_idx" { expr = "distinct_id" diff --git a/posthog/clickhouse/hcl/sql/local-multi/data.sql b/posthog/clickhouse/hcl/sql/local-multi/data.sql index 5d74dd07abef..0d4a31e5cc85 100644 --- a/posthog/clickhouse/hcl/sql/local-multi/data.sql +++ b/posthog/clickhouse/hcl/sql/local-multi/data.sql @@ -1187,15 +1187,15 @@ CREATE TABLE posthog.sharded_flag_evaluations ( group3_properties String, group4_properties String, inserted_at DateTime64(6, 'UTC') DEFAULT timestamp, - $group_0 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_0'), '^"|"$', '') COMMENT 'column_materializer::$group_0', - $group_1 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_1'), '^"|"$', '') COMMENT 'column_materializer::$group_1', - $group_2 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_2'), '^"|"$', '') COMMENT 'column_materializer::$group_2', - $group_3 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_3'), '^"|"$', '') COMMENT 'column_materializer::$group_3', - $group_4 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_4'), '^"|"$', '') COMMENT 'column_materializer::$group_4', - flag_key String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag', - response LowCardinality(String) MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_response', - session_id String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$session_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$session_id', - request_id String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_request_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_request_id', + $group_0 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_0'), '^"|"$', '') COMMENT 'column_materializer::$group_0', + $group_1 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_1'), '^"|"$', '') COMMENT 'column_materializer::$group_1', + $group_2 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_2'), '^"|"$', '') COMMENT 'column_materializer::$group_2', + $group_3 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_3'), '^"|"$', '') COMMENT 'column_materializer::$group_3', + $group_4 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_4'), '^"|"$', '') COMMENT 'column_materializer::$group_4', + flag_key String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag', + response LowCardinality(String) DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_response', + session_id String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$session_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$session_id', + request_id String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_request_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_request_id', _timestamp DateTime, _offset UInt64, _partition UInt64, diff --git a/posthog/clickhouse/migrations/0299_flag_evaluations_default_columns.py b/posthog/clickhouse/migrations/0299_flag_evaluations_default_columns.py new file mode 100644 index 000000000000..21e754459dd7 --- /dev/null +++ b/posthog/clickhouse/migrations/0299_flag_evaluations_default_columns.py @@ -0,0 +1,49 @@ +from posthog.clickhouse.client.connection import NodeRole +from posthog.clickhouse.client.migration_tools import run_sql_with_exceptions +from posthog.models.flag_evaluations.sql import ( + DISTRIBUTED_FLAG_EVALUATIONS_TABLE_SQL, + DROP_FLAG_EVALUATIONS_TABLE_SQL, + FLAG_EVALUATIONS_TABLE, + FLAG_EVALUATIONS_TABLE_SQL, + FLAG_EVALUATIONS_WRITABLE_TABLE, + WRITABLE_FLAG_EVALUATIONS_TABLE_SQL, +) + +# Recreates flag_evaluations with its nine typed property columns as DEFAULT +# instead of MATERIALIZED, so the property-removal path can reset them by +# ALTER UPDATE. posthog/models/flag_evaluations/sql.py carries the full rationale. +# +# Drop-and-recreate rather than ALTER, as in 0297: nothing produces to the topic +# yet, so the family is empty everywhere and this must land before the producer +# does. Unlike 0297 the Kafka table and the MV stay untouched, because their DDL +# does not change; only the sharded table's column kind does. The two Distributed +# fronts render identical DDL too, but recreating them is free while the family +# is empty and reconciles any environment whose fronts drifted from the repo +# rendering. The sharded table is replicated, so its drop carries SYNC to clear +# ZooKeeper metadata before the recreate; the Distributed fronts drop plain. +operations = [ + run_sql_with_exceptions( + f"DROP TABLE IF EXISTS {FLAG_EVALUATIONS_TABLE}", + node_roles=[NodeRole.DATA], + ), + run_sql_with_exceptions( + f"DROP TABLE IF EXISTS {FLAG_EVALUATIONS_WRITABLE_TABLE}", + node_roles=[NodeRole.INGESTION_MEDIUM], + ), + run_sql_with_exceptions( + DROP_FLAG_EVALUATIONS_TABLE_SQL(), + node_roles=[NodeRole.DATA], + ), + run_sql_with_exceptions( + FLAG_EVALUATIONS_TABLE_SQL(), + node_roles=[NodeRole.DATA], + ), + run_sql_with_exceptions( + WRITABLE_FLAG_EVALUATIONS_TABLE_SQL(), + node_roles=[NodeRole.INGESTION_MEDIUM], + ), + run_sql_with_exceptions( + DISTRIBUTED_FLAG_EVALUATIONS_TABLE_SQL(), + node_roles=[NodeRole.DATA], + ), +] diff --git a/posthog/clickhouse/migrations/max_migration.txt b/posthog/clickhouse/migrations/max_migration.txt index a3e59dd17ae5..df674c25cfa3 100644 --- a/posthog/clickhouse/migrations/max_migration.txt +++ b/posthog/clickhouse/migrations/max_migration.txt @@ -1 +1 @@ -0298_logs34_mv_explicit_columns +0299_flag_evaluations_default_columns diff --git a/posthog/clickhouse/test/__snapshots__/test_schema.ambr b/posthog/clickhouse/test/__snapshots__/test_schema.ambr index a44bb4562cdc..46c896c6c52b 100644 --- a/posthog/clickhouse/test/__snapshots__/test_schema.ambr +++ b/posthog/clickhouse/test/__snapshots__/test_schema.ambr @@ -6928,15 +6928,15 @@ group4_properties String, inserted_at DateTime64(6, 'UTC') DEFAULT timestamp - , $group_0 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_0'), '^"|"$', '') COMMENT 'column_materializer::$group_0' - , $group_1 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_1'), '^"|"$', '') COMMENT 'column_materializer::$group_1' - , $group_2 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_2'), '^"|"$', '') COMMENT 'column_materializer::$group_2' - , $group_3 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_3'), '^"|"$', '') COMMENT 'column_materializer::$group_3' - , $group_4 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_4'), '^"|"$', '') COMMENT 'column_materializer::$group_4' - , flag_key String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag' - , response LowCardinality(String) MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_response' - , session_id String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$session_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$session_id' - , request_id String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_request_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_request_id' + , $group_0 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_0'), '^"|"$', '') COMMENT 'column_materializer::$group_0' + , $group_1 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_1'), '^"|"$', '') COMMENT 'column_materializer::$group_1' + , $group_2 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_2'), '^"|"$', '') COMMENT 'column_materializer::$group_2' + , $group_3 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_3'), '^"|"$', '') COMMENT 'column_materializer::$group_3' + , $group_4 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_4'), '^"|"$', '') COMMENT 'column_materializer::$group_4' + , flag_key String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag' + , response LowCardinality(String) DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_response' + , session_id String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$session_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$session_id' + , request_id String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_request_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_request_id' , INDEX distinct_id_idx distinct_id TYPE bloom_filter(0.01) GRANULARITY 1 @@ -11904,15 +11904,15 @@ group4_properties String, inserted_at DateTime64(6, 'UTC') DEFAULT timestamp - , $group_0 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_0'), '^"|"$', '') COMMENT 'column_materializer::$group_0' - , $group_1 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_1'), '^"|"$', '') COMMENT 'column_materializer::$group_1' - , $group_2 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_2'), '^"|"$', '') COMMENT 'column_materializer::$group_2' - , $group_3 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_3'), '^"|"$', '') COMMENT 'column_materializer::$group_3' - , $group_4 String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$group_4'), '^"|"$', '') COMMENT 'column_materializer::$group_4' - , flag_key String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag' - , response LowCardinality(String) MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_response' - , session_id String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$session_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$session_id' - , request_id String MATERIALIZED replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_request_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_request_id' + , $group_0 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_0'), '^"|"$', '') COMMENT 'column_materializer::$group_0' + , $group_1 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_1'), '^"|"$', '') COMMENT 'column_materializer::$group_1' + , $group_2 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_2'), '^"|"$', '') COMMENT 'column_materializer::$group_2' + , $group_3 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_3'), '^"|"$', '') COMMENT 'column_materializer::$group_3' + , $group_4 String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$group_4'), '^"|"$', '') COMMENT 'column_materializer::$group_4' + , flag_key String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag' + , response LowCardinality(String) DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_response'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_response' + , session_id String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$session_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$session_id' + , request_id String DEFAULT replaceRegexpAll(JSONExtractRaw(properties, '$feature_flag_request_id'), '^"|"$', '') COMMENT 'column_materializer::properties::$feature_flag_request_id' , INDEX distinct_id_idx distinct_id TYPE bloom_filter(0.01) GRANULARITY 1 diff --git a/posthog/clickhouse/test/test_schema.py b/posthog/clickhouse/test/test_schema.py index 17be8e15860c..67ad72906160 100644 --- a/posthog/clickhouse/test/test_schema.py +++ b/posthog/clickhouse/test/test_schema.py @@ -111,11 +111,12 @@ def test_flag_evaluations_mv_projection_matches_column_template(): def test_flag_evaluations_read_table_declares_every_stored_column(): - # The materialized columns are declared on sharded_flag_evaluations, which - # computes them, and repeated as plain columns on the Distributed read table, - # which computes nothing. Those two lists are maintained by hand, so a column - # or a type changed in one and not the other stays invisible until a query - # asks flag_evaluations for something only the shards have. + # The typed property columns carry their DEFAULT expression on + # sharded_flag_evaluations, which computes them, and are repeated as plain + # columns on the Distributed read table, which computes nothing. Those two + # lists are maintained by hand, so a column or a type changed in one and not + # the other stays invisible until a query asks flag_evaluations for + # something only the shards have. stored_columns = _flag_evaluations_table_columns(FLAG_EVALUATIONS_TABLE_SQL()) assert stored_columns diff --git a/posthog/dags/data_deletion_requests.py b/posthog/dags/data_deletion_requests.py index 1298545c779d..90abbb327419 100644 --- a/posthog/dags/data_deletion_requests.py +++ b/posthog/dags/data_deletion_requests.py @@ -596,9 +596,10 @@ def execute_event_deletion( _PROPERTY_REWRITE_UNSWEEPABLE_REASON = ( - "its typed columns are MATERIALIZED from properties, so ClickHouse rejects both an assignment " - "to them and an update of properties itself; rewriting these rows needs a re-insert instead of " - "the staging-table mutation this job runs. There is no way to complete this request today: " + "the property-rewrite machinery is scoped to the events tables and does not reach it; a " + "request naming $feature_flag additionally cannot " + "be honored by mutation at all, because flag_key sits in the table's sort key where no UPDATE " + "can reset it. There is no way to complete this request today: " "either narrow its events to ones this table never stores, or wait out the table's TTL. " f"See {_COVERAGE_DOC}." ) diff --git a/posthog/dags/tests/test_data_deletion_requests.py b/posthog/dags/tests/test_data_deletion_requests.py index 870ee91f63e0..5317ed7ecb66 100644 --- a/posthog/dags/tests/test_data_deletion_requests.py +++ b/posthog/dags/tests/test_data_deletion_requests.py @@ -2054,9 +2054,10 @@ def test_deferred_event_removal_queues_flag_evaluations_and_blocks_promotion(clu @pytest.mark.django_db def test_get_property_removal_shards_refuses_when_flag_evaluations_holds_matching_rows(cluster: ClickhouseCluster): - # Property removal cannot rewrite flag_evaluations: its typed columns are MATERIALIZED, so - # ClickHouse rejects both an assignment to them and an update of properties itself. Completing - # the request anyway would report the property erased while a copy of it survived. + # Property removal cannot rewrite flag_evaluations: the rewrite machinery is scoped to the + # events tables, and flag_key sits in the sort key where no mutation can reset it, so even + # that machinery could not fully honor a request naming $feature_flag. Completing the + # request anyway would report the property erased while a copy of it survived. request = DataDeletionRequest.objects.create( team_id=PROP_TEAM_ID, request_type=RequestType.PROPERTY_REMOVAL, diff --git a/posthog/models/deletion_targets.py b/posthog/models/deletion_targets.py index 3f3713d1f2a5..b078ee4dd06a 100644 --- a/posthog/models/deletion_targets.py +++ b/posthog/models/deletion_targets.py @@ -66,8 +66,11 @@ class DeletionTarget: # exists. A compiled fragment names physical columns (mat_*, the property-group maps, or JSON # subcolumns), so it only runs against the schema it was compiled for. hogql_schema: HogQLSchema | None = None - # properties/person_properties can be rewritten in place. True only where the property columns - # are DEFAULT-kind, as materialize() mints them, and so can be reset by ALTER UPDATE. + # properties/person_properties can be rewritten in place. True needs both halves: the property + # columns are DEFAULT-kind (assignable by ALTER UPDATE, as materialize() mints them), and the + # property-rewrite machinery in posthog/dags/data_deletion_requests.py actually sweeps the + # table, which today is hardcoded to the events tables. flag_evaluations satisfies only the + # schema half, so flipping this without extending the sweep silently under-deletes. accepts_property_rewrite: bool = False # Read uuids from this table when queueing a deferred deletion. False where the rows duplicate # another target's uuids, which would queue each one twice. diff --git a/posthog/models/flag_evaluations/sql.py b/posthog/models/flag_evaluations/sql.py index 2ec2ccb3582e..339f75497740 100644 --- a/posthog/models/flag_evaluations/sql.py +++ b/posthog/models/flag_evaluations/sql.py @@ -48,8 +48,9 @@ # The sort key matches the queries we run: per-flag usage over a date range, and # uniques within one flag. toDate(timestamp) sits inside it because PARTITION BY # is monthly — without it, a one-day query for one flag would read that flag's -# whole month. flag_key is a materialized column; ClickHouse computes those before -# it sorts a part, so one can carry a sort key. The trailing hash intentionally +# whole month. flag_key is a DEFAULT column; ClickHouse fills column defaults at +# insert, before it sorts a part, so one can carry a sort key, though a key +# column can never be ALTER UPDATEd, whatever its kind. The trailing hash intentionally # differs from the sharding key — cityHash64 is the events table's convention for # within-shard ordering — and a MergeTree ORDER BY is immutable once data exists, # so the two must not silently move together. @@ -111,24 +112,36 @@ # # $group_0..$group_4 carry the events table's names, types and comment form so # group filtering resolves the same columns on both. -_FLAG_EVALUATIONS_MATERIALIZED_COLUMNS = f""" - , $group_0 String MATERIALIZED {trim_quotes_expr("JSONExtractRaw(properties, '$group_0')")} COMMENT 'column_materializer::$group_0' - , $group_1 String MATERIALIZED {trim_quotes_expr("JSONExtractRaw(properties, '$group_1')")} COMMENT 'column_materializer::$group_1' - , $group_2 String MATERIALIZED {trim_quotes_expr("JSONExtractRaw(properties, '$group_2')")} COMMENT 'column_materializer::$group_2' - , $group_3 String MATERIALIZED {trim_quotes_expr("JSONExtractRaw(properties, '$group_3')")} COMMENT 'column_materializer::$group_3' - , $group_4 String MATERIALIZED {trim_quotes_expr("JSONExtractRaw(properties, '$group_4')")} COMMENT 'column_materializer::$group_4' - , flag_key String MATERIALIZED {trim_quotes_expr("JSONExtractRaw(properties, '$feature_flag')")} COMMENT 'column_materializer::properties::$feature_flag' - , response LowCardinality(String) MATERIALIZED {trim_quotes_expr("JSONExtractRaw(properties, '$feature_flag_response')")} COMMENT 'column_materializer::properties::$feature_flag_response' - , session_id String MATERIALIZED {trim_quotes_expr("JSONExtractRaw(properties, '$session_id')")} COMMENT 'column_materializer::properties::$session_id' - , request_id String MATERIALIZED {trim_quotes_expr("JSONExtractRaw(properties, '$feature_flag_request_id')")} COMMENT 'column_materializer::properties::$feature_flag_request_id' +# +# DEFAULT rather than MATERIALIZED, the kind materialize() mints on sharded_events: +# both compute the expression when an insert omits the column, but only a DEFAULT +# column accepts ALTER UPDATE, which the events property-removal path relies on to +# reset extracted values whose source property was erased (see +# docs/internal/clickhouse-deletion-coverage.md). An UPDATE of properties does not +# recompute these columns, so a rewrite must reset each affected column in the +# same mutation. The cost is a footgun MATERIALIZED did not have: an insert that +# names one of these columns stores the given value even when it contradicts +# properties. Producers must omit them, which the Kafka path enforces by +# writable_flag_evaluations not declaring them. +_FLAG_EVALUATIONS_TYPED_COLUMNS = f""" + , $group_0 String DEFAULT {trim_quotes_expr("JSONExtractRaw(properties, '$group_0')")} COMMENT 'column_materializer::$group_0' + , $group_1 String DEFAULT {trim_quotes_expr("JSONExtractRaw(properties, '$group_1')")} COMMENT 'column_materializer::$group_1' + , $group_2 String DEFAULT {trim_quotes_expr("JSONExtractRaw(properties, '$group_2')")} COMMENT 'column_materializer::$group_2' + , $group_3 String DEFAULT {trim_quotes_expr("JSONExtractRaw(properties, '$group_3')")} COMMENT 'column_materializer::$group_3' + , $group_4 String DEFAULT {trim_quotes_expr("JSONExtractRaw(properties, '$group_4')")} COMMENT 'column_materializer::$group_4' + , flag_key String DEFAULT {trim_quotes_expr("JSONExtractRaw(properties, '$feature_flag')")} COMMENT 'column_materializer::properties::$feature_flag' + , response LowCardinality(String) DEFAULT {trim_quotes_expr("JSONExtractRaw(properties, '$feature_flag_response')")} COMMENT 'column_materializer::properties::$feature_flag_response' + , session_id String DEFAULT {trim_quotes_expr("JSONExtractRaw(properties, '$session_id')")} COMMENT 'column_materializer::properties::$session_id' + , request_id String DEFAULT {trim_quotes_expr("JSONExtractRaw(properties, '$feature_flag_request_id')")} COMMENT 'column_materializer::properties::$feature_flag_request_id' """ # A Distributed engine computes nothing, so the read table repeats the same names # and types without the expression, which is what lets a query against # flag_evaluations select the columns the shards store. The writable table omits -# them entirely, because rows arrive there without these columns and the shard -# fills them in. -_FLAG_EVALUATIONS_PROXY_MATERIALIZED_COLUMNS = """ +# them entirely: carrying the DEFAULT expressions there would compute the values +# on the ingestion nodes and ship the widened rows over the network, so rows +# arrive narrow and the shard computes them, matching writable_events. +_FLAG_EVALUATIONS_PROXY_TYPED_COLUMNS = """ , $group_0 String COMMENT 'column_materializer::$group_0' , $group_1 String COMMENT 'column_materializer::$group_1' , $group_2 String COMMENT 'column_materializer::$group_2' @@ -176,7 +189,7 @@ def FLAG_EVALUATIONS_DATA_TABLE_ENGINE() -> MergeTreeEngine: CREATE TABLE IF NOT EXISTS {FLAG_EVALUATIONS_DATA_TABLE} ( {_FLAG_EVALUATIONS_COLUMNS} - {_FLAG_EVALUATIONS_MATERIALIZED_COLUMNS} + {_FLAG_EVALUATIONS_TYPED_COLUMNS} {_FLAG_EVALUATIONS_INDEXES} {KAFKA_COLUMNS_WITH_PARTITION} ) @@ -213,12 +226,12 @@ def DROP_FLAG_EVALUATIONS_TABLE_SQL() -> str: return f"DROP TABLE IF EXISTS {FLAG_EVALUATIONS_DATA_TABLE} SYNC" -def _distributed_table_sql(table_name: str, *, materialized_columns: str = "") -> str: +def _distributed_table_sql(table_name: str, *, typed_columns: str = "") -> str: return f""" CREATE TABLE IF NOT EXISTS {table_name} ( {_FLAG_EVALUATIONS_COLUMNS} - {materialized_columns} + {typed_columns} {KAFKA_COLUMNS_WITH_PARTITION} ) ENGINE = {Distributed(data_table=FLAG_EVALUATIONS_DATA_TABLE, sharding_key=FLAG_EVALUATIONS_SHARDING_KEY)} @@ -230,7 +243,7 @@ def _distributed_table_sql(table_name: str, *, materialized_columns: str = "") - # Read path on DATA nodes, and the name queries use. DISTRIBUTED_FLAG_EVALUATIONS_TABLE_SQL = lambda: _distributed_table_sql( - FLAG_EVALUATIONS_TABLE, materialized_columns=_FLAG_EVALUATIONS_PROXY_MATERIALIZED_COLUMNS + FLAG_EVALUATIONS_TABLE, typed_columns=_FLAG_EVALUATIONS_PROXY_TYPED_COLUMNS ) From 2909b9a0204fa74c382d64d180cac4c9c6390db5 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Fri, 14 Aug 2026 16:48:10 -0400 Subject: [PATCH 034/289] fix(capture): reconcile token limiter with per-key limiter skip from master Master now skips the per-key limiter for events whose person processing is already off. Update the token limiter's comments and parity test to the merged semantics: a token-limited batch stops feeding the per-key limiter, shielding its cache and Redis pipeline from the flood's cardinality. Also pass the new router arg in the person-processing matrix test that landed on master. Co-Authored-By: Claude Fable 5 --- rust/capture/src/events/analytics.rs | 6 ++--- rust/capture/src/v1/analytics/process.rs | 24 ++++++++++--------- .../integration_person_processing_matrix.rs | 1 + 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/rust/capture/src/events/analytics.rs b/rust/capture/src/events/analytics.rs index 4265e927b046..058388588e81 100644 --- a/rust/capture/src/events/analytics.rs +++ b/rust/capture/src/events/analytics.rs @@ -480,9 +480,9 @@ async fn process_events_inner( // overflowable lane is reachable, so behavior is identical across paths. if context.capture_mode.applies_global_rate_limit() { // Token-level aggregate first. Stamps only person processing and the - // overflow reroute -- events stay live, so the per-key loop below still - // consults the shared per-key limiter for every one of them, keeping - // v0/v1 per-key counts identical (see the invariant note above). + // overflow reroute -- and since the per-key loop below skips events + // whose person processing is already off, a token-level flood also + // stops feeding the per-key limiter's cache and Redis pipeline. if let Some(ref limiter) = global_rate_limiter_token { let event_count = events.len() as u64; let cache_key = GlobalRateLimitKey::Token(&context.token).to_cache_key(); diff --git a/rust/capture/src/v1/analytics/process.rs b/rust/capture/src/v1/analytics/process.rs index 1ff625a2c4e8..63ec6482edfc 100644 --- a/rust/capture/src/v1/analytics/process.rs +++ b/rust/capture/src/v1/analytics/process.rs @@ -167,10 +167,11 @@ pub async fn process_batch( if state.capture_mode.applies_global_rate_limit() { // Token-level aggregate first: a token-limited event keeps // `EventResult::Ok` (only person processing and destination are - // stamped), so the per-key loop below still counts its volume -- - // preserving the v0/v1 invariant that both pipelines feed the shared - // per-key limiter identical counts. The loop then reports such events - // as `already_disabled` and never double-stamps them. + // stamped). The per-key loop below skips events whose person + // processing is already off, so a token-level flood also stops feeding + // the per-key limiter -- its cache, channel, and Redis pipeline are + // shielded from exactly the cardinality burst that trips the token + // level. The loop reports such events as `already_disabled`. if let Some(ref limiter) = state.global_rate_limiter_token { apply_token_limits(limiter, context, &mut events).await; } @@ -2501,11 +2502,12 @@ mod tests { } #[tokio::test] - async fn token_limited_events_still_feed_per_key_limiter() { - // The v0/v1 invariant: both pipelines consult the shared per-key - // limiter for every non-dropped event. A token-limited batch must not - // vanish from the per-key counts -- the per-key loop still evaluates - // each event and reports it as already_disabled. + async fn token_limited_events_skip_per_key_limiter() { + // A token-limited event has person processing off already, and the + // per-key stage skips such events entirely (the limiter has nothing + // left to take away). That skip is also what shields the per-key + // limiter's cache and Redis pipeline from the cardinality of a + // token-level flood -- this test pins that shielding. let token_limiter = mock_limiter(vec!["phc_tok"]); let (per_key_limiter, calls) = mock_limiter_with_log(vec![]); let ctx = td_context(); @@ -2519,8 +2521,8 @@ mod tests { assert_eq!( calls.lock().unwrap().len(), - 2, - "per-key limiter must still count token-limited events" + 0, + "token-limited events must not reach the per-key limiter" ); assert_eq!(tally.already_disabled, 2); assert_eq!(tally.limited, 0); diff --git a/rust/capture/tests/integration_person_processing_matrix.rs b/rust/capture/tests/integration_person_processing_matrix.rs index c055a59af26b..ad49a5bbf343 100644 --- a/rust/capture/tests/integration_person_processing_matrix.rs +++ b/rust/capture/tests/integration_person_processing_matrix.rs @@ -242,6 +242,7 @@ async fn run_v0(inputs: Inputs, distinct_ids: &[&str]) -> Batch { Arc::new(sink), redis, Some(Arc::new(limiter)), + None, // global_rate_limiter_token quota_limiter, TokenDropper::default(), Some(service), From efeea8898356fb4c1e7ae2b7bed2e1daf248854a Mon Sep 17 00:00:00 2001 From: Phil Haack Date: Fri, 14 Aug 2026 14:22:37 -0700 Subject: [PATCH 035/289] chore(flags): scope migration 0299 header to the schema obstacle Generated-By: PostHog Desktop Task-Id: a81b6b1e-b91d-42af-9ccc-108b819f65ea --- .../migrations/0299_flag_evaluations_default_columns.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/posthog/clickhouse/migrations/0299_flag_evaluations_default_columns.py b/posthog/clickhouse/migrations/0299_flag_evaluations_default_columns.py index 21e754459dd7..54df9c6b19ab 100644 --- a/posthog/clickhouse/migrations/0299_flag_evaluations_default_columns.py +++ b/posthog/clickhouse/migrations/0299_flag_evaluations_default_columns.py @@ -10,8 +10,11 @@ ) # Recreates flag_evaluations with its nine typed property columns as DEFAULT -# instead of MATERIALIZED, so the property-removal path can reset them by -# ALTER UPDATE. posthog/models/flag_evaluations/sql.py carries the full rationale. +# instead of MATERIALIZED, the assignable kind the property-removal rewrite +# resets by ALTER UPDATE. This clears the schema obstacle only: the rewrite +# machinery still sweeps just the events tables, so property removal does not +# reach this table yet (docs/internal/clickhouse-deletion-coverage.md). +# posthog/models/flag_evaluations/sql.py carries the full rationale. # # Drop-and-recreate rather than ALTER, as in 0297: nothing produces to the topic # yet, so the family is empty everywhere and this must land before the producer From f1fd59967aeefac92039bf6a38c4e232b5136fe0 Mon Sep 17 00:00:00 2001 From: Julian Bez Date: Mon, 17 Aug 2026 10:44:07 +0200 Subject: [PATCH 036/289] feat(stamphog): port stacked PR support to the hosted runtime The stacked-PR work on this branch was built for the Action runtime, before products/stamphog existed. The hosted sandbox already clones and checks out the PR head for every review, so parent-PR symbols resolve there by construction; what it lacked was the rest of the design. - PRData.stacked keys off the repo's default branch instead of a hardcoded "master" - the hosted runtime reviews repos whose trunk is "main". Both runtimes pass base_ref/default_branch through. - review_local.py runs the pipeline with head_checkout=True, so a stacked PR in the sandbox gets the prompt note but never spins up the Action's head worktree (a full-tree checkout the sandbox already has). - The Agent SDK isolation (setting_sources=[], strict_mcp_config) now covers every hosted review by construction, since the checkout is the PR head; documented as an invariant in products/stamphog/AGENTS.md. - The diff scratch file is created with mkstemp under an unpredictable name in both runtimes, so a tracked symlink in a PR-authored checkout can't redirect the write. - post_verdict rechecks the live base ref against the reviewed one before posting - the hosted equivalent of the Action's base/head recheck. The retarget webhook already retracts and re-queues, but it can trail the activity. - The `edited` fanout budget goes 3 -> 4 for the Action's retarget dispatch (the WF008 lint landed after this branch was written), and the review job's !cancelled() condition matches master's later form. --- .github/workflows/pr-approval-agent.yml | 7 +- .gitignore | 2 +- products/stamphog/AGENTS.md | 11 ++- products/stamphog/README.md | 4 ++ .../stamphog/backend/temporal/activities.py | 15 ++++- .../backend/tests/test_integration.py | 31 +++++++++ .../workflow_lint/checks/pr_event_fanout.py | 4 +- tools/pr-approval-agent/README.md | 67 ++++++++++++------- tools/pr-approval-agent/github.py | 30 ++++++++- tools/pr-approval-agent/review_local.py | 15 ++++- tools/pr-approval-agent/review_pr.py | 43 +++++++----- tools/pr-approval-agent/reviewer.py | 30 ++++----- tools/pr-approval-agent/test_review_local.py | 65 ++++++++++++++++++ tools/pr-approval-agent/test_review_pr.py | 36 +++++++--- tools/pr-approval-agent/test_reviewer.py | 19 +++--- 15 files changed, 288 insertions(+), 91 deletions(-) diff --git a/.github/workflows/pr-approval-agent.yml b/.github/workflows/pr-approval-agent.yml index 5dcb53c3efca..6f52e1562cdd 100644 --- a/.github/workflows/pr-approval-agent.yml +++ b/.github/workflows/pr-approval-agent.yml @@ -31,8 +31,7 @@ jobs: # LLM review instead of posting a stamphog approval for a commit that is no longer # HEAD. The explicit decide-delta checks below keep the fail-closed re-review paths. if: >- - always() - && !cancelled() + !cancelled() && !github.event.pull_request.draft && github.event.pull_request.user.type != 'Bot' && !contains(github.event.pull_request.user.login, '[bot]') @@ -98,7 +97,7 @@ jobs: --output-json /tmp/review.json - name: Post review - if: always() && !cancelled() + if: ${{ !cancelled() }} env: # Everything stamphog does — the approval, the sticky comment, # the label strip — posts as the Stamphog app (GH_TOKEN) so it @@ -250,7 +249,7 @@ jobs: fi - name: Upload evidence - if: always() && !cancelled() + if: ${{ !cancelled() }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: name: review-${{ github.event.pull_request.number }} diff --git a/.gitignore b/.gitignore index 8bd2298c21be..5395b0757f4f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ # people's OCDs ^_^ .cache .planning -.pr-review-diff.patch +.pr-review-diff*.patch .qa-frontend/ __emails__ __pycache__/ diff --git a/products/stamphog/AGENTS.md b/products/stamphog/AGENTS.md index d1b7b6005f24..8f22cdb37edb 100644 --- a/products/stamphog/AGENTS.md +++ b/products/stamphog/AGENTS.md @@ -49,9 +49,10 @@ A newer relevant delivery supersedes older non-terminal runs. Rules that keep th - Terminal states (`TERMINAL_STATUSES` in `facade/enums.py`) are never rewritten — `mark_review_failed` must not clobber a delivered outcome, and terminal saves are conditional (`.exclude(status=SUPERSEDED).update(...)`), never plain `save()`. -- `post_verdict` guards before ANY GitHub write: superseded status, current head vs run head, and - a last fresh status read. Losing the final conditional update means dismiss-your-own-approval, - not "log and return". +- `post_verdict` guards before ANY GitHub write: superseded status, current head vs run head, + current base ref vs the reviewed one (a retarget rewrites the diff with the head unchanged, and + the retarget delivery can trail the activity), and a last fresh status read. Losing the final + conditional update means dismiss-your-own-approval, not "log and return". - Out-of-order webhook deliveries are dropped by the `payload_updated_at` clock — checked before the transaction AND re-checked under the row lock, and the descriptive-field refresh is gated on the same clock inside the UPDATE's WHERE clause. @@ -76,6 +77,10 @@ add a read-then-act path, pin it; this class of bug has been found on five separ `STAMPHOG_SANDBOX_EXTRA_EGRESS_DOMAINS`, not code edits. - Everything posted to GitHub goes through `_scrub_credentials` AND `_neutralize_active_markdown` (GitHub's camo proxy auto-fetches images — a markdown image URL is an exfiltration channel). +- The sandbox checkout is the PR head, so the engine's Agent SDK session runs with + `setting_sources=[]` + `strict_mcp_config` (reviewer.py): a PR-shipped `.claude/settings.json` + hook, `CLAUDE.md`, or `.mcp.json` is readable as untrusted content, never loaded as + configuration. Don't reintroduce filesystem settings discovery there. ## The self-driving inbox carve-out (the one exception to the bot-author refusal) diff --git a/products/stamphog/README.md b/products/stamphog/README.md index 18deac742ddd..1d37080eeb98 100644 --- a/products/stamphog/README.md +++ b/products/stamphog/README.md @@ -17,6 +17,10 @@ There is one non-webhook entry: **self-driving inbox PRs**. When a self-driving On top of reviews, a repo can enable a daily Slack digest of its merged PRs (`backend/tasks/digest.py`): merges are stamped with an audience (author's GitHub team, or a channel the repo declares under `digest:` in `.stamphog/policy.yml`), summarized with a small model, and posted per channel. Only stamphog-approved merges are digested, so the digest needs reviews enabled for the repo. +## Stacked PRs + +A stacked PR targets its parent's branch, not the repo's default branch, and depends on parent code that hasn't merged yet. The sandbox clones and checks out the PR head for every review, so the reviewer's Read/Grep/Glob already see the post-stack tree and parent symbols resolve; the engine is told the checkout is the head (`head_checkout=True`) so it never builds the Action's separate head worktree, and the prompt flags the PR as stacked (`PRData.stacked`, keyed on the repo's actual default branch). The diff stays scoped `base...head`. When the parent merges and GitHub retargets the child onto the default branch, the diff changes without a push: the webhook path retracts the standing approval and queues a fresh run, and `post_verdict` rechecks the live base against the reviewed one before posting. Engine details: [`tools/pr-approval-agent/README.md`](../../tools/pr-approval-agent/README.md#stacked-prs-graphite--git-stacks). + ## Configuration Per-repo settings live on `StamphogRepoConfig` (synced via the GitHub App install flow, managed in the Stamphog scene): review on/off, review mode (auto vs trigger label), digest on/off. Review policy (gates, deny-lists, tiers, ownership) is read from `.stamphog/policy.yml` on the repo's **default branch** — never from the PR head — layered over hosted defaults in [`backend/logic/policy_defaults/`](backend/logic/policy_defaults/). diff --git a/products/stamphog/backend/temporal/activities.py b/products/stamphog/backend/temporal/activities.py index 4c64308c6de9..6f3239b658cd 100644 --- a/products/stamphog/backend/temporal/activities.py +++ b/products/stamphog/backend/temporal/activities.py @@ -671,7 +671,18 @@ def post_verdict(input: StamphogReviewInput) -> dict: return {"verdict": "skipped_superseded"} current_pr = client.get_pr(repo, pull_request.pr_number) current_head = ((current_pr.get("head") or {}).get("sha") or "").strip() + # A base retarget (a stacked PR's parent merged, or a manual base switch) rewrites the reviewed + # diff with the head SHA unchanged, so the head guard alone can't see it. The retarget delivery + # retracts approvals and queues a fresh run, but that delivery can trail this activity. + reviewed_base_ref = ((output.get("pr") or {}).get("base") or {}).get("ref") or "" + current_base_ref = ((current_pr.get("base") or {}).get("ref") or "").strip() + drift: tuple[str, str] | None = None if current_head and current_head != run.head_sha: + drift = ("head_moved", f"head moved {run.head_sha} -> {current_head}") + elif reviewed_base_ref and current_base_ref and current_base_ref != reviewed_base_ref: + drift = ("base_retargeted", f"base retargeted {reviewed_base_ref} -> {current_base_ref}") + if drift is not None: + kind, detail = drift # Conditional: a retry after the terminal save already committed (e.g. the trailing digest # stamp crashed) must not rewrite a delivered COMPLETED outcome to SUPERSEDED — terminal # states are history. The stale-approval sweep retires that approval on the next delivery. @@ -680,8 +691,8 @@ def post_verdict(input: StamphogReviewInput) -> dict: ) if run.verdict != ReviewVerdict.APPROVED: _dismiss_orphaned_approval(client, run, input.team_id) - activity.logger.info(f"Skipping verdict for run {run.id}: head moved {run.head_sha} -> {current_head}") - return {"verdict": "skipped_head_moved"} + activity.logger.info(f"Skipping verdict for run {run.id}: {detail}") + return {"verdict": f"skipped_{kind}"} parsed = parse_reviewer_output(raw) diff --git a/products/stamphog/backend/tests/test_integration.py b/products/stamphog/backend/tests/test_integration.py index 861f356891a2..1451f438597f 100644 --- a/products/stamphog/backend/tests/test_integration.py +++ b/products/stamphog/backend/tests/test_integration.py @@ -939,6 +939,37 @@ def test_retry_after_head_move_never_rewrites_a_terminal_run(team, stamphog_chai assert [w for w in recorder.github_writes if w["kind"] == "dismiss_review"] == [] +@pytest.mark.django_db(databases=PRODUCT_DATABASES) +def test_post_verdict_skips_when_the_base_was_retargeted_under_the_run(team, stamphog_chain: StamphogChain) -> None: + # A stacked PR's parent merged mid-review: the child is retargeted to master, which rewrites + # the reviewed diff while the head SHA stays put. The retarget delivery retracts and re-queues, + # but it can trail this activity — post_verdict must recheck the live base itself, or an + # approval for the old base..head diff lands on the new one. + repo_config = _repo_config(team.id) + recorder = stamphog_chain.recorder + head_sha = "sha119a" + live_pr = _pr_object(119, "devex-dev", head_sha) | {"base": {"sha": "master-tip", "ref": "master"}} + recorder.register_pr(REPO, 119, live_pr, _pr_files()) + pull_request = PullRequest.objects.for_team(team.id).create( + team_id=team.id, repo_config=repo_config, pr_number=119, author_login="devex-dev" + ) + reviewed_pr = _pr_object(119, "devex-dev", head_sha) | {"base": {"sha": "parent-tip", "ref": "feat/parent"}} + run = ReviewRun.objects.for_team(team.id).create( + team_id=team.id, + pull_request=pull_request, + head_sha=head_sha, + status=ReviewRunStatus.REVIEWING, + output={"pr": reviewed_pr, "reviewer_raw": fakes.approved_engine_output().splitlines()[-1]}, + ) + + result = _run_activity(post_verdict, StamphogReviewInput(review_run_id=str(run.id), team_id=team.id)) + + assert result == {"verdict": "skipped_base_retargeted"} + assert [w for w in recorder.github_writes if w["kind"] == "approve_review"] == [] + run.refresh_from_db() + assert run.status == ReviewRunStatus.SUPERSEDED + + @pytest.mark.django_db(databases=PRODUCT_DATABASES) def test_bot_eyes_on_a_later_reactions_page_still_counts_as_in_flight(team, stamphog_chain: StamphogChain) -> None: # Anyone can react on a public PR, so an author could bury the trusted bot's fresh 👀 past the diff --git a/tools/hogli-commands/hogli_commands/workflow_lint/checks/pr_event_fanout.py b/tools/hogli-commands/hogli_commands/workflow_lint/checks/pr_event_fanout.py index f5554264a2c2..4be2a9d38d15 100644 --- a/tools/hogli-commands/hogli_commands/workflow_lint/checks/pr_event_fanout.py +++ b/tools/hogli-commands/hogli_commands/workflow_lint/checks/pr_event_fanout.py @@ -30,7 +30,9 @@ PR_EVENT_FANOUT_BUDGET: Mapping[str, int] = { "closed": 3, "converted_to_draft": 1, - "edited": 3, + # pr-approval-agent listens for base retargets (`edited` with `changes.base`); title/body + # edits skip every job there, but GitHub still counts the dispatch. + "edited": 4, "opened": 28, "ready_for_review": 11, "reopened": 24, diff --git a/tools/pr-approval-agent/README.md b/tools/pr-approval-agent/README.md index 9a01c6aed5ec..f4498f3b0f59 100644 --- a/tools/pr-approval-agent/README.md +++ b/tools/pr-approval-agent/README.md @@ -147,40 +147,57 @@ Every other verdict (REFUSED, ESCALATE, WAIT, ERROR) goes into a single sticky c ## Stacked PRs (Graphite / git stacks) -A stacked PR targets its parent branch, not master, and depends on code the -parent introduces but hasn't merged yet. Two parts make stamphog correct on -these: - -- **Exploration sees the post-stack tree.** The workflow checks out master - (hardcoded, so a PR can't swap the review script), but the LLM reviewer's - `Read`/`Grep`/`Glob` run in a detached **worktree at the PR head** instead. - The head tree already contains the parent PRs' code, so symbols from a - not-yet-merged parent resolve and aren't flagged as broken imports. The diff - itself is still computed `base_sha...head_sha`, so the review is scoped to - exactly this PR's changes. If the worktree cannot be created, stamphog - returns `ERROR` and retains the label rather than reviewing against the wrong - source tree. - - **Security:** the worktree is PR-authored content. The reviewer runs the - Agent SDK with `setting_sources=[]` (isolation mode), so it does **not** - load `.claude/settings.json` hooks (command execution) or `CLAUDE.md` - (injected instructions) from the head tree. Those files are still readable - as untrusted _content_ under the anti-injection notice — never as - configuration. Stacked PR heads with tracked symbolic links fail closed, - so a PR path cannot resolve outside the worktree. +A stacked PR targets its parent branch, not the repo's default branch, and +depends on code the parent introduces but hasn't merged yet. `PRData.stacked` +(`base_ref != default_branch`, so repos whose trunk is `main` work too) drives +the handling; the reviewer prompt tells the agent it is looking at a stacked +PR. Two parts make stamphog correct on these: + +- **Exploration sees the post-stack tree.** The LLM reviewer's + `Read`/`Grep`/`Glob` must run over a tree that already contains the parent + PRs' code, so symbols from a not-yet-merged parent resolve and aren't flagged + as broken imports. The diff itself is still computed `base_sha...head_sha`, + so the review is scoped to exactly this PR's changes. How the head tree is + materialized differs per runtime: + - **Action:** the workflow checks out master (hardcoded, so a PR can't swap + the review script), so the reviewer explores a detached **worktree at the + PR head** created just for stacked PRs. If the worktree cannot be created, + stamphog returns `ERROR` and retains the label rather than reviewing against + the wrong source tree. Heads with tracked symbolic links fail closed, so a + PR path cannot resolve outside the worktree. + - **Hosted:** the sandbox clones and checks out the PR head for every review, + so nothing extra is needed — `review_local.py` runs the pipeline with + `head_checkout=True` and no worktree is created. + - **Security (both runtimes):** the explored tree is PR-authored content. The + reviewer runs the Agent SDK with `setting_sources=[]` (isolation mode) plus + `strict_mcp_config`, so it does **not** load `.claude/settings.json` hooks + (command execution), `CLAUDE.md` (injected instructions), or `.mcp.json` + from the tree. Those files are still readable as untrusted _content_ under + the anti-injection notice — never as configuration. The diff scratch file is + created with `mkstemp` under an unpredictable name, so a tracked symlink in + the tree cannot redirect the write. - **Base retarget dismisses the stale approval.** When a stack's parent merges, the child PR is retargeted from the parent branch onto master, changing its effective diff **without a push** — so no `synchronize` fires and the normal push-dismiss path is skipped. Under the master ruleset (`dismiss_stale_reviews_on_push=false`), a prior bot approval would silently - carry onto the new base. The workflow listens for the `edited` event and, when + carry onto the new base. The Action listens for the `edited` event and, when the base changed, dismisses the bot approval and re-reviews against the new - base (if the label is still present). + base (if the label is still present); the approval step also rechecks the live + base and head SHAs right before posting. The hosted runtime does the same from + the webhook (`_retract_approvals_on_base_retarget`, then a fresh run) and + `post_verdict` rechecks the live base ref against the reviewed one. -The base commit of a stacked PR is its parent branch tip, which the master -checkout doesn't fetch by default — `github.ensure_commits` and the +The base commit of a stacked PR is its parent branch tip, which the Action's +master checkout doesn't fetch by default — `github.ensure_commits` and the `decide-delta` job both fetch the base branch so `git diff base_sha...head_sha` -and the dismiss-time merge classification resolve it. +and the dismiss-time merge classification resolve it. The hosted sandbox fetches +the base SHA explicitly during the clone. + +Known limitation (both runtimes): a parent branch force-push or rebase without +restacking the child emits no child PR event, so the child's approval is only +revalidated once the child is restacked or pushed. ## Tiers diff --git a/tools/pr-approval-agent/github.py b/tools/pr-approval-agent/github.py index c8091293bb6d..77d5244d0db4 100644 --- a/tools/pr-approval-agent/github.py +++ b/tools/pr-approval-agent/github.py @@ -5,8 +5,10 @@ Also handles team membership checks for the ownership gate. """ +import os import re import json +import tempfile import subprocess from collections.abc import Callable from dataclasses import dataclass, field @@ -36,6 +38,14 @@ class PRData: pr_reactions: list[dict] = field(default_factory=list) body: str = "" discussion: list[dict] = field(default_factory=list) + # The repo's default branch, so stacked-ness isn't tied to "master" (the hosted + # runtime reviews repos whose trunk is "main"). + default_branch: str = "master" + + @property + def stacked(self) -> bool: + """True when the PR targets another PR's branch rather than the repo's trunk.""" + return self.base_ref != self.default_branch @property def file_paths(self) -> list[str]: @@ -442,13 +452,27 @@ def _git_diff_files(base_sha: str, head_sha: str, repo_root: Path) -> list[dict] return files -def write_pr_diff(base_sha: str, head_sha: str, dest: Path, repo_root: Path) -> Path: - """Write the base...head PR diff to `dest` from the local checkout. +def new_diff_file(directory: Path) -> Path: + """Create a fresh, empty diff file under an unpredictable name inside ``directory``. + + The directory can be PR-authored (the hosted sandbox's head checkout, or + the Action's stacked-PR worktree), where a predictable name could be a + tracked symlink redirecting the write. ``mkstemp`` creates a new regular + file, so PR content cannot redirect it. Callers own the cleanup. + """ + fd, path = tempfile.mkstemp(prefix=".pr-review-diff-", suffix=".patch", dir=directory) + os.close(fd) + return Path(path) + + +def write_pr_diff(base_sha: str, head_sha: str, repo_root: Path) -> Path: + """Write the base...head PR diff to a fresh file in the checkout and return its path. Shared by the reviewer (feeds the LLM the diff to read) and the familiarity signal (parses the same diff for base-side modified line ranges), so the `git diff` invocation lives in one place. """ + dest = new_diff_file(repo_root) result = subprocess.run( ["git", "diff", f"{base_sha}...{head_sha}"], capture_output=True, @@ -599,6 +623,7 @@ def fetch_pr(pr_number: int, repo: str, repo_root: Path | None = None) -> PRData print(f"warning: discussion fetch failed ({exc}); continuing with no discussion context") # noqa: T201 base_ref = pr["base"]["ref"] + default_branch = pr["base"]["repo"]["default_branch"] base_sha = pr["base"]["sha"] head_sha = pr["head"]["sha"] check_runs_resp = _gh_api(f"repos/{repo}/commits/{head_sha}/check-runs") @@ -629,6 +654,7 @@ def fetch_pr(pr_number: int, repo: str, repo_root: Path | None = None) -> PRData pr_reactions=pr_reactions, body=pr.get("body") or "", discussion=discussion, + default_branch=default_branch, ) diff --git a/tools/pr-approval-agent/review_local.py b/tools/pr-approval-agent/review_local.py index 110c4edacef4..74a21c6909ff 100644 --- a/tools/pr-approval-agent/review_local.py +++ b/tools/pr-approval-agent/review_local.py @@ -124,8 +124,13 @@ def _build_pr_data(context: dict) -> PRData: """ pr = context.get("pr") or {} user = pr.get("user") or {} - base_sha = context.get("base_sha") or (pr.get("base") or {}).get("sha") or "" + base = pr.get("base") or {} + base_sha = context.get("base_sha") or base.get("sha") or "" head_sha = context.get("head_sha") or (pr.get("head") or {}).get("sha") or "" + # Both feed PRData.stacked (the stacked-PR prompt note). A lean context without them reads as + # non-stacked, matching the Action's default. + default_branch = (base.get("repo") or {}).get("default_branch") or "master" + base_ref = base.get("ref") or default_branch files = _git_diff_files(base_sha, head_sha, REPO_ROOT) if not files: @@ -198,6 +203,7 @@ def _build_pr_data(context: dict) -> PRData: mergeable_state=pr.get("mergeable_state") or "unknown", author=user.get("login") or "", labels=[label.get("name", "") for label in pr.get("labels") or []], + base_ref=base_ref, base_sha=base_sha, head_sha=head_sha, files=files, @@ -208,6 +214,7 @@ def _build_pr_data(context: dict) -> PRData: pr_reactions=pr_reactions, body=pr.get("body") or "", discussion=_normalize_discussion_for_prompt(context.get("discussion") or []), + default_branch=default_branch, ) @@ -315,7 +322,11 @@ def run(context: dict) -> dict: """Run the full offline review and return the to_dict() contract.""" # The hosted server sets self_driving_review only for PRs it verified came from a self-driving # Inbox implementation run. Action contexts never carry it, so bot authors are refused as before. - pipeline = Pipeline(0, context.get("repo") or "", self_driving=bool(context.get("self_driving_review"))) + # head_checkout: the sandbox clones and checks out the PR head before this runs (see the server's + # _clone_pr), so parent-PR symbols already resolve for stacked PRs and no worktree is needed. + pipeline = Pipeline( + 0, context.get("repo") or "", self_driving=bool(context.get("self_driving_review")), head_checkout=True + ) pipeline.pr = _build_pr_data(context) if pipeline.pr.author_is_bot and not pipeline.self_driving: diff --git a/tools/pr-approval-agent/review_pr.py b/tools/pr-approval-agent/review_pr.py index d0d031d10134..71d63e78ef4e 100644 --- a/tools/pr-approval-agent/review_pr.py +++ b/tools/pr-approval-agent/review_pr.py @@ -200,7 +200,14 @@ class Pipeline: """Orchestrates the full PR review: fetch → classify → gates → LLM review.""" def __init__( - self, pr_number: int, repo: str, *, dry_run: bool = False, verbose: bool = False, self_driving: bool = False + self, + pr_number: int, + repo: str, + *, + dry_run: bool = False, + verbose: bool = False, + self_driving: bool = False, + head_checkout: bool = False, ): self.pr_number = pr_number self.repo = repo @@ -210,6 +217,10 @@ def __init__( # implementation run. It relaxes two gates (bot author, draft) and swaps author trust for # task provenance. self.self_driving = self_driving + # True when REPO_ROOT already holds the PR head (the hosted sandbox clones and checks out + # the head for every review). The Action reviews from a trunk checkout, so a stacked PR + # needs a separate head worktree there — see _pr_head_worktree. + self.head_checkout = head_checkout self._wait_refetched_pr = False self.pr: PRData | None = None self.provenance: CommitProvenance | None = None @@ -559,9 +570,7 @@ def _ensure_diff_path(self) -> Path: cleanup so the file never lingers in the repo working tree. """ if self._diff_path is None: - self._diff_path = write_pr_diff( - self.pr.base_sha, self.pr.head_sha, REPO_ROOT / ".pr-review-diff.patch", REPO_ROOT - ) + self._diff_path = write_pr_diff(self.pr.base_sha, self.pr.head_sha, REPO_ROOT) return self._diff_path def _run_gates(self) -> None: @@ -704,22 +713,24 @@ def _check_tier(self) -> tuple[bool, str]: @contextmanager def _pr_head_worktree(self): - """Yield a detached worktree at the PR head, or None for non-stacked PRs. - - Only stacked PRs need this: their head contains code from parent PRs - that aren't on the base branch yet, so without materializing the head - tree those parents' symbols look like broken imports and the reviewer - false-refuses. A non-stacked PR (base is master) reviews from the master - checkout exactly as before — yield None and skip the full-tree checkout. - The main checkout stays master (the workflow hardcodes that so a PR can't - swap the review script), so the worktree is the only place the head tree - is materialized. Cleaned up on exit; stacked PRs fail closed if creation - fails rather than reviewing against the wrong source tree. + """Yield a detached worktree at the PR head, or None when none is needed. + + Only stacked PRs reviewed from a trunk checkout need this: their head + contains code from parent PRs that aren't on the base branch yet, so + without materializing the head tree those parents' symbols look like + broken imports and the reviewer false-refuses. A non-stacked PR reviews + from the trunk checkout exactly as before, and a runtime whose checkout + already IS the head (hosted sandbox, head_checkout=True) needs nothing + extra — both yield None and skip the full-tree checkout. In the Action + the main checkout stays master (the workflow hardcodes that so a PR + can't swap the review script), so the worktree is the only place the + head tree is materialized. Cleaned up on exit; stacked PRs fail closed + if creation fails rather than reviewing against the wrong source tree. SECURITY: the worktree is PR-authored content; isolation from it as *configuration* is enforced by setting_sources=[] in Reviewer. """ - if self.pr.base_ref == "master": + if self.head_checkout or not self.pr.stacked: yield None return diff --git a/tools/pr-approval-agent/reviewer.py b/tools/pr-approval-agent/reviewer.py index 5330930e25f0..c47b48ea9958 100644 --- a/tools/pr-approval-agent/reviewer.py +++ b/tools/pr-approval-agent/reviewer.py @@ -9,14 +9,13 @@ import json import shutil import asyncio -import tempfile import textwrap from pathlib import Path from claude_agent_sdk import ClaudeAgentOptions, ResultMessage, query from claude_agent_sdk.types import AssistantMessage, ToolUseBlock from gateway import analytics_extra_properties, gateway_env, resolve_gateway_config -from github import PRData, write_pr_diff +from github import PRData, new_diff_file, write_pr_diff from policy import _sanitize_untrusted, review_guidance_path, steering_path from version import STAMPHOG_VERSION @@ -302,14 +301,8 @@ def review(self, pr: PRData, classification: dict, gate_context: dict, diff_path return asyncio.run(self._review(pr, classification, gate_context, diff_path)) def _copy_diff_into_explore_root(self, diff_path: Path) -> Path: - """Copy the diff to a runner-created path the agent can read. - - A predictable worktree path could be a tracked symlink. ``mkstemp`` - creates a fresh regular file, so PR content cannot redirect this copy. - """ - fd, copied_path = tempfile.mkstemp(prefix=".pr-review-diff-", suffix=".patch", dir=self.explore_root) - os.close(fd) - copied_diff_path = Path(copied_path) + """Copy the diff to a runner-created path the agent can read (see new_diff_file).""" + copied_diff_path = new_diff_file(self.explore_root) try: shutil.copyfile(diff_path, copied_diff_path) except OSError: @@ -343,7 +336,9 @@ async def _review( disallowed_tools=["Write", "Edit", "NotebookEdit", "Bash", "Agent", "WebFetch", "WebSearch"], cwd=str(self.explore_root), # SECURITY: explore_root holds PR-authored content (a worktree at - # the PR head for stacked PRs). With the default (None) the SDK + # the PR head for stacked PRs in the Action; the whole checkout in + # the hosted sandbox, which clones the head for every review). With + # the default (None) the SDK # loads filesystem settings from cwd like the CLI does — including # .claude/settings.json hooks (arbitrary command execution) and # CLAUDE.md (injected as instructions). A PR could ship either. @@ -493,8 +488,7 @@ def _log_tool_call(self, block: ToolUseBlock) -> None: def _write_diff_file(self, pr: PRData) -> Path: """Write the PR diff to a temp file so the LLM can Read it on demand.""" - diff_path = self.repo_root / ".pr-review-diff.patch" - return write_pr_diff(pr.base_sha, pr.head_sha, diff_path, self.repo_root) + return write_pr_diff(pr.base_sha, pr.head_sha, self.repo_root) def _build_review_prompt(self, pr: PRData, cl: dict, gate_context: dict, diff_path: Path) -> str: safe_title = _sanitize_untrusted(pr.title, max_len=200) @@ -585,12 +579,12 @@ def _build_review_prompt(self, pr: PRData, cl: dict, gate_context: dict, diff_pa # For a stacked PR the working tree is the PR head, so parent-PR symbols # resolve in Read/Grep/Glob though absent from the diff; tell the agent. - if pr.base_ref != "master": + if pr.stacked: constraint += ( - f"\nStacked PR: this targets `{pr.base_ref}`, not master. The working tree reflects the " - "codebase as it will look after the whole stack lands, so symbols defined in parent PRs " - "resolve via Read/Grep/Glob even though they're absent from the diff below. Review only the " - "diff's changes; do not flag imports or references that resolve in the tree as missing." + f"\nStacked PR: this targets `{pr.base_ref}`, not `{pr.default_branch}`. The working tree " + "reflects the codebase as it will look after the whole stack lands, so symbols defined in " + "parent PRs resolve via Read/Grep/Glob even though they're absent from the diff below. Review " + "only the diff's changes; do not flag imports or references that resolve in the tree as missing." ) file_list = "\n".join( diff --git a/tools/pr-approval-agent/test_review_local.py b/tools/pr-approval-agent/test_review_local.py index c90af4324b7c..ae68d335d587 100644 --- a/tools/pr-approval-agent/test_review_local.py +++ b/tools/pr-approval-agent/test_review_local.py @@ -12,6 +12,7 @@ sys.modules.setdefault("claude_agent_sdk", MagicMock()) sys.modules.setdefault("claude_agent_sdk.types", MagicMock()) +import review_pr # noqa: E402 import review_local # noqa: E402 from review_pr import Pipeline # noqa: E402 @@ -313,3 +314,67 @@ def fake_llm(self, gate_verdict: str) -> None: prerequisites = next(g for g in result["gates"] if g["gate"] == "prerequisites") assert prerequisites["passed"] is True # the draft issue is carved out for this run assert result["classification"]["self_driving"] is True # provenance rides into the output contract + + +def _stacked_context(base_ref: str, default_branch: str) -> dict: + return { + "repo": "PostHog/posthog", + "head_sha": "abc123", + "base_sha": "def456", + "pr": { + "number": 11, + "title": "feat: child of a stack", + "state": "OPEN", + "draft": False, + "user": {"login": "author", "type": "User"}, + "base": {"ref": base_ref, "sha": "def456", "repo": {"default_branch": default_branch}}, + }, + } + + +@pytest.mark.parametrize( + "base_ref, default_branch, expect_stacked", + [ + pytest.param("master", "master", False, id="trunk-pr"), + pytest.param("feat/parent", "master", True, id="stacked-on-a-parent-branch"), + pytest.param("main", "main", False, id="trunk-named-main"), + ], +) +def test_stacked_detection_follows_the_repo_default_branch( + monkeypatch, base_ref: str, default_branch: str, expect_stacked: bool +) -> None: + # The hosted runtime reviews repos whose trunk is "main"; a hardcoded "master" would tag every + # PR there as stacked and mis-brief the reviewer. + monkeypatch.setattr(review_local, "_git_diff_files", lambda *a, **k: []) + + pr = review_local._build_pr_data(_stacked_context(base_ref, default_branch)) + + assert pr.stacked is expect_stacked + + +def test_hosted_stacked_review_never_creates_a_worktree(monkeypatch) -> None: + # The sandbox clones and checks out the PR head before the engine runs, so parent-PR symbols + # already resolve. Reviving the Action's stacked-PR worktree here would be a wasted full-tree + # checkout per stacked review, plus its symlink-rejection failure mode. + monkeypatch.setattr(review_local, "_git_diff_files", lambda *a, **k: []) + real_run = review_pr.subprocess.run + + def guarded_run(cmd, *args, **kwargs): + assert "worktree" not in cmd, f"hosted review must not create a worktree: {cmd}" + return real_run(cmd, *args, **kwargs) + + monkeypatch.setattr(review_pr.subprocess, "run", guarded_run) + seen: dict = {} + + def fake_review(self, pr, classification, gate_context, diff_path=None): + seen["explore_root"] = self.explore_root + seen["stacked"] = pr.stacked + return {"verdict": "APPROVE", "reasoning": "ok", "risk": "low", "issues": []} + + monkeypatch.setattr(review_pr.Reviewer, "review", fake_review) + + result = review_local.run(_stacked_context("feat/parent", "master")) + + assert result["final_verdict"] == "APPROVED" + assert seen["stacked"] is True + assert seen["explore_root"] == review_pr.REPO_ROOT diff --git a/tools/pr-approval-agent/test_review_pr.py b/tools/pr-approval-agent/test_review_pr.py index 89599f249120..9f34a14d2cd2 100644 --- a/tools/pr-approval-agent/test_review_pr.py +++ b/tools/pr-approval-agent/test_review_pr.py @@ -29,7 +29,7 @@ def _no_live_team_lookup(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(review_pr, "compute_familiarity", lambda **_k: None) -def _fake_pr(head_sha: str, base_ref: str = "master") -> PRData: +def _fake_pr(head_sha: str, base_ref: str = "master", default_branch: str = "master") -> PRData: return PRData( number=1, repo="PostHog/posthog", @@ -46,6 +46,7 @@ def _fake_pr(head_sha: str, base_ref: str = "master") -> PRData: reviews=[], review_comments=[], check_runs=[], + default_branch=default_branch, ) @@ -130,13 +131,16 @@ def review(self, *args: object, **kwargs: object) -> dict: ], ) def test_backend_failure_yields_error_except_when_gates_deny( - monkeypatch: pytest.MonkeyPatch, gate_verdict: str, expected_final: str + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, gate_verdict: str, expected_final: str ) -> None: """A failed LLM call must surface as ERROR (label retained) unless gates already DENIED — a deterministic denial outranks an unavailable reviewer.""" monkeypatch.setattr(review_pr, "Reviewer", _RaisingReviewer) monkeypatch.setattr(review_pr.time, "sleep", lambda _s: None) monkeypatch.setattr(review_pr, "_POSTHOG_AVAILABLE", False) + # _llm_review is called directly, so run()'s diff cleanup never happens — keep the scratch + # diff out of the real checkout. + monkeypatch.setattr(review_pr, "REPO_ROOT", tmp_path) pipeline = Pipeline(pr_number=1, repo="PostHog/posthog") pipeline.pr = _fake_pr(head_sha="abc123") @@ -161,7 +165,9 @@ def test_backend_failure_yields_error_except_when_gates_deny( ("DENIED", "REFUSED"), ], ) -def test_turn_limit_error_not_retried(monkeypatch: pytest.MonkeyPatch, gate_verdict: str, expected_final: str) -> None: +def test_turn_limit_error_not_retried( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, gate_verdict: str, expected_final: str +) -> None: """A turn-limit error is non-retryable and should give a clear message about complexity rather than blaming infrastructure. When gates DENIED, the deterministic denial still outranks the error.""" @@ -177,6 +183,7 @@ def counting_review(self, *args, **kwargs): monkeypatch.setattr(_TurnLimitReviewer, "review", counting_review) monkeypatch.setattr(review_pr.time, "sleep", lambda _s: None) monkeypatch.setattr(review_pr, "_POSTHOG_AVAILABLE", False) + monkeypatch.setattr(review_pr, "REPO_ROOT", tmp_path) pipeline = Pipeline(pr_number=1, repo="PostHog/posthog") pipeline.pr = _fake_pr(head_sha="abc123") @@ -555,17 +562,28 @@ def unavailable_worktree(): assert pipeline.reviewer_output["issues"] == ["checkout unavailable"] -def test_pr_head_worktree_skipped_for_non_stacked(monkeypatch: pytest.MonkeyPatch) -> None: - """A non-stacked PR (base is master) reviews from master — no worktree, - so git is never invoked and the full-tree checkout cost is skipped.""" +@pytest.mark.parametrize( + "base_ref, default_branch, head_checkout", + [ + pytest.param("master", "master", False, id="trunk-is-master"), + pytest.param("main", "main", False, id="trunk-is-main"), + pytest.param("feat/parent-branch", "master", True, id="hosted-checkout-already-at-head"), + ], +) +def test_pr_head_worktree_skipped_when_not_needed( + monkeypatch: pytest.MonkeyPatch, base_ref: str, default_branch: str, head_checkout: bool +) -> None: + """No worktree — so git is never invoked and the full-tree checkout cost is + skipped — for a PR targeting the repo's trunk (whatever it is named), and + for a runtime whose checkout already is the PR head (the hosted sandbox).""" def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: - raise AssertionError(f"non-stacked PR must not touch git: {cmd}") + raise AssertionError(f"must not touch git: {cmd}") monkeypatch.setattr(review_pr.subprocess, "run", fake_run) - pipeline = Pipeline(pr_number=7, repo="PostHog/posthog") - pipeline.pr = _fake_pr(head_sha="cafe123", base_ref="master") + pipeline = Pipeline(pr_number=7, repo="PostHog/posthog", head_checkout=head_checkout) + pipeline.pr = _fake_pr(head_sha="cafe123", base_ref=base_ref, default_branch=default_branch) with pipeline._pr_head_worktree() as explore_root: assert explore_root is None diff --git a/tools/pr-approval-agent/test_reviewer.py b/tools/pr-approval-agent/test_reviewer.py index f13d1aa6c5a2..784cddbc33b6 100644 --- a/tools/pr-approval-agent/test_reviewer.py +++ b/tools/pr-approval-agent/test_reviewer.py @@ -212,20 +212,23 @@ def test_copy_diff_into_explore_root_cannot_follow_pr_symlink(tmp_path: Path) -> @pytest.mark.parametrize( - "base_ref, expect_stack_note", + "base_ref, default_branch, expect_stack_note", [ - ("master", False), - ("query-validations", True), + ("master", "master", False), + ("query-validations", "master", True), + # Stacked-ness keys off the repo's own trunk, not a hardcoded "master". + ("main", "main", False), + ("master", "main", True), ], ) -def test_prompt_stack_note(base_ref: str, expect_stack_note: bool) -> None: - # A stacked PR (base != master) gets a note telling the agent that - # parent-PR symbols resolve in the tree and aren't missing. - prompt = _prompt(_pr(base_ref=base_ref)) +def test_prompt_stack_note(base_ref: str, default_branch: str, expect_stack_note: bool) -> None: + # A stacked PR (base != the repo's default branch) gets a note telling the + # agent that parent-PR symbols resolve in the tree and aren't missing. + prompt = _prompt(_pr(base_ref=base_ref, default_branch=default_branch)) assert ("Stacked PR" in prompt) is expect_stack_note if expect_stack_note: - assert base_ref in prompt + assert f"targets `{base_ref}`, not `{default_branch}`" in prompt def _fake_stamphog_dir(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, guidance: str) -> Path: From a1fe0f1292bc5def0abc5ed550cdebb0a6808b5b Mon Sep 17 00:00:00 2001 From: Julian Bez Date: Mon, 17 Aug 2026 11:03:34 +0200 Subject: [PATCH 037/289] chore(stamphog): use semantic line breaks in the stacked-PR docs --- products/stamphog/AGENTS.md | 12 ++--- products/stamphog/README.md | 7 ++- tools/pr-approval-agent/README.md | 78 +++++++++++-------------------- 3 files changed, 37 insertions(+), 60 deletions(-) diff --git a/products/stamphog/AGENTS.md b/products/stamphog/AGENTS.md index 8f22cdb37edb..6b4a7ef87cb6 100644 --- a/products/stamphog/AGENTS.md +++ b/products/stamphog/AGENTS.md @@ -49,10 +49,8 @@ A newer relevant delivery supersedes older non-terminal runs. Rules that keep th - Terminal states (`TERMINAL_STATUSES` in `facade/enums.py`) are never rewritten — `mark_review_failed` must not clobber a delivered outcome, and terminal saves are conditional (`.exclude(status=SUPERSEDED).update(...)`), never plain `save()`. -- `post_verdict` guards before ANY GitHub write: superseded status, current head vs run head, - current base ref vs the reviewed one (a retarget rewrites the diff with the head unchanged, and - the retarget delivery can trail the activity), and a last fresh status read. Losing the final - conditional update means dismiss-your-own-approval, not "log and return". +- `post_verdict` guards before ANY GitHub write: superseded status, current head vs run head, current base ref vs the reviewed one (a retarget rewrites the diff with the head unchanged, and the retarget delivery can trail the activity), and a last fresh status read. + Losing the final conditional update means dismiss-your-own-approval, not "log and return". - Out-of-order webhook deliveries are dropped by the `payload_updated_at` clock — checked before the transaction AND re-checked under the row lock, and the descriptive-field refresh is gated on the same clock inside the UPDATE's WHERE clause. @@ -77,10 +75,8 @@ add a read-then-act path, pin it; this class of bug has been found on five separ `STAMPHOG_SANDBOX_EXTRA_EGRESS_DOMAINS`, not code edits. - Everything posted to GitHub goes through `_scrub_credentials` AND `_neutralize_active_markdown` (GitHub's camo proxy auto-fetches images — a markdown image URL is an exfiltration channel). -- The sandbox checkout is the PR head, so the engine's Agent SDK session runs with - `setting_sources=[]` + `strict_mcp_config` (reviewer.py): a PR-shipped `.claude/settings.json` - hook, `CLAUDE.md`, or `.mcp.json` is readable as untrusted content, never loaded as - configuration. Don't reintroduce filesystem settings discovery there. +- The sandbox checkout is the PR head, so the engine's Agent SDK session runs with `setting_sources=[]` + `strict_mcp_config` (reviewer.py): a PR-shipped `.claude/settings.json` hook, `CLAUDE.md`, or `.mcp.json` is readable as untrusted content, never loaded as configuration. + Don't reintroduce filesystem settings discovery there. ## The self-driving inbox carve-out (the one exception to the bot-author refusal) diff --git a/products/stamphog/README.md b/products/stamphog/README.md index 1d37080eeb98..4a1ef060a8b1 100644 --- a/products/stamphog/README.md +++ b/products/stamphog/README.md @@ -19,7 +19,12 @@ On top of reviews, a repo can enable a daily Slack digest of its merged PRs (`ba ## Stacked PRs -A stacked PR targets its parent's branch, not the repo's default branch, and depends on parent code that hasn't merged yet. The sandbox clones and checks out the PR head for every review, so the reviewer's Read/Grep/Glob already see the post-stack tree and parent symbols resolve; the engine is told the checkout is the head (`head_checkout=True`) so it never builds the Action's separate head worktree, and the prompt flags the PR as stacked (`PRData.stacked`, keyed on the repo's actual default branch). The diff stays scoped `base...head`. When the parent merges and GitHub retargets the child onto the default branch, the diff changes without a push: the webhook path retracts the standing approval and queues a fresh run, and `post_verdict` rechecks the live base against the reviewed one before posting. Engine details: [`tools/pr-approval-agent/README.md`](../../tools/pr-approval-agent/README.md#stacked-prs-graphite--git-stacks). +A stacked PR targets its parent's branch, not the repo's default branch, and depends on parent code that hasn't merged yet. +The sandbox clones and checks out the PR head for every review, so the reviewer's Read/Grep/Glob already see the post-stack tree and parent symbols resolve. +The engine is told the checkout is the head (`head_checkout=True`) so it never builds the Action's separate head worktree, and the prompt flags the PR as stacked (`PRData.stacked`, keyed on the repo's actual default branch). +The diff stays scoped `base...head`. +When the parent merges and GitHub retargets the child onto the default branch, the diff changes without a push: the webhook path retracts the standing approval and queues a fresh run, and `post_verdict` rechecks the live base against the reviewed one before posting. +Engine details: [`tools/pr-approval-agent/README.md`](../../tools/pr-approval-agent/README.md#stacked-prs-graphite--git-stacks). ## Configuration diff --git a/tools/pr-approval-agent/README.md b/tools/pr-approval-agent/README.md index f4498f3b0f59..15546164cf60 100644 --- a/tools/pr-approval-agent/README.md +++ b/tools/pr-approval-agent/README.md @@ -147,57 +147,33 @@ Every other verdict (REFUSED, ESCALATE, WAIT, ERROR) goes into a single sticky c ## Stacked PRs (Graphite / git stacks) -A stacked PR targets its parent branch, not the repo's default branch, and -depends on code the parent introduces but hasn't merged yet. `PRData.stacked` -(`base_ref != default_branch`, so repos whose trunk is `main` work too) drives -the handling; the reviewer prompt tells the agent it is looking at a stacked -PR. Two parts make stamphog correct on these: - -- **Exploration sees the post-stack tree.** The LLM reviewer's - `Read`/`Grep`/`Glob` must run over a tree that already contains the parent - PRs' code, so symbols from a not-yet-merged parent resolve and aren't flagged - as broken imports. The diff itself is still computed `base_sha...head_sha`, - so the review is scoped to exactly this PR's changes. How the head tree is - materialized differs per runtime: - - **Action:** the workflow checks out master (hardcoded, so a PR can't swap - the review script), so the reviewer explores a detached **worktree at the - PR head** created just for stacked PRs. If the worktree cannot be created, - stamphog returns `ERROR` and retains the label rather than reviewing against - the wrong source tree. Heads with tracked symbolic links fail closed, so a - PR path cannot resolve outside the worktree. - - **Hosted:** the sandbox clones and checks out the PR head for every review, - so nothing extra is needed — `review_local.py` runs the pipeline with - `head_checkout=True` and no worktree is created. - - **Security (both runtimes):** the explored tree is PR-authored content. The - reviewer runs the Agent SDK with `setting_sources=[]` (isolation mode) plus - `strict_mcp_config`, so it does **not** load `.claude/settings.json` hooks - (command execution), `CLAUDE.md` (injected instructions), or `.mcp.json` - from the tree. Those files are still readable as untrusted _content_ under - the anti-injection notice — never as configuration. The diff scratch file is - created with `mkstemp` under an unpredictable name, so a tracked symlink in - the tree cannot redirect the write. - -- **Base retarget dismisses the stale approval.** When a stack's parent merges, - the child PR is retargeted from the parent branch onto master, changing its - effective diff **without a push** — so no `synchronize` fires and the normal - push-dismiss path is skipped. Under the master ruleset - (`dismiss_stale_reviews_on_push=false`), a prior bot approval would silently - carry onto the new base. The Action listens for the `edited` event and, when - the base changed, dismisses the bot approval and re-reviews against the new - base (if the label is still present); the approval step also rechecks the live - base and head SHAs right before posting. The hosted runtime does the same from - the webhook (`_retract_approvals_on_base_retarget`, then a fresh run) and - `post_verdict` rechecks the live base ref against the reviewed one. - -The base commit of a stacked PR is its parent branch tip, which the Action's -master checkout doesn't fetch by default — `github.ensure_commits` and the -`decide-delta` job both fetch the base branch so `git diff base_sha...head_sha` -and the dismiss-time merge classification resolve it. The hosted sandbox fetches -the base SHA explicitly during the clone. - -Known limitation (both runtimes): a parent branch force-push or rebase without -restacking the child emits no child PR event, so the child's approval is only -revalidated once the child is restacked or pushed. +A stacked PR targets its parent branch, not the repo's default branch, and depends on code the parent introduces but hasn't merged yet. +`PRData.stacked` (`base_ref != default_branch`, so repos whose trunk is `main` work too) drives the handling; the reviewer prompt tells the agent it is looking at a stacked PR. +Two parts make stamphog correct on these: + +- **Exploration sees the post-stack tree.** + The LLM reviewer's `Read`/`Grep`/`Glob` must run over a tree that already contains the parent PRs' code, so symbols from a not-yet-merged parent resolve and aren't flagged as broken imports. + The diff itself is still computed `base_sha...head_sha`, so the review is scoped to exactly this PR's changes. + How the head tree is materialized differs per runtime: + - **Action:** the workflow checks out master (hardcoded, so a PR can't swap the review script), so the reviewer explores a detached **worktree at the PR head** created just for stacked PRs. + If the worktree cannot be created, stamphog returns `ERROR` and retains the label rather than reviewing against the wrong source tree. + Heads with tracked symbolic links fail closed, so a PR path cannot resolve outside the worktree. + - **Hosted:** the sandbox clones and checks out the PR head for every review, so nothing extra is needed — `review_local.py` runs the pipeline with `head_checkout=True` and no worktree is created. + - **Security (both runtimes):** the explored tree is PR-authored content. + The reviewer runs the Agent SDK with `setting_sources=[]` (isolation mode) plus `strict_mcp_config`, so it does **not** load `.claude/settings.json` hooks (command execution), `CLAUDE.md` (injected instructions), or `.mcp.json` from the tree. + Those files are still readable as untrusted _content_ under the anti-injection notice — never as configuration. + The diff scratch file is created with `mkstemp` under an unpredictable name, so a tracked symlink in the tree cannot redirect the write. + +- **Base retarget dismisses the stale approval.** + When a stack's parent merges, the child PR is retargeted from the parent branch onto master, changing its effective diff **without a push** — so no `synchronize` fires and the normal push-dismiss path is skipped. + Under the master ruleset (`dismiss_stale_reviews_on_push=false`), a prior bot approval would silently carry onto the new base. + The Action listens for the `edited` event and, when the base changed, dismisses the bot approval and re-reviews against the new base (if the label is still present); the approval step also rechecks the live base and head SHAs right before posting. + The hosted runtime does the same from the webhook (`_retract_approvals_on_base_retarget`, then a fresh run) and `post_verdict` rechecks the live base ref against the reviewed one. + +The base commit of a stacked PR is its parent branch tip, which the Action's master checkout doesn't fetch by default — `github.ensure_commits` and the `decide-delta` job both fetch the base branch so `git diff base_sha...head_sha` and the dismiss-time merge classification resolve it. +The hosted sandbox fetches the base SHA explicitly during the clone. + +Known limitation (both runtimes): a parent branch force-push or rebase without restacking the child emits no child PR event, so the child's approval is only revalidated once the child is restacked or pushed. ## Tiers From ab34c42899a0460133ad455ee66d9014902d84cf Mon Sep 17 00:00:00 2001 From: Julian Bez Date: Mon, 17 Aug 2026 11:06:04 +0200 Subject: [PATCH 038/289] fix(stamphog): compare the reviewed base SHA too before posting a verdict Ref-only comparison misses a parent branch moving under a stacked PR while keeping its name. GitHub pins base.sha at the last PR event rather than tracking the trunk tip, so the SHA only differs when the PR itself was touched, which is exactly when the reviewed diff is stale. Mirrors the Action's pre-approval base/head SHA recheck. --- products/stamphog/AGENTS.md | 2 +- products/stamphog/README.md | 2 +- .../stamphog/backend/temporal/activities.py | 21 +++++++++++++----- .../backend/tests/test_integration.py | 22 ++++++++++++++----- tools/pr-approval-agent/README.md | 2 +- 5 files changed, 35 insertions(+), 14 deletions(-) diff --git a/products/stamphog/AGENTS.md b/products/stamphog/AGENTS.md index 6b4a7ef87cb6..277d214a37f1 100644 --- a/products/stamphog/AGENTS.md +++ b/products/stamphog/AGENTS.md @@ -49,7 +49,7 @@ A newer relevant delivery supersedes older non-terminal runs. Rules that keep th - Terminal states (`TERMINAL_STATUSES` in `facade/enums.py`) are never rewritten — `mark_review_failed` must not clobber a delivered outcome, and terminal saves are conditional (`.exclude(status=SUPERSEDED).update(...)`), never plain `save()`. -- `post_verdict` guards before ANY GitHub write: superseded status, current head vs run head, current base ref vs the reviewed one (a retarget rewrites the diff with the head unchanged, and the retarget delivery can trail the activity), and a last fresh status read. +- `post_verdict` guards before ANY GitHub write: superseded status, current head vs run head, current base (ref and SHA) vs the reviewed one (a retarget, or a parent branch moving under a stacked PR, rewrites the diff with the head unchanged, and the retarget delivery can trail the activity), and a last fresh status read. Losing the final conditional update means dismiss-your-own-approval, not "log and return". - Out-of-order webhook deliveries are dropped by the `payload_updated_at` clock — checked before the transaction AND re-checked under the row lock, and the descriptive-field refresh is gated on diff --git a/products/stamphog/README.md b/products/stamphog/README.md index 4a1ef060a8b1..9f57c63a72e7 100644 --- a/products/stamphog/README.md +++ b/products/stamphog/README.md @@ -23,7 +23,7 @@ A stacked PR targets its parent's branch, not the repo's default branch, and dep The sandbox clones and checks out the PR head for every review, so the reviewer's Read/Grep/Glob already see the post-stack tree and parent symbols resolve. The engine is told the checkout is the head (`head_checkout=True`) so it never builds the Action's separate head worktree, and the prompt flags the PR as stacked (`PRData.stacked`, keyed on the repo's actual default branch). The diff stays scoped `base...head`. -When the parent merges and GitHub retargets the child onto the default branch, the diff changes without a push: the webhook path retracts the standing approval and queues a fresh run, and `post_verdict` rechecks the live base against the reviewed one before posting. +When the parent merges and GitHub retargets the child onto the default branch, the diff changes without a push: the webhook path retracts the standing approval and queues a fresh run, and `post_verdict` rechecks the live base (ref and SHA) against the reviewed one before posting. Engine details: [`tools/pr-approval-agent/README.md`](../../tools/pr-approval-agent/README.md#stacked-prs-graphite--git-stacks). ## Configuration diff --git a/products/stamphog/backend/temporal/activities.py b/products/stamphog/backend/temporal/activities.py index 6f3239b658cd..2b83257d1801 100644 --- a/products/stamphog/backend/temporal/activities.py +++ b/products/stamphog/backend/temporal/activities.py @@ -673,14 +673,25 @@ def post_verdict(input: StamphogReviewInput) -> dict: current_head = ((current_pr.get("head") or {}).get("sha") or "").strip() # A base retarget (a stacked PR's parent merged, or a manual base switch) rewrites the reviewed # diff with the head SHA unchanged, so the head guard alone can't see it. The retarget delivery - # retracts approvals and queues a fresh run, but that delivery can trail this activity. - reviewed_base_ref = ((output.get("pr") or {}).get("base") or {}).get("ref") or "" - current_base_ref = ((current_pr.get("base") or {}).get("ref") or "").strip() + # retracts approvals and queues a fresh run, but that delivery can trail this activity. The SHA + # is compared too: GitHub pins base.sha at the last PR event rather than tracking the trunk tip, + # so it only moves when the PR itself was touched — the diff the sandbox reviewed is stale then. + reviewed_base = (output.get("pr") or {}).get("base") or {} + current_base = current_pr.get("base") or {} + reviewed_base_ref = reviewed_base.get("ref") or "" + reviewed_base_sha = reviewed_base.get("sha") or "" + current_base_ref = (current_base.get("ref") or "").strip() + current_base_sha = (current_base.get("sha") or "").strip() + base_ref_moved = bool(reviewed_base_ref and current_base_ref and reviewed_base_ref != current_base_ref) + base_sha_moved = bool(reviewed_base_sha and current_base_sha and reviewed_base_sha != current_base_sha) drift: tuple[str, str] | None = None if current_head and current_head != run.head_sha: drift = ("head_moved", f"head moved {run.head_sha} -> {current_head}") - elif reviewed_base_ref and current_base_ref and current_base_ref != reviewed_base_ref: - drift = ("base_retargeted", f"base retargeted {reviewed_base_ref} -> {current_base_ref}") + elif base_ref_moved or base_sha_moved: + drift = ( + "base_retargeted", + f"base moved {reviewed_base_ref}@{reviewed_base_sha} -> {current_base_ref}@{current_base_sha}", + ) if drift is not None: kind, detail = drift # Conditional: a retry after the terminal save already committed (e.g. the trailing digest diff --git a/products/stamphog/backend/tests/test_integration.py b/products/stamphog/backend/tests/test_integration.py index 1451f438597f..316c8221f26c 100644 --- a/products/stamphog/backend/tests/test_integration.py +++ b/products/stamphog/backend/tests/test_integration.py @@ -939,16 +939,26 @@ def test_retry_after_head_move_never_rewrites_a_terminal_run(team, stamphog_chai assert [w for w in recorder.github_writes if w["kind"] == "dismiss_review"] == [] +@pytest.mark.parametrize( + "live_base", + [ + pytest.param({"sha": "master-tip", "ref": "master"}, id="retargeted-to-master"), + pytest.param({"sha": "parent-tip-2", "ref": "feat/parent"}, id="same-ref-parent-moved"), + ], +) @pytest.mark.django_db(databases=PRODUCT_DATABASES) -def test_post_verdict_skips_when_the_base_was_retargeted_under_the_run(team, stamphog_chain: StamphogChain) -> None: - # A stacked PR's parent merged mid-review: the child is retargeted to master, which rewrites - # the reviewed diff while the head SHA stays put. The retarget delivery retracts and re-queues, - # but it can trail this activity — post_verdict must recheck the live base itself, or an - # approval for the old base..head diff lands on the new one. +def test_post_verdict_skips_when_the_base_moved_under_the_run( + team, stamphog_chain: StamphogChain, live_base: dict +) -> None: + # A stacked PR's parent merged mid-review (child retargeted to master), or the parent branch + # itself moved under the same ref: either rewrites the reviewed diff while the head SHA stays + # put. The retarget delivery retracts and re-queues, but it can trail this activity — + # post_verdict must recheck the live base itself, or an approval for the old base..head diff + # lands on the new one. repo_config = _repo_config(team.id) recorder = stamphog_chain.recorder head_sha = "sha119a" - live_pr = _pr_object(119, "devex-dev", head_sha) | {"base": {"sha": "master-tip", "ref": "master"}} + live_pr = _pr_object(119, "devex-dev", head_sha) | {"base": live_base} recorder.register_pr(REPO, 119, live_pr, _pr_files()) pull_request = PullRequest.objects.for_team(team.id).create( team_id=team.id, repo_config=repo_config, pr_number=119, author_login="devex-dev" diff --git a/tools/pr-approval-agent/README.md b/tools/pr-approval-agent/README.md index 15546164cf60..f008948f339f 100644 --- a/tools/pr-approval-agent/README.md +++ b/tools/pr-approval-agent/README.md @@ -168,7 +168,7 @@ Two parts make stamphog correct on these: When a stack's parent merges, the child PR is retargeted from the parent branch onto master, changing its effective diff **without a push** — so no `synchronize` fires and the normal push-dismiss path is skipped. Under the master ruleset (`dismiss_stale_reviews_on_push=false`), a prior bot approval would silently carry onto the new base. The Action listens for the `edited` event and, when the base changed, dismisses the bot approval and re-reviews against the new base (if the label is still present); the approval step also rechecks the live base and head SHAs right before posting. - The hosted runtime does the same from the webhook (`_retract_approvals_on_base_retarget`, then a fresh run) and `post_verdict` rechecks the live base ref against the reviewed one. + The hosted runtime does the same from the webhook (`_retract_approvals_on_base_retarget`, then a fresh run) and `post_verdict` rechecks the live base ref and SHA against the reviewed ones. The base commit of a stacked PR is its parent branch tip, which the Action's master checkout doesn't fetch by default — `github.ensure_commits` and the `decide-delta` job both fetch the base branch so `git diff base_sha...head_sha` and the dismiss-time merge classification resolve it. The hosted sandbox fetches the base SHA explicitly during the clone. From 20abc9feab617568e8446dc8f1717e9a8927fefa Mon Sep 17 00:00:00 2001 From: Julian Bez Date: Mon, 17 Aug 2026 11:08:17 +0200 Subject: [PATCH 039/289] fix(stamphog): clean up diff scratch files on every reviewer exit path An API error or cancellation mid-query skipped the unlink of the copied and owned diff files, leaving PR-authored content on the runner. --- tools/pr-approval-agent/reviewer.py | 63 +++++++++++++++-------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/tools/pr-approval-agent/reviewer.py b/tools/pr-approval-agent/reviewer.py index c47b48ea9958..31f33ff03692 100644 --- a/tools/pr-approval-agent/reviewer.py +++ b/tools/pr-approval-agent/reviewer.py @@ -435,36 +435,39 @@ async def _review( active_query = query structured_output = None - async for message in active_query(prompt=prompt, options=options, **posthog_kwargs): - if self.verbose: - print(f"\033[2m [{type(message).__name__}]\033[0m", flush=True) - if isinstance(message, ResultMessage): - if message.subtype == "error_max_structured_output_retries": - raise RuntimeError("Agent could not produce valid structured output after retries") - if getattr(message, "is_error", False): - # An API-level failure (auth, rate limit, overload, quota) surfaces - # here with subtype "success" and the real HTTP status in - # api_error_status. Raise with that detail now — otherwise the CLI - # process exits right after this message and the SDK's read loop - # replaces it with the generic, status-less "Claude Code returned - # an error result: success" once the exception reaches us anyway. - # getattr guards older SDK builds that lack these attributes. - api_status = getattr(message, "api_error_status", None) - status = f" (HTTP {api_status})" if api_status else "" - raise RuntimeError(f"Anthropic API error{status}: {message.result or message.subtype}") - if message.structured_output: - structured_output = message.structured_output - # Stamp the LLM verdict onto the trace properties - props["stamphog_llm_verdict"] = structured_output.get("verdict", "") - elif isinstance(message, AssistantMessage): - for block in message.content: - if isinstance(block, ToolUseBlock) and self.verbose: - self._log_tool_call(block) - - if copied_diff_path is not None: - copied_diff_path.unlink(missing_ok=True) - if owns_diff: - original_diff.unlink(missing_ok=True) + try: + async for message in active_query(prompt=prompt, options=options, **posthog_kwargs): + if self.verbose: + print(f"\033[2m [{type(message).__name__}]\033[0m", flush=True) + if isinstance(message, ResultMessage): + if message.subtype == "error_max_structured_output_retries": + raise RuntimeError("Agent could not produce valid structured output after retries") + if getattr(message, "is_error", False): + # An API-level failure (auth, rate limit, overload, quota) surfaces + # here with subtype "success" and the real HTTP status in + # api_error_status. Raise with that detail now — otherwise the CLI + # process exits right after this message and the SDK's read loop + # replaces it with the generic, status-less "Claude Code returned + # an error result: success" once the exception reaches us anyway. + # getattr guards older SDK builds that lack these attributes. + api_status = getattr(message, "api_error_status", None) + status = f" (HTTP {api_status})" if api_status else "" + raise RuntimeError(f"Anthropic API error{status}: {message.result or message.subtype}") + if message.structured_output: + structured_output = message.structured_output + # Stamp the LLM verdict onto the trace properties + props["stamphog_llm_verdict"] = structured_output.get("verdict", "") + elif isinstance(message, AssistantMessage): + for block in message.content: + if isinstance(block, ToolUseBlock) and self.verbose: + self._log_tool_call(block) + finally: + # Runs on every exit path (API error, cancellation): PR-authored diff copies must not + # linger on the runner. + if copied_diff_path is not None: + copied_diff_path.unlink(missing_ok=True) + if owns_diff: + original_diff.unlink(missing_ok=True) if structured_output is None: raise RuntimeError("Reviewer agent returned no structured output") From e6284632d5bd2bae1afd533310f03d6bede43cc9 Mon Sep 17 00:00:00 2001 From: Julian Bez Date: Mon, 17 Aug 2026 11:12:13 +0200 Subject: [PATCH 040/289] fix(stamphog): only reject symlinks a stacked PR adds, not the trunk's own The head-worktree symlink check rejected any tracked symlink, but the trunk carries dozens (CLAUDE.md -> AGENTS.md and friends), so every Action-side stacked review failed closed before reaching the reviewer. Diff the head's symlinks (path + blob) against the default branch and fail only on links the PR adds or repoints. The baseline is the default branch rather than the PR base, since a stacked PR's base is PR-authored too. --- tools/pr-approval-agent/README.md | 2 +- tools/pr-approval-agent/review_pr.py | 45 ++++++++++++++++------- tools/pr-approval-agent/test_review_pr.py | 38 +++++++++++++++---- 3 files changed, 63 insertions(+), 22 deletions(-) diff --git a/tools/pr-approval-agent/README.md b/tools/pr-approval-agent/README.md index f008948f339f..4515caacfbbb 100644 --- a/tools/pr-approval-agent/README.md +++ b/tools/pr-approval-agent/README.md @@ -157,7 +157,7 @@ Two parts make stamphog correct on these: How the head tree is materialized differs per runtime: - **Action:** the workflow checks out master (hardcoded, so a PR can't swap the review script), so the reviewer explores a detached **worktree at the PR head** created just for stacked PRs. If the worktree cannot be created, stamphog returns `ERROR` and retains the label rather than reviewing against the wrong source tree. - Heads with tracked symbolic links fail closed, so a PR path cannot resolve outside the worktree. + Symbolic links the PR adds or repoints (relative to the default branch's tree, which already carries trusted ones like `CLAUDE.md`) fail closed, so a PR path cannot resolve outside the worktree. - **Hosted:** the sandbox clones and checks out the PR head for every review, so nothing extra is needed — `review_local.py` runs the pipeline with `head_checkout=True` and no worktree is created. - **Security (both runtimes):** the explored tree is PR-authored content. The reviewer runs the Agent SDK with `setting_sources=[]` (isolation mode) plus `strict_mcp_config`, so it does **not** load `.claude/settings.json` hooks (command execution), `CLAUDE.md` (injected instructions), or `.mcp.json` from the tree. diff --git a/tools/pr-approval-agent/review_pr.py b/tools/pr-approval-agent/review_pr.py index 71d63e78ef4e..6ec6edb44721 100644 --- a/tools/pr-approval-agent/review_pr.py +++ b/tools/pr-approval-agent/review_pr.py @@ -711,6 +711,29 @@ def _check_tier(self) -> tuple[bool, str]: return True, f"T0 auto-approve: {summary}" return True, summary + @staticmethod + def _tracked_symlinks(rev: str) -> set[tuple[str, str]]: + """(path, blob) pairs of the symlinks tracked at ``rev``; same blob means same target.""" + try: + listing = subprocess.run( + ["git", "ls-tree", "-r", "--full-tree", rev], + capture_output=True, + text=True, + timeout=30, + cwd=REPO_ROOT, + ) + except subprocess.TimeoutExpired as exc: + raise WorktreeUnavailableError(f"symlink check timed out for {rev}") from exc + if listing.returncode != 0: + raise WorktreeUnavailableError(f"symlink check failed for {rev}: {listing.stderr.strip()}") + links: set[tuple[str, str]] = set() + for line in listing.stdout.splitlines(): + if not line.startswith("120000 "): + continue + meta, path = line.split("\t", 1) + links.add((path, meta.split()[2])) + return links + @contextmanager def _pr_head_worktree(self): """Yield a detached worktree at the PR head, or None when none is needed. @@ -735,20 +758,16 @@ def _pr_head_worktree(self): return worktree_dir = Path(tempfile.gettempdir()) / f"pr-review-{self.pr_number}-{uuid.uuid4().hex[:8]}" - try: - symlink_check = subprocess.run( - ["git", "ls-tree", "-r", "--full-tree", self.pr.head_sha], - capture_output=True, - text=True, - timeout=30, - cwd=REPO_ROOT, + # A symlink in the head tree can point outside the worktree, and the agent's Read follows + # it — so only symlinks the trunk already carries (same path, same target) are trusted; a + # stacked PR's base is PR-authored too, so the baseline is the default branch, not the base. + added_links = self._tracked_symlinks(self.pr.head_sha) - self._tracked_symlinks( + f"origin/{self.pr.default_branch}" + ) + if added_links: + raise WorktreeUnavailableError( + f"PR head adds symbolic links: {', '.join(sorted(p for p, _ in added_links))}" ) - except subprocess.TimeoutExpired as exc: - raise WorktreeUnavailableError("symlink check timed out") from exc - if symlink_check.returncode != 0: - raise WorktreeUnavailableError(f"symlink check failed: {symlink_check.stderr.strip()}") - if any(line.startswith("120000 ") for line in symlink_check.stdout.splitlines()): - raise WorktreeUnavailableError("PR head contains symbolic links") try: result = subprocess.run( diff --git a/tools/pr-approval-agent/test_review_pr.py b/tools/pr-approval-agent/test_review_pr.py index 9f34a14d2cd2..4823140cf847 100644 --- a/tools/pr-approval-agent/test_review_pr.py +++ b/tools/pr-approval-agent/test_review_pr.py @@ -506,25 +506,47 @@ def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: assert not any("remove" in c for c in calls) -def test_pr_head_worktree_rejects_symlinks(monkeypatch: pytest.MonkeyPatch) -> None: +@pytest.mark.parametrize( + "head_links, trunk_links, expect_reject", + [ + pytest.param("120000 blob abcdef\tlink\n", "", True, id="pr-adds-symlink"), + pytest.param( + "120000 blob abcdef\tCLAUDE.md\n", "120000 blob abcdef\tCLAUDE.md\n", False, id="trunk-symlink-kept" + ), + pytest.param( + "120000 blob 000000\tCLAUDE.md\n", "120000 blob abcdef\tCLAUDE.md\n", True, id="pr-retargets-symlink" + ), + ], +) +def test_pr_head_worktree_rejects_only_symlinks_the_pr_adds( + monkeypatch: pytest.MonkeyPatch, head_links: str, trunk_links: str, expect_reject: bool +) -> None: + # The trunk carries tracked symlinks (CLAUDE.md -> AGENTS.md and friends); rejecting any symlink + # in the head would fail every stacked review closed. Only links the PR adds or repoints, + # relative to the default branch, are untrusted. calls: list[list[str]] = [] def fake_run(cmd: list[str], **kwargs: object) -> _FakeCompleted: calls.append(cmd) if "ls-tree" in cmd: - return _FakeCompleted(0, stdout="120000 blob abcdef\tlink\n") - raise AssertionError(f"symlinked PR must not create a worktree: {cmd}") + return _FakeCompleted(0, stdout=trunk_links if cmd[-1] == "origin/master" else head_links) + if expect_reject: + raise AssertionError(f"symlinked PR must not create a worktree: {cmd}") + return _FakeCompleted(0) monkeypatch.setattr(review_pr.subprocess, "run", fake_run) pipeline = Pipeline(pr_number=7, repo="PostHog/posthog") pipeline.pr = _fake_pr(head_sha="deadbeef", base_ref="feat/parent-branch") - with pytest.raises(review_pr.WorktreeUnavailableError, match="symbolic links"): - with pipeline._pr_head_worktree(): - pass - - assert len(calls) == 1 + if expect_reject: + with pytest.raises(review_pr.WorktreeUnavailableError, match="adds symbolic links"): + with pipeline._pr_head_worktree(): + pass + assert len(calls) == 2 + else: + with pipeline._pr_head_worktree() as explore_root: + assert explore_root is not None def test_pr_head_worktree_ignores_cleanup_timeout(monkeypatch: pytest.MonkeyPatch) -> None: From 6229ffba758ef92b4b0c3582ab8216abfd82b00d Mon Sep 17 00:00:00 2001 From: Julian Bez Date: Mon, 17 Aug 2026 11:20:16 +0200 Subject: [PATCH 041/289] chore(stamphog): return the symlink baseline as a path->blob dict Keeps the devex tuple-return semgrep rule quiet and reads better than a set of pairs. --- tools/pr-approval-agent/review_pr.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tools/pr-approval-agent/review_pr.py b/tools/pr-approval-agent/review_pr.py index 6ec6edb44721..e5fda00fbdbf 100644 --- a/tools/pr-approval-agent/review_pr.py +++ b/tools/pr-approval-agent/review_pr.py @@ -712,8 +712,8 @@ def _check_tier(self) -> tuple[bool, str]: return True, summary @staticmethod - def _tracked_symlinks(rev: str) -> set[tuple[str, str]]: - """(path, blob) pairs of the symlinks tracked at ``rev``; same blob means same target.""" + def _tracked_symlinks(rev: str) -> dict[str, str]: + """Symlinks tracked at ``rev`` as path -> blob; the same blob means the same target.""" try: listing = subprocess.run( ["git", "ls-tree", "-r", "--full-tree", rev], @@ -726,12 +726,12 @@ def _tracked_symlinks(rev: str) -> set[tuple[str, str]]: raise WorktreeUnavailableError(f"symlink check timed out for {rev}") from exc if listing.returncode != 0: raise WorktreeUnavailableError(f"symlink check failed for {rev}: {listing.stderr.strip()}") - links: set[tuple[str, str]] = set() + links: dict[str, str] = {} for line in listing.stdout.splitlines(): if not line.startswith("120000 "): continue meta, path = line.split("\t", 1) - links.add((path, meta.split()[2])) + links[path] = meta.split()[2] return links @contextmanager @@ -761,13 +761,12 @@ def _pr_head_worktree(self): # A symlink in the head tree can point outside the worktree, and the agent's Read follows # it — so only symlinks the trunk already carries (same path, same target) are trusted; a # stacked PR's base is PR-authored too, so the baseline is the default branch, not the base. - added_links = self._tracked_symlinks(self.pr.head_sha) - self._tracked_symlinks( - f"origin/{self.pr.default_branch}" + trusted_links = self._tracked_symlinks(f"origin/{self.pr.default_branch}") + added_links = sorted( + path for path, blob in self._tracked_symlinks(self.pr.head_sha).items() if trusted_links.get(path) != blob ) if added_links: - raise WorktreeUnavailableError( - f"PR head adds symbolic links: {', '.join(sorted(p for p, _ in added_links))}" - ) + raise WorktreeUnavailableError(f"PR head adds symbolic links: {', '.join(added_links)}") try: result = subprocess.run( From 6206bf6f0399c036aa47073efe8791f3bb25072e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 17 Aug 2026 13:00:43 +0200 Subject: [PATCH 042/289] fix(slack): resolve the repo from thread links --- posthog/git.py | 19 +++++++ .../ai/slack_app/activities/repo_selection.py | 34 ++++++++---- .../ai/slack_app/eval_slack_repo_selection.py | 24 +++++++-- .../slack_app/posthog_code_slack_mention.py | 1 + .../tests/ai/test_cascade_team_install.py | 52 +++++++++++++++++++ posthog/test/test_git.py | 39 +++++++++++++- products/slack_app/backend/api.py | 15 +++++- 7 files changed, 170 insertions(+), 14 deletions(-) diff --git a/posthog/git.py b/posthog/git.py index b0cbcac5a0d1..4325c15d6d88 100644 --- a/posthog/git.py +++ b/posthog/git.py @@ -113,3 +113,22 @@ def extract_explicit_repo(text: str, all_repos: list[str]) -> str | None: linked.add(match) return next(iter(linked)) if len(linked) == 1 else None + + +def extract_explicit_repo_from_scopes(scopes: list[str], all_repos: list[str]) -> str | None: + """Return the connected repo named by the first of `scopes` to name one. + + Callers order `scopes` strongest evidence first, so the text someone wrote while asking + beats the text that happened to be nearby. + + Each scope is matched on its own rather than joined into one string, which is what keeps + the ambiguity rule in `extract_explicit_repo` meaningful. Two repos named inside a single + scope is one person naming two things at once, so nothing resolves. Two repos named across + separate scopes is an ordinary thread accumulating links, so the stronger scope answers. + Joining them first would collapse that distinction and make a long thread resolve to + nothing almost every time. + """ + for scope in scopes: + if match := extract_explicit_repo(scope, all_repos): + return match + return None diff --git a/posthog/temporal/ai/slack_app/activities/repo_selection.py b/posthog/temporal/ai/slack_app/activities/repo_selection.py index 251d8bfefa94..9db95af99574 100644 --- a/posthog/temporal/ai/slack_app/activities/repo_selection.py +++ b/posthog/temporal/ai/slack_app/activities/repo_selection.py @@ -22,18 +22,24 @@ def cascade_posthog_code_repository_activity( inputs: PostHogCodeSlackMentionWorkflowInputs, event_text: str, user_id: int | None = None, + thread_messages: list[dict[str, str]] | None = None, ) -> PostHogCodeRepoCascadeOutcome: """Synchronous fast-path before the discovery agent. - Resolves the trivial cases — no GitHub repos connected to the mentioning user's - personal install, exactly one connected, or an explicit `org/repo` mentioned in the - message — without paying for the sandbox-backed agent. Anything else returns - `mode='agent_needed'` and the workflow takes over. + Resolves the trivial cases without paying for the sandbox-backed agent: no GitHub + repos connected to the mentioning user's personal install, exactly one connected, or + an explicit `org/repo` named in the mention or in the thread it sits in. Anything + else returns `mode='agent_needed'` and the workflow takes over. - ``user_id`` defaults to ``None`` for backwards compatibility with the pre-2026-06 - call shape: if a worker drains an activity task that was scheduled by an older - workflow (recorded with two positional args), the call still binds, and an - unidentifiable mentioner resolves no repos anyway. + The discovery agent this preempts reads the whole thread, so resolving from the + mention alone would hand it asks it then answers from thread text, which is the + sandbox run the fast path exists to avoid. + + ``user_id`` and ``thread_messages`` default to ``None`` for backwards compatibility + with older call shapes: if a worker drains an activity task that was scheduled by an + older workflow (recorded with two or three positional args), the call still binds. + An unidentifiable mentioner resolves no repos anyway, and a missing thread degrades + to the mention-only behavior. """ from posthog.models.integration import Integration @@ -45,7 +51,11 @@ def cascade_posthog_code_repository_activity( ) return PostHogCodeRepoCascadeOutcome(mode="no_repo", repository=None, reason="legacy_no_user_id") - from products.slack_app.backend.api import _extract_explicit_repo, _get_full_repo_names + from products.slack_app.backend.api import ( + _extract_explicit_repo, + _extract_explicit_repo_from_thread, + _get_full_repo_names, + ) integration = Integration.objects.select_related("team", "team__organization").get( id=inputs.integration_id, @@ -68,6 +78,12 @@ def cascade_posthog_code_repository_activity( if explicit_repo: return PostHogCodeRepoCascadeOutcome(mode="auto", repository=explicit_repo, reason="explicit_mention") + # Separate reason from `explicit_mention` so the share of asks the thread scope saves from + # the discovery agent is readable without reproducing the resolution order in a query. + thread_repo = _extract_explicit_repo_from_thread(thread_messages or [], all_repos) + if thread_repo: + return PostHogCodeRepoCascadeOutcome(mode="auto", repository=thread_repo, reason="explicit_thread_mention") + return PostHogCodeRepoCascadeOutcome(mode="agent_needed", repository=None, reason="needs_agent") diff --git a/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py b/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py index 387b4b16c34f..7ea9d79b6aaf 100644 --- a/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py +++ b/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py @@ -83,7 +83,7 @@ from posthog.temporal.ai.slack_app import POSTHOG_CODE_SLACK_MENTION_PICKER_GUIDANCE from posthog.temporal.ai.slack_app.activities.classifiers import classify_task_needs_repo -from products.slack_app.backend.api import _extract_explicit_repo +from products.slack_app.backend.api import _extract_explicit_repo, _extract_explicit_repo_from_thread from products.tasks.backend.facade import api as tasks_facade from products.tasks.backend.facade.repo_selection import ( RepoSelectionRejectedError, @@ -161,6 +161,20 @@ def status(self) -> Literal["PASS", "FAIL", "SKIP"]: expected_stage="cascade", expected_outcome="auto", ), + Case( + name="ci_run_link_earlier_in_the_thread", + description="Cascade reads the repo from a link someone posted before the mention, the usual shape of a CI ask.", + text_template="@PostHog is this one flaky?", + thread_messages=[ + { + "user": "tester", + "text": "https://github.com/{first_repo}/actions/runs/30560492835 went red again", + }, + {"user": "tester", "text": "@PostHog is this one flaky?"}, + ], + expected_stage="cascade", + expected_outcome="auto", + ), # --- Haiku gate short-circuits (heuristic + LLM) --------------------------- Case( name="billing_question", @@ -438,8 +452,12 @@ def _run_case(self, case: Case, *, ctx: TeamContext, flags: RunFlags) -> CaseRes self.stdout.write(f" text: {text}") self.stdout.write(f" expected: {case.expected_stage}/{case.expected_outcome}") - # Stage 1: cascade (synchronous, no LLM) - explicit = _extract_explicit_repo(text, ctx.all_repos) + # Stage 1: cascade (synchronous, no LLM). Mention first, then the thread, matching + # `cascade_posthog_code_repository_activity`; reading only the mention here would pass + # every case whose link sits in the thread while production sent them to the agent. + explicit = _extract_explicit_repo(text, ctx.all_repos) or _extract_explicit_repo_from_thread( + thread_messages, ctx.all_repos + ) if explicit: self.stdout.write(self.style.SUCCESS(f" cascade → auto: {explicit}")) return CaseResult(case=case, actual_stage="cascade", actual_outcome="auto", detail=explicit) diff --git a/posthog/temporal/ai/slack_app/posthog_code_slack_mention.py b/posthog/temporal/ai/slack_app/posthog_code_slack_mention.py index 12efb7292d76..63d6797109cb 100644 --- a/posthog/temporal/ai/slack_app/posthog_code_slack_mention.py +++ b/posthog/temporal/ai/slack_app/posthog_code_slack_mention.py @@ -168,6 +168,7 @@ async def run(self, inputs: PostHogCodeSlackMentionWorkflowInputs) -> None: inputs, event.get("text", ""), user_id, + thread_messages, ) if cascade.mode == "auto": diff --git a/posthog/temporal/tests/ai/test_cascade_team_install.py b/posthog/temporal/tests/ai/test_cascade_team_install.py index 08e6dd061a6c..a6f6df6decb1 100644 --- a/posthog/temporal/tests/ai/test_cascade_team_install.py +++ b/posthog/temporal/tests/ai/test_cascade_team_install.py @@ -80,3 +80,55 @@ def test_a_personal_install_resolves_its_single_repo(self, mock_user_github_clas assert outcome.mode == "auto" assert outcome.repository == "posthog/posthog" + + @parameterized.expand( + [ + ( + "link_sits_in_the_thread_and_the_mention_carries_none", + "<@BOT> is this one flaky?", + [ + {"user": "amy", "text": "https://github.com/posthog/posthog-js/actions/runs/2 failed again"}, + {"user": "bo", "text": "<@BOT> is this one flaky?"}, + ], + "posthog/posthog-js", + "explicit_thread_mention", + ), + ( + "mention_names_a_repo_the_thread_did_not", + "<@BOT> look at posthog/posthog instead", + [ + {"user": "amy", "text": "https://github.com/posthog/posthog-js/actions/runs/2 failed again"}, + {"user": "bo", "text": "<@BOT> look at posthog/posthog instead"}, + ], + "posthog/posthog", + "explicit_mention", + ), + ] + ) + @patch("products.slack_app.backend.api.UserGitHubIntegration") + def test_the_thread_resolves_a_repo_the_mention_left_out( + self, _name, event_text, thread_messages, expected_repository, expected_reason, mock_user_github_class + ): + from posthog.models.user_integration import UserIntegration + + UserIntegration.objects.create( + user=self.user, + kind=UserIntegration.IntegrationKind.GITHUB, + integration_id="gh-user-1", + config={}, + sensitive_config={"access_token": "gh-user-token"}, + ) + mock_user_github = MagicMock() + mock_user_github.list_all_cached_repositories.return_value = [ + {"id": 1, "name": "posthog", "full_name": "posthog/posthog"}, + {"id": 2, "name": "posthog-js", "full_name": "posthog/posthog-js"}, + ] + mock_user_github_class.return_value = mock_user_github + + outcome = cascade_posthog_code_repository_activity( + _make_inputs(self.slack_integration.id), event_text, self.user.id, thread_messages + ) + + assert outcome.mode == "auto" + assert outcome.repository == expected_repository + assert outcome.reason == expected_reason diff --git a/posthog/test/test_git.py b/posthog/test/test_git.py index 1f958656bf0c..5224c21b20de 100644 --- a/posthog/test/test_git.py +++ b/posthog/test/test_git.py @@ -1,6 +1,6 @@ from parameterized import parameterized -from posthog.git import extract_explicit_repo +from posthog.git import extract_explicit_repo, extract_explicit_repo_from_scopes REPOS = ["posthog/posthog", "posthog/posthog-js", "posthog/posthog.com"] @@ -66,3 +66,40 @@ def test_extracts_matching_repo(self, _name: str, text: str, expected: str | Non ) def test_returns_none_on_empty_inputs(self, _name: str, text: str, repos: list[str]): assert extract_explicit_repo(text, repos) is None + + +class TestExtractExplicitRepoFromScopes: + @parameterized.expand( + [ + ( + "later_scope_answers_when_earlier_names_nothing", + ["can you look at this?", "https://github.com/posthog/posthog-js/actions/runs/2"], + "posthog/posthog-js", + ), + ( + "typed_token_in_an_earlier_scope_beats_a_link_in_a_later_one", + ["fix posthog/posthog", "https://github.com/posthog/posthog-js/actions/runs/2"], + "posthog/posthog", + ), + ( + "two_repos_in_one_scope_stay_ambiguous", + [ + "https://github.com/posthog/posthog/pull/1 broke https://github.com/posthog/posthog-js/actions/runs/2", + "https://github.com/posthog/posthog.com/pull/3", + ], + "posthog/posthog.com", + ), + ( + "two_repos_across_separate_scopes_resolve_to_the_first", + [ + "https://github.com/posthog/posthog/pull/1", + "https://github.com/posthog/posthog-js/actions/runs/2", + ], + "posthog/posthog", + ), + ("no_scope_names_a_repo", ["can you look at this?", "it broke again"], None), + ("no_scopes_at_all", [], None), + ] + ) + def test_first_scope_to_name_a_repo_answers(self, _name: str, scopes: list[str], expected: str | None): + assert extract_explicit_repo_from_scopes(scopes, REPOS) == expected diff --git a/products/slack_app/backend/api.py b/products/slack_app/backend/api.py index c81a7718fc81..b27f60331542 100644 --- a/products/slack_app/backend/api.py +++ b/products/slack_app/backend/api.py @@ -23,7 +23,7 @@ from temporalio.common import WorkflowIDConflictPolicy, WorkflowIDReusePolicy from posthog.event_usage import groups -from posthog.git import extract_explicit_repo +from posthog.git import extract_explicit_repo, extract_explicit_repo_from_scopes from posthog.helpers.slack_scopes import REQUIRED_SLACK_SCOPES from posthog.models.integration import ( SLACK_INTEGRATION_KINDS, @@ -917,6 +917,19 @@ def _extract_explicit_repo(text: str, all_repos: list[str]) -> str | None: return extract_explicit_repo(_strip_bot_mentions(text), all_repos) +def _extract_explicit_repo_from_thread(thread_messages: list[dict[str, str]], all_repos: list[str]) -> str | None: + """Repo named by the thread around a mention, newest message first. + + People paste the run or pull request link into the thread and then mention the bot in a + later reply that carries no link of its own. Reading only the mention hands those asks to + the discovery agent, which resolves them from the same thread text this never looked at. + Newest first because the link under discussion is the one most recently posted. + """ + return extract_explicit_repo_from_scopes( + [_strip_bot_mentions(message.get("text", "")) for message in reversed(thread_messages)], all_repos + ) + + def _get_full_repo_names(integration: Integration, *, user_id: int | None) -> list[str]: """Repo names available to the mentioner, from their personal GitHub install. Cached per user.""" if user_id is None: From 54e5fa5c8ba74de575ee07298f843d26bb86a0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 17 Aug 2026 13:04:45 +0200 Subject: [PATCH 043/289] chore(slack): log the cascade repo outcome --- .../ai/slack_app/activities/repo_selection.py | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/posthog/temporal/ai/slack_app/activities/repo_selection.py b/posthog/temporal/ai/slack_app/activities/repo_selection.py index 9db95af99574..f2efd28ae8b7 100644 --- a/posthog/temporal/ai/slack_app/activities/repo_selection.py +++ b/posthog/temporal/ai/slack_app/activities/repo_selection.py @@ -51,11 +51,7 @@ def cascade_posthog_code_repository_activity( ) return PostHogCodeRepoCascadeOutcome(mode="no_repo", repository=None, reason="legacy_no_user_id") - from products.slack_app.backend.api import ( - _extract_explicit_repo, - _extract_explicit_repo_from_thread, - _get_full_repo_names, - ) + from products.slack_app.backend.api import _get_full_repo_names integration = Integration.objects.select_related("team", "team__organization").get( id=inputs.integration_id, @@ -74,13 +70,29 @@ def cascade_posthog_code_repository_activity( if len(all_repos) == 1: return PostHogCodeRepoCascadeOutcome(mode="auto", repository=all_repos[0], reason="single_repo") + outcome = _resolve_from_text(event_text, thread_messages or [], all_repos) + # The reason carries which scope answered, and whether anything did. Without it the share of + # mentions the fast path saves from the discovery agent is only measurable by rerunning the + # resolution order over Slack text, which is not something to reconstruct in a query. + logger.info( + "posthog_code_cascade_outcome", + reason=outcome.reason, + integration_id=inputs.integration_id, + ) + return outcome + + +def _resolve_from_text( + event_text: str, thread_messages: list[dict[str, str]], all_repos: list[str] +) -> PostHogCodeRepoCascadeOutcome: + """Repo named by the mention, then by the thread, each reported under its own reason.""" + from products.slack_app.backend.api import _extract_explicit_repo, _extract_explicit_repo_from_thread + explicit_repo = _extract_explicit_repo(event_text, all_repos) if explicit_repo: return PostHogCodeRepoCascadeOutcome(mode="auto", repository=explicit_repo, reason="explicit_mention") - # Separate reason from `explicit_mention` so the share of asks the thread scope saves from - # the discovery agent is readable without reproducing the resolution order in a query. - thread_repo = _extract_explicit_repo_from_thread(thread_messages or [], all_repos) + thread_repo = _extract_explicit_repo_from_thread(thread_messages, all_repos) if thread_repo: return PostHogCodeRepoCascadeOutcome(mode="auto", repository=thread_repo, reason="explicit_thread_mention") From df879533009878578f0e33740b65f67b3af5293f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Negr=C3=B3n?= Date: Mon, 17 Aug 2026 13:22:40 +0200 Subject: [PATCH 044/289] refactor(slack): tighten repo resolution comments --- posthog/git.py | 16 ++++++---------- .../ai/slack_app/activities/repo_selection.py | 9 ++++----- .../ai/slack_app/eval_slack_repo_selection.py | 5 ++--- products/slack_app/backend/api.py | 8 ++++---- 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/posthog/git.py b/posthog/git.py index 4325c15d6d88..1f2fdca010db 100644 --- a/posthog/git.py +++ b/posthog/git.py @@ -116,17 +116,13 @@ def extract_explicit_repo(text: str, all_repos: list[str]) -> str | None: def extract_explicit_repo_from_scopes(scopes: list[str], all_repos: list[str]) -> str | None: - """Return the connected repo named by the first of `scopes` to name one. + """Return the connected repo named by the first of `scopes` to name one, callers ordering + them strongest evidence first. - Callers order `scopes` strongest evidence first, so the text someone wrote while asking - beats the text that happened to be nearby. - - Each scope is matched on its own rather than joined into one string, which is what keeps - the ambiguity rule in `extract_explicit_repo` meaningful. Two repos named inside a single - scope is one person naming two things at once, so nothing resolves. Two repos named across - separate scopes is an ordinary thread accumulating links, so the stronger scope answers. - Joining them first would collapse that distinction and make a long thread resolve to - nothing almost every time. + Each scope is matched on its own rather than joined into one string, which keeps the + ambiguity rule in `extract_explicit_repo` meaningful: two repos named inside one scope is + someone naming two things at once and resolves to nothing, while two repos named across + separate scopes is a thread accumulating links and lets the stronger scope answer. """ for scope in scopes: if match := extract_explicit_repo(scope, all_repos): diff --git a/posthog/temporal/ai/slack_app/activities/repo_selection.py b/posthog/temporal/ai/slack_app/activities/repo_selection.py index f2efd28ae8b7..71b18801f953 100644 --- a/posthog/temporal/ai/slack_app/activities/repo_selection.py +++ b/posthog/temporal/ai/slack_app/activities/repo_selection.py @@ -70,10 +70,9 @@ def cascade_posthog_code_repository_activity( if len(all_repos) == 1: return PostHogCodeRepoCascadeOutcome(mode="auto", repository=all_repos[0], reason="single_repo") - outcome = _resolve_from_text(event_text, thread_messages or [], all_repos) - # The reason carries which scope answered, and whether anything did. Without it the share of - # mentions the fast path saves from the discovery agent is only measurable by rerunning the - # resolution order over Slack text, which is not something to reconstruct in a query. + outcome = _resolve_explicit_repo(event_text, thread_messages or [], all_repos) + # Logged so the share of mentions the fast path saves from the discovery agent, and which + # scope produced it, is measurable without rerunning the resolution order over Slack text. logger.info( "posthog_code_cascade_outcome", reason=outcome.reason, @@ -82,7 +81,7 @@ def cascade_posthog_code_repository_activity( return outcome -def _resolve_from_text( +def _resolve_explicit_repo( event_text: str, thread_messages: list[dict[str, str]], all_repos: list[str] ) -> PostHogCodeRepoCascadeOutcome: """Repo named by the mention, then by the thread, each reported under its own reason.""" diff --git a/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py b/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py index 7ea9d79b6aaf..b8b054609836 100644 --- a/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py +++ b/posthog/temporal/ai/slack_app/eval_slack_repo_selection.py @@ -452,9 +452,8 @@ def _run_case(self, case: Case, *, ctx: TeamContext, flags: RunFlags) -> CaseRes self.stdout.write(f" text: {text}") self.stdout.write(f" expected: {case.expected_stage}/{case.expected_outcome}") - # Stage 1: cascade (synchronous, no LLM). Mention first, then the thread, matching - # `cascade_posthog_code_repository_activity`; reading only the mention here would pass - # every case whose link sits in the thread while production sent them to the agent. + # Stage 1: cascade (synchronous, no LLM). Mirrors `cascade_posthog_code_repository_activity`, + # because reading only the mention here would pass cases that production sends to the agent. explicit = _extract_explicit_repo(text, ctx.all_repos) or _extract_explicit_repo_from_thread( thread_messages, ctx.all_repos ) diff --git a/products/slack_app/backend/api.py b/products/slack_app/backend/api.py index c4b60de6fa8c..7c91822187ae 100644 --- a/products/slack_app/backend/api.py +++ b/products/slack_app/backend/api.py @@ -922,10 +922,10 @@ def _extract_explicit_repo(text: str, all_repos: list[str]) -> str | None: def _extract_explicit_repo_from_thread(thread_messages: list[dict[str, str]], all_repos: list[str]) -> str | None: """Repo named by the thread around a mention, newest message first. - People paste the run or pull request link into the thread and then mention the bot in a - later reply that carries no link of its own. Reading only the mention hands those asks to - the discovery agent, which resolves them from the same thread text this never looked at. - Newest first because the link under discussion is the one most recently posted. + People paste the link into the thread and mention the bot in a later reply that carries no + link of its own. Reading only the mention hands those asks to the discovery agent, which + answers them from this same thread text. Newest first because the link under discussion is + the one most recently posted. """ return extract_explicit_repo_from_scopes( [_strip_bot_mentions(message.get("text", "")) for message in reversed(thread_messages)], all_repos From ad81f739034f2fe444ba0eca32830a3549d06e0e Mon Sep 17 00:00:00 2001 From: JonathanLab Date: Mon, 17 Aug 2026 13:47:09 +0200 Subject: [PATCH 045/289] feat(desktop): enrich Pi source reads with PostHog context --- .../desktop-pi-rpc-client-factory.test.ts | 17 +- .../desktop-pi-rpc-client-factory.ts | 12 +- .../tests/e2e/tests/pi-enrichment.spec.ts | 265 ++++++++++++++++++ .../agent/src/enrichment/file-enricher.ts | 1 + .../agent/src/pi/enrichment-extension.test.ts | 83 ++++++ .../agent/src/pi/enrichment-extension.ts | 74 +++++ .../packages/agent/src/pi/rpc-client.test.ts | 12 + .../packages/agent/src/pi/rpc-client.ts | 5 + .../desktop/packages/agent/src/pi/rpc-host.ts | 4 + .../agent/src/server/pi-agent-server.ts | 5 + .../desktop/packages/agent/tsup.config.ts | 19 +- products/desktop/packages/enricher/README.md | 3 + .../enricher/src/comment-formatter.test.ts | 64 +++++ .../enricher/src/comment-formatter.ts | 47 +++- .../packages/enricher/src/enricher.test.ts | 57 ++++ .../packages/enricher/src/parse-result.ts | 2 +- .../packages/enricher/src/posthog-api.ts | 54 +++- .../desktop/packages/enricher/src/types.ts | 1 + products/desktop/packages/shared/src/task.ts | 1 + 19 files changed, 703 insertions(+), 23 deletions(-) create mode 100644 products/desktop/apps/code/tests/e2e/tests/pi-enrichment.spec.ts create mode 100644 products/desktop/packages/agent/src/pi/enrichment-extension.test.ts create mode 100644 products/desktop/packages/agent/src/pi/enrichment-extension.ts diff --git a/products/desktop/apps/code/src/main/platform-adapters/desktop-pi-rpc-client-factory.test.ts b/products/desktop/apps/code/src/main/platform-adapters/desktop-pi-rpc-client-factory.test.ts index 1e3dad18786e..d600a81f9020 100644 --- a/products/desktop/apps/code/src/main/platform-adapters/desktop-pi-rpc-client-factory.test.ts +++ b/products/desktop/apps/code/src/main/platform-adapters/desktop-pi-rpc-client-factory.test.ts @@ -35,9 +35,17 @@ describe("DesktopPiRpcClientFactory", () => { region: "eu" as const, })), getState: vi.fn(() => ({ currentProjectId: 1 })), + getValidAccessToken: vi.fn(async () => ({ + accessToken: "access-token", + apiHost: "https://eu.posthog.com", + })), } as unknown as AgentAuth; const authProxy = { - start: vi.fn(async () => "http://127.0.0.1:1234"), + start: vi.fn(async (url: string) => + url === "https://eu.posthog.com" + ? "http://127.0.0.1:5678" + : "http://127.0.0.1:1234", + ), } as unknown as AuthProxyService; const policies = [ { @@ -79,8 +87,15 @@ describe("DesktopPiRpcClientFactory", () => { getLlmGatewayUrl(getCloudUrlFromRegion("eu")), { "X-PostHog-Project-Id": "1" }, ); + expect(authProxy.start).toHaveBeenCalledWith("https://eu.posthog.com"); expect(createPiRpcClient).toHaveBeenCalledWith({ cwd: "/workspace", + enrichment: { + apiUrl: "http://127.0.0.1:5678", + publicApiUrl: "https://eu.posthog.com", + projectId: 1, + apiKey: "posthog-code-auth-proxy", + }, mcpToolPolicies: policies, runtimeMcpServers: { posthog: { diff --git a/products/desktop/apps/code/src/main/platform-adapters/desktop-pi-rpc-client-factory.ts b/products/desktop/apps/code/src/main/platform-adapters/desktop-pi-rpc-client-factory.ts index 3cf1bccd9f37..acef96807a54 100644 --- a/products/desktop/apps/code/src/main/platform-adapters/desktop-pi-rpc-client-factory.ts +++ b/products/desktop/apps/code/src/main/platform-adapters/desktop-pi-rpc-client-factory.ts @@ -43,7 +43,11 @@ export class DesktopPiRpcClientFactory implements PiRpcClientFactory { if (!projectId) { throw new Error("Pi requires a selected PostHog project"); } - const baseUrl = await this.getProxyUrl(credentials.region, projectId); + const access = await this.auth.getValidAccessToken(); + const [baseUrl, enrichmentApiUrl] = await Promise.all([ + this.getProxyUrl(credentials.region, projectId), + this.authProxy.start(access.apiHost), + ]); const mcpConfiguration = await this.mcpServerSource.getMcpRuntimeConfiguration(); @@ -54,6 +58,12 @@ export class DesktopPiRpcClientFactory implements PiRpcClientFactory { model: input.model, sessionFile: input.sessionFile, projectTrusted: input.projectTrusted, + enrichment: { + apiUrl: enrichmentApiUrl, + publicApiUrl: access.apiHost, + projectId, + apiKey: PROXY_API_KEY, + }, runtimeMcpServers, mcpToolPolicies: mcpConfiguration.policies, providerOptions: { diff --git a/products/desktop/apps/code/tests/e2e/tests/pi-enrichment.spec.ts b/products/desktop/apps/code/tests/e2e/tests/pi-enrichment.spec.ts new file mode 100644 index 000000000000..48b9b8ba6717 --- /dev/null +++ b/products/desktop/apps/code/tests/e2e/tests/pi-enrichment.spec.ts @@ -0,0 +1,265 @@ +import { existsSync } from "node:fs"; +import { mkdir, writeFile } from "node:fs/promises"; +import { + createServer, + type IncomingMessage, + type ServerResponse, +} from "node:http"; +import path from "node:path"; +import type { PiRpcClient } from "@posthog/agent/pi/rpc-client"; +import { expect, test } from "../fixtures/electron"; + +const HOME_ENVIRONMENT_KEYS = [ + "APPDATA", + "HOME", + "LOCALAPPDATA", + "USERPROFILE", + "XDG_CONFIG_HOME", +] as const; + +function createAnthropicStream(content: "tool" | "text"): string { + const messageStart = { + type: "message_start", + message: { + id: `msg_${content}`, + type: "message", + role: "assistant", + content: [], + model: "claude-haiku-4-5", + stop_reason: null, + stop_sequence: null, + usage: { input_tokens: 10, output_tokens: 0 }, + }, + }; + const events = + content === "tool" + ? [ + messageStart, + { + type: "content_block_start", + index: 0, + content_block: { + type: "tool_use", + id: "toolu_read_1", + name: "read", + input: {}, + }, + }, + { + type: "content_block_delta", + index: 0, + delta: { + type: "input_json_delta", + partial_json: JSON.stringify({ path: "example.ts" }), + }, + }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { stop_reason: "tool_use", stop_sequence: null }, + usage: { output_tokens: 10 }, + }, + { type: "message_stop" }, + ] + : [ + messageStart, + { + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: "Enrichment observed." }, + }, + { type: "content_block_stop", index: 0 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn", stop_sequence: null }, + usage: { output_tokens: 5 }, + }, + { type: "message_stop" }, + ]; + + return events + .map((event) => `event: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`) + .join(""); +} + +async function readJson(request: IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of request) { + chunks.push(Buffer.from(chunk)); + } + + return JSON.parse(Buffer.concat(chunks).toString("utf8")); +} + +function sendJson(response: ServerResponse, data: unknown): void { + response.writeHead(200, { "Content-Type": "application/json" }); + response.end(JSON.stringify(data)); +} + +test.describe("Pi enrichment", () => { + test("enriches a read result through the bundled RPC host", async ({ + electronApp, + }) => { + const { e2eHome, resourcesPath } = await electronApp.evaluate( + async ({ app }) => ({ + e2eHome: process.env.HOME ?? app.getPath("home"), + resourcesPath: process.resourcesPath, + }), + ); + const rpcHostPath = path.join( + resourcesPath, + "app.asar.unpacked", + ".vite", + "build", + "rpc-host.js", + ); + expect(existsSync(rpcHostPath)).toBe(true); + + const workspace = path.join(e2eHome, "enrichment-workspace"); + await mkdir(workspace, { recursive: true }); + await writeFile( + path.join(workspace, "example.ts"), + 'posthog.capture("checkout_completed");\n', + ); + + let enrichedModelRequest = ""; + let eventDefinitionRequests = 0; + let eventStatsRequests = 0; + const server = createServer(async (request, response) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + + if (request.method === "GET" && url.pathname === "/v1/models") { + sendJson(response, { + data: [ + { + id: "claude-haiku-4-5", + owned_by: "anthropic", + context_window: 200000, + supports_vision: true, + }, + ], + }); + return; + } + + if (request.method === "POST" && url.pathname === "/v1/messages") { + const body = await readJson(request); + const serialized = JSON.stringify(body); + const hasToolResult = serialized.includes('"tool_result"'); + if (hasToolResult) { + enrichedModelRequest = serialized; + } + response.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + }); + response.end(createAnthropicStream(hasToolResult ? "text" : "tool")); + return; + } + + if ( + request.method === "GET" && + url.pathname === "/api/projects/1/event_definitions/" + ) { + eventDefinitionRequests += 1; + sendJson(response, { + results: [ + { + id: "event-1", + name: "checkout_completed", + tags: ["revenue"], + last_seen_at: "2026-08-13T12:00:00Z", + verified: true, + }, + ], + }); + return; + } + + if ( + request.method === "POST" && + url.pathname === "/api/projects/1/query/" + ) { + eventStatsRequests += 1; + sendJson(response, { + results: [["checkout_completed", 321, 87, "2026-08-13T12:00:00Z"]], + }); + return; + } + + response.writeHead(404); + response.end(); + }); + + await new Promise((resolveListening) => { + server.listen(0, "127.0.0.1", resolveListening); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("Fake PostHog server did not bind a TCP port"); + } + + const previousEnvironment = new Map( + HOME_ENVIRONMENT_KEYS.map((key) => [key, process.env[key]]), + ); + for (const key of HOME_ENVIRONMENT_KEYS) { + process.env[key] = e2eHome; + } + + const baseUrl = `http://127.0.0.1:${address.port}`; + let client: PiRpcClient | undefined; + try { + const { createPiRpcClient } = await import( + "@posthog/agent/pi/rpc-client" + ); + client = createPiRpcClient({ + cliPath: rpcHostPath, + cwd: workspace, + model: "claude-haiku-4-5", + projectTrusted: false, + providerOptions: { apiKey: "gateway-test-key", baseUrl }, + enrichment: { + apiUrl: baseUrl, + publicApiUrl: "https://us.posthog.com", + projectId: 1, + apiKey: "posthog-test-key", + }, + }); + await client.start(); + const settled = client.waitForIdle(); + await client.prompt("Read example.ts and report what you find."); + await settled; + + expect(eventDefinitionRequests).toBe(1); + expect(eventStatsRequests).toBe(1); + expect(enrichedModelRequest).toContain( + '[PostHog] Event: \\"checkout_completed\\"', + ); + expect(enrichedModelRequest).toContain("321 events"); + expect(enrichedModelRequest).toContain("87 users"); + } finally { + await client?.stop(); + for (const [key, value] of previousEnvironment) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + await new Promise((resolveClosed, rejectClosed) => { + server.close((error) => { + if (error) { + rejectClosed(error); + } else { + resolveClosed(); + } + }); + }); + } + }); +}); diff --git a/products/desktop/packages/agent/src/enrichment/file-enricher.ts b/products/desktop/packages/agent/src/enrichment/file-enricher.ts index 4bce2658e07d..1aa5529f9808 100644 --- a/products/desktop/packages/agent/src/enrichment/file-enricher.ts +++ b/products/desktop/packages/agent/src/enrichment/file-enricher.ts @@ -78,6 +78,7 @@ export async function enrichFileForAgent( const enriched = await parsed.enrichFromApi({ apiKey, host: deps.apiConfig.apiUrl, + publicHost: deps.apiConfig.publicApiUrl, projectId: deps.apiConfig.projectId, timeoutMs: 5_000, }); diff --git a/products/desktop/packages/agent/src/pi/enrichment-extension.test.ts b/products/desktop/packages/agent/src/pi/enrichment-extension.test.ts new file mode 100644 index 000000000000..75c12b3d283d --- /dev/null +++ b/products/desktop/packages/agent/src/pi/enrichment-extension.test.ts @@ -0,0 +1,83 @@ +import type { + ExtensionAPI, + ExtensionContext, + ToolResultEvent, +} from "@earendil-works/pi-coding-agent"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createPiEnrichmentExtension } from "./enrichment-extension"; + +type ToolResultPatch = { content?: ToolResultEvent["content"] }; +type ToolResultHandler = ( + event: ToolResultEvent, + ctx: ExtensionContext, +) => Promise | ToolResultPatch | undefined; + +describe("createPiEnrichmentExtension", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("adds live PostHog metadata to Pi read results", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async (input: string | URL | Request) => { + const url = String(input); + const data = url.includes("event_definitions") + ? { + results: [ + { + id: "event-1", + name: "checkout", + tags: [], + last_seen_at: "2026-01-01T00:00:00Z", + verified: true, + }, + ], + } + : { + results: [["checkout", 120, 40, "2026-01-01T00:00:00Z"]], + }; + return new Response(JSON.stringify(data), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + }), + ); + + let handler: ToolResultHandler | undefined; + const extension = createPiEnrichmentExtension({ + apiUrl: "https://us.posthog.com", + projectId: 1, + apiKey: "token", + }); + await extension.factory({ + on: (event: string, registeredHandler: ToolResultHandler) => { + if (event === "tool_result") { + handler = registeredHandler; + } + }, + } as unknown as ExtensionAPI); + + const result = await handler?.( + { + type: "tool_result", + toolCallId: "read-1", + toolName: "read", + input: { path: "/tmp/example.ts" }, + content: [{ type: "text", text: 'posthog.capture("checkout");' }], + details: undefined, + isError: false, + }, + { cwd: "/tmp" } as ExtensionContext, + ); + + expect(result?.content).toEqual([ + { + type: "text", + text: expect.stringContaining( + '[PostHog] Event: "checkout" — (verified) — 120 events — 40 users', + ), + }, + ]); + }); +}); diff --git a/products/desktop/packages/agent/src/pi/enrichment-extension.ts b/products/desktop/packages/agent/src/pi/enrichment-extension.ts new file mode 100644 index 000000000000..0bfa2beedfaf --- /dev/null +++ b/products/desktop/packages/agent/src/pi/enrichment-extension.ts @@ -0,0 +1,74 @@ +import { resolve } from "node:path"; +import { + type ExtensionAPI, + type ExtensionFactory, + isReadToolResult, +} from "@earendil-works/pi-coding-agent"; +import { + createEnrichment, + enrichFileForAgent, +} from "../enrichment/file-enricher"; + +export interface PiEnrichmentConfig { + apiUrl: string; + publicApiUrl?: string; + projectId: number; + apiKey: string; +} + +export function createPiEnrichmentExtension(config: PiEnrichmentConfig): { + name: string; + factory: ExtensionFactory; +} { + return { + name: "posthog-enricher", + factory: (pi: ExtensionAPI) => { + const enrichment = createEnrichment({ + apiUrl: config.apiUrl, + publicApiUrl: config.publicApiUrl, + projectId: config.projectId, + getApiKey: () => config.apiKey, + }); + if (!enrichment) { + return; + } + + pi.on("tool_result", async (event, ctx) => { + if (!isReadToolResult(event) || event.isError) { + return; + } + + const rawPath = event.input.path; + if (typeof rawPath !== "string") { + return; + } + + const textIndex = event.content.findIndex( + (content) => content.type === "text", + ); + const textContent = event.content[textIndex]; + if (!textContent || textContent.type !== "text") { + return; + } + + const filePath = resolve(ctx.cwd, rawPath.replace(/^@/, "")); + const enriched = await enrichFileForAgent( + enrichment.deps, + filePath, + textContent.text, + ); + if (!enriched) { + return; + } + + const content = [...event.content]; + content[textIndex] = { type: "text", text: enriched }; + return { content }; + }); + + pi.on("session_shutdown", () => { + enrichment.dispose(); + }); + }, + }; +} diff --git a/products/desktop/packages/agent/src/pi/rpc-client.test.ts b/products/desktop/packages/agent/src/pi/rpc-client.test.ts index 901b8eefac5a..5cb32cbf44db 100644 --- a/products/desktop/packages/agent/src/pi/rpc-client.test.ts +++ b/products/desktop/packages/agent/src/pi/rpc-client.test.ts @@ -108,6 +108,12 @@ process.stdin.resume(); projectTrusted: true, extensions: ["auto-publish"], providerOptions: { apiKey: "proxy-key" }, + enrichment: { + apiUrl: "http://127.0.0.1:5678", + publicApiUrl: "https://us.posthog.com", + projectId: 2, + apiKey: "enrichment-proxy-key", + }, }); try { @@ -116,6 +122,12 @@ process.stdin.resume(); await expect(readFile(capturePath, "utf8")).resolves.toBe( JSON.stringify({ providerOptions: { apiKey: "proxy-key" }, + enrichment: { + apiUrl: "http://127.0.0.1:5678", + publicApiUrl: "https://us.posthog.com", + projectId: 2, + apiKey: "enrichment-proxy-key", + }, projectTrusted: true, extensions: ["auto-publish"], }), diff --git a/products/desktop/packages/agent/src/pi/rpc-client.ts b/products/desktop/packages/agent/src/pi/rpc-client.ts index 6d33fccfc2d9..0ea88b171f31 100644 --- a/products/desktop/packages/agent/src/pi/rpc-client.ts +++ b/products/desktop/packages/agent/src/pi/rpc-client.ts @@ -16,6 +16,7 @@ import type { McpToolPermissionRequest, McpToolPolicy, } from "@posthog/shared"; +import type { PiEnrichmentConfig } from "./enrichment-extension"; import { safePiEnvironment } from "./rpc-environment"; import type { PiExtensionEvent, @@ -50,6 +51,7 @@ export interface PiRpcProviderOptions { export interface PiRpcBootstrap { providerOptions: PiRpcProviderOptions; + enrichment?: PiEnrichmentConfig; runtimeMcpServers?: PiRuntimeMcpServers; mcpToolPolicies?: McpToolPolicy[]; projectTrusted?: boolean; @@ -431,6 +433,7 @@ export type PiRpcClientOptions = Pick< > & { sessionFile?: string; providerOptions: PiRpcProviderOptions; + enrichment?: PiEnrichmentConfig; runtimeMcpServers?: PiRuntimeMcpServers; mcpToolPolicies?: McpToolPolicy[]; projectTrusted?: boolean; @@ -441,6 +444,7 @@ export function createPiRpcClient(options: PiRpcClientOptions): PiRpcClient { const { sessionFile, providerOptions, + enrichment, runtimeMcpServers, mcpToolPolicies, projectTrusted, @@ -460,6 +464,7 @@ export function createPiRpcClient(options: PiRpcClientOptions): PiRpcClient { }, { providerOptions, + enrichment, runtimeMcpServers, mcpToolPolicies, projectTrusted: projectTrusted ?? false, diff --git a/products/desktop/packages/agent/src/pi/rpc-host.ts b/products/desktop/packages/agent/src/pi/rpc-host.ts index 2de5282f3152..a3495a1b8dee 100644 --- a/products/desktop/packages/agent/src/pi/rpc-host.ts +++ b/products/desktop/packages/agent/src/pi/rpc-host.ts @@ -10,6 +10,7 @@ import type { McpToolPermissionDecision, McpToolPermissionRequest, } from "@posthog/shared"; +import { createPiEnrichmentExtension } from "./enrichment-extension"; import { POSTHOG_PI_QUEUE_ENTRY_TYPE, readPersistedPiQueue, @@ -78,6 +79,9 @@ const extensionFactories: Record = { const runtimeExtensions = (bootstrap.extensions ?? []).map( (extension) => extensionFactories[extension], ); +if (bootstrap.enrichment) { + runtimeExtensions.push(createPiEnrichmentExtension(bootstrap.enrichment)); +} const runtime = await createHarnessRuntime({ cwd, diff --git a/products/desktop/packages/agent/src/server/pi-agent-server.ts b/products/desktop/packages/agent/src/server/pi-agent-server.ts index f34b90705502..b42d1d1654ae 100644 --- a/products/desktop/packages/agent/src/server/pi-agent-server.ts +++ b/products/desktop/packages/agent/src/server/pi-agent-server.ts @@ -541,6 +541,11 @@ export class PiAgentServer { cwd, model: this.config.model, sessionFile: restoredSessionFile, + enrichment: { + apiUrl: this.config.apiUrl, + projectId: this.config.projectId, + apiKey: this.config.apiKey, + }, runtimeMcpServers, mcpToolPolicies: mcpConfiguration.policies, providerOptions: { diff --git a/products/desktop/packages/agent/tsup.config.ts b/products/desktop/packages/agent/tsup.config.ts index b8167cd2392e..2e98c6a95a49 100644 --- a/products/desktop/packages/agent/tsup.config.ts +++ b/products/desktop/packages/agent/tsup.config.ts @@ -48,9 +48,17 @@ function copyAssets() { const distDir = resolve(import.meta.dirname, "dist"); const templatesDir = resolve(distDir, "templates"); const claudeCliDir = resolve(distDir, "claude-cli"); + const enricherGrammarsSource = resolve( + import.meta.dirname, + "../enricher/grammars", + ); + const enricherGrammarsTarget = resolve(distDir, "grammars"); mkdirSync(templatesDir, { recursive: true }); mkdirSync(claudeCliDir, { recursive: true }); + cpSync(enricherGrammarsSource, enricherGrammarsTarget, { + recursive: true, + }); const srcTemplatesDir = resolve(import.meta.dirname, "src/templates"); if (existsSync(srcTemplatesDir)) { @@ -78,6 +86,9 @@ function copyAssets() { ); } +const nodeEsmBanner = + 'import { createRequire as __createRequire } from "node:module"; import { fileURLToPath as __fileURLToPath } from "node:url"; import { dirname as __pathDirname } from "node:path"; const require = __createRequire(import.meta.url); const __filename = __fileURLToPath(import.meta.url); const __dirname = __pathDirname(__filename);'; + const sharedOptions = { sourcemap: true, splitting: false, @@ -158,9 +169,7 @@ export default defineConfig([ // dynamic `require(...)` calls throw in ESM output unless a real require // exists. Entries spawned directly by node (local-tools-mcp-server.js) // crash at import time without this shim. - banner: { - js: 'import { createRequire as __createRequire } from "node:module"; const require = __createRequire(import.meta.url);', - }, + banner: { js: nodeEsmBanner }, ...sharedOptions, onSuccess: async () => { copyAssets(); @@ -187,9 +196,7 @@ export default defineConfig([ format: ["esm"], dts: false, clean: false, - banner: { - js: 'import { createRequire as __createRequire } from "node:module"; const require = __createRequire(import.meta.url);', - }, + banner: { js: nodeEsmBanner }, ...sharedOptions, noExternal: [/^(?!node:)/], external: [...builtinModules, ...builtinModules.map((m) => `node:${m}`)], diff --git a/products/desktop/packages/enricher/README.md b/products/desktop/packages/enricher/README.md index 9dfae9129d8c..10931f29e161 100644 --- a/products/desktop/packages/enricher/README.md +++ b/products/desktop/packages/enricher/README.md @@ -130,10 +130,13 @@ Returned by `enrich()` or `enrichFromApi()`. Detection combined with PostHog con interface EnricherApiConfig { apiKey: string; host: string; // e.g. "https://us.posthog.com" + publicHost?: string; projectId: number; } ``` +Set `publicHost` when API requests use a private proxy but annotation links should use the public PostHog URL. + ### `EnrichedFlag` ```typescript diff --git a/products/desktop/packages/enricher/src/comment-formatter.test.ts b/products/desktop/packages/enricher/src/comment-formatter.test.ts index 11c1b32955bb..a220e47fc659 100644 --- a/products/desktop/packages/enricher/src/comment-formatter.test.ts +++ b/products/desktop/packages/enricher/src/comment-formatter.test.ts @@ -41,6 +41,23 @@ describe("formatInlineComments", () => { ); }); + test("replaces an existing inline annotation", () => { + const source = `posthog.capture('a'); // [PostHog] Event: "old"`; + const items = [eventItem("a", 0, false)]; + const events = new Map([["a", enrichedEvent("a")]]); + const out = formatInlineComments( + source, + "javascript", + items, + new Map(), + events, + ); + + expect(out.match(/\[PostHog\]/g)).toHaveLength(1); + expect(out).toContain(`Event: "a" \u2014 (verified)`); + expect(out).not.toContain(`Event: "old"`); + }); + test("pure JSX line uses {/* */} suffix", () => { const source = `