diff --git a/.claude/skills/meridian-brainstorm/brainstorm-history.jsonl b/.claude/skills/meridian-brainstorm/brainstorm-history.jsonl index 9d29fbf348..87656ceb48 100644 --- a/.claude/skills/meridian-brainstorm/brainstorm-history.jsonl +++ b/.claude/skills/meridian-brainstorm/brainstorm-history.jsonl @@ -6,3 +6,4 @@ {"session_date": "2026-03-27", "mode": "Domain-Focused", "themes": ["Script-to-strategy promotion wizard", "Script notebook mode (multi-cell)", "Script library / import system", "Scheduled scripts and Alert API", "QuantScript parameter grid search API", "Live streaming data in scripts (v2)", "L3 x QuantScript bridge (L3.Simulate())", "Calibration quality report card", "Toxic flow tagging on fill tape", "L3 simulation replay visualizer", "Options queue model extension", "Symbol-partitioned collection (sharding)", "Leader-elected coordinator instance", "Cross-instance deduplication ledger (IDedupStore)", "Distributed backfill work queue", "Cluster health dashboard panel", "Distributed grid search (Track G x I)", "Cross-symbol correlation regime signals"], "ideas_count": 18, "track": "G-H-I", "notes": "First round of Track G-I brainstorm. Defined L3.Simulate() as highest-leverage cross-track bridge. IDedupStore as foundational Track I seam. Recommended sequencing: CalibrationReportCard -> ScriptLibrary -> L3Bridge -> GridSearch -> IDedupStore -> LeaderElection -> Sharding -> DistributedBackfill."} {"session_date": "2026-03-27", "mode": "Domain-Focused", "themes": ["Script debugger (step/watch/breakpoint)", "Script run history and result diffing", "Security Master access in QuantScript (Securities global)", "Script output export (PDF/Parquet/Excel/HTML)", "Script expression REPL", "Venue-specific L3 calibration profiles (VenueCalibrationPriors)", "TCA report: execution cost attribution", "Multi-day queue state continuity (carry-state)", "Dark pool / hidden liquidity estimator", "Zero-downtime rolling upgrade orchestration", "Geo-distributed collection by exchange timezone", "Split-brain detection and recovery"], "ideas_count": 12, "track": "G-H-I", "notes": "Second round of Track G-I brainstorm. TCA Report identified as highest-leverage institutional positioning item. VenueCalibrationPriors and SplitBrainDetector identified as platform bets. Script REPL flagged as surprise high-leverage small-effort item. Sequencing: REPL + SecurityMaster -> TCA Report -> RunHistory + VenueCalibration -> SplitBrain -> RollingUpgrade + DarkPool + GeoDistributed + Debugger + MultiDayContinuity."} {"session_date": "2026-06-25", "mode": "Domain-Focused", "themes": ["Provider Setup Wizard with Live Certification", "Hot-Reload Provider Re-Registration", "Credential Vault with Encrypted Persistence", "Runtime Failover Chain Editor", "Per-Provider Partial Config Mutation API", "Ingestion Session Continuity on Restart", "Provider Capability Browser in Data Workspace"], "ideas_count": 7, "document_updated": null, "notes": "Fresh domain covering runtime provider lifecycle management — an unexplored area in all prior sessions. Platform bets: Hot-Reload Re-Registration (ConfigWatcher already exists, ProviderRegistry.RegisterStreamingFactory already supports replace) and Per-Provider Partial Config Mutation (ConfigStore.PatchAsync primitive). Recommended sequence: Capability Browser (S) -> Partial Config API (M) -> Hot-Reload (M) -> Failover Editor (S) -> Setup Wizard (M) -> Credential Vault (M) -> Session Continuity (L)."} +{"session_date": "2026-07-01", "mode": "Competitive / Open Exploration", "themes": ["live/paper TCA best-execution report pack", "Alpaca live broker adapter (IExecutionGateway)", "reconciliation break-resolution agent (MCP + CoS gates)", "custodian/broker statement connector library (IB Flex, OFX/CSV profiles)", "DuckDB analytics workbench + /api/query", "Python meridian client package (promoted)", "W6 backtesting studio evidence loop (manifest + run comparison)", "LEAN/NautilusTrader data-feed bridge", "governed investor report delivery portal-lite", "data-cost savings meter"], "ideas_count": 10, "document_updated": "docs/product/high-value-code-brainstorm-2026-07.md", "notes": "Market-researched prioritization session. Signals: TCA now mandatory infrastructure (~$2.8B market), Arcesium productizing Reconciliation Agent, LEAN BYO-data pain, Databento/Polygon pricing validates self-hosted economics. Sequencing: cost meter + TCA + IB Flex connector first; DuckDB -> Python package; Alpaca adapter feeding W7; W6 studio slice; recon agent after connector coverage."} diff --git a/Directory.Packages.props b/Directory.Packages.props index 9bee5686aa..cf407d5686 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -133,6 +133,7 @@ + diff --git a/docs/generated/repository-structure.md b/docs/generated/repository-structure.md index d614db683d..f829d9887b 100644 --- a/docs/generated/repository-structure.md +++ b/docs/generated/repository-structure.md @@ -1766,6 +1766,7 @@ Meridian-main │ │ └── web-ui-development-pivot.md │ ├── product │ │ ├── deferred-expansion-boundaries.md +│ │ ├── high-value-code-brainstorm-2026-07.md │ │ ├── implementation-todo-list.md │ │ ├── meridian-design-document.md │ │ └── README.md @@ -3581,7 +3582,8 @@ Meridian-main │ │ │ ├── PositionLotSelector.cs │ │ │ ├── PositionReconciliationService.cs │ │ │ ├── PositionSyncOptions.cs -│ │ │ └── ReplayDriftDetector.cs +│ │ │ ├── ReplayDriftDetector.cs +│ │ │ └── SessionTcaReporter.cs │ │ ├── TaxLotAccounting │ │ │ ├── ITaxLotSelector.cs │ │ │ ├── TaxLotAccountingMethod.cs @@ -3974,6 +3976,7 @@ Meridian-main │ │ ├── Reconciliation │ │ │ ├── BrokerStatementInfrastructure.cs │ │ │ ├── BrokerStatementNormalizer.cs +│ │ │ ├── IbFlexStatementService.cs │ │ │ └── ReconciliationCaseInfrastructure.cs │ │ ├── Resilience │ │ │ ├── HttpResiliencePolicy.cs @@ -4482,6 +4485,8 @@ Meridian-main │ │ │ └── PortableDataPackager.Validation.cs │ │ ├── Policies │ │ │ └── JsonlStoragePolicy.cs +│ │ ├── Query +│ │ │ └── DuckDbQueryService.cs │ │ ├── Replay │ │ │ ├── JsonlReplayer.cs │ │ │ └── MemoryMappedJsonlReader.cs @@ -4559,6 +4564,7 @@ Meridian-main │ │ │ ├── DataLineageService.cs │ │ │ ├── DataQualityScoringService.cs │ │ │ ├── DataQualityService.cs +│ │ │ ├── DataReplacementCostEstimator.cs │ │ │ ├── EventBuffer.cs │ │ │ ├── FileMaintenanceService.cs │ │ │ ├── FilePermissionsService.cs @@ -5017,6 +5023,8 @@ Meridian-main │ │ │ │ │ ├── data-screen.evidence-timeline.ts │ │ │ │ │ ├── data-screen.linked-context.ts │ │ │ │ │ ├── data-screen.operator-focus.ts +│ │ │ │ │ ├── data-screen.query-panel.view-model.test.ts +│ │ │ │ │ ├── data-screen.query-panel.view-model.ts │ │ │ │ │ ├── data-screen.security-master.ts │ │ │ │ │ ├── data-screen.test.tsx │ │ │ │ │ ├── data-screen.tsx @@ -6707,6 +6715,7 @@ Meridian-main │ │ │ ├── PaperTradingPortfolioLotSnapshotTests.cs │ │ │ ├── PaperTradingPortfolioTests.cs │ │ │ ├── PositionLotSelectorTests.cs +│ │ │ ├── SessionTcaReporterTests.cs │ │ │ └── TradierExecutionReconciliationTests.cs │ │ ├── FinancialOperations │ │ │ ├── AccountingClose @@ -6956,6 +6965,7 @@ Meridian-main │ │ │ │ ├── statement-invalid-blockers.csv │ │ │ │ └── statement-unresolved-breaks.csv │ │ │ ├── BrokerCustodianMatchingPipelineTests.cs +│ │ │ ├── IbFlexStatementServiceTests.cs │ │ │ ├── ReconciliationCaseServiceTests.cs │ │ │ ├── ReconciliationContractsTests.cs │ │ │ ├── StatementBreakClassifierTests.cs @@ -7036,8 +7046,10 @@ Meridian-main │ │ │ ├── CompositeSinkTests.cs │ │ │ ├── DataLineageServiceTests.cs │ │ │ ├── DataQualityScoringServiceTests.cs +│ │ │ ├── DataReplacementCostEstimatorTests.cs │ │ │ ├── DataValidatorTests.cs │ │ │ ├── DirectLendingMigrationTests.cs +│ │ │ ├── DuckDbQueryServiceTests.cs │ │ │ ├── EventBufferTests.cs │ │ │ ├── ExportValidatorTests.cs │ │ │ ├── FilePermissionsServiceTests.cs diff --git a/docs/product/README.md b/docs/product/README.md index 1a949d3fe1..487f152a84 100644 --- a/docs/product/README.md +++ b/docs/product/README.md @@ -14,6 +14,9 @@ It routes non-technical audiences to verified evidence and prevents duplicate cl - [Meridian Implementation TODO List](implementation-todo-list.md) — single planning-tooling tracker for implemented design-document items and remaining TODOs - [Roadmap Registry](../roadmap/README.md) - [Roadmap Generated Summary](../roadmap/generated/ROADMAP_SUMMARY.md) + - [High-Value Code Brainstorm (2026-07)](high-value-code-brainstorm-2026-07.md) — market-researched + prioritization of the next highest-value implementable features, sequenced against the W6/W7 + roadmap lanes - If you need evidence-backed examples of current operations, check: - [generated roadmap outputs](../roadmap/generated/) - Current project source-of-truth references listed below. diff --git a/docs/product/high-value-code-brainstorm-2026-07.md b/docs/product/high-value-code-brainstorm-2026-07.md new file mode 100644 index 0000000000..dac0cecc7c --- /dev/null +++ b/docs/product/high-value-code-brainstorm-2026-07.md @@ -0,0 +1,380 @@ +# High-Value Code Brainstorm — Market-Researched Prioritization (2026-07) + +> **Mode:** Competitive / Open Exploration hybrid — the request asks for market research plus a +> prioritized set of implementable, high-end-user-value features that strengthen the total product +> value proposition. +> +> **Grounding:** `.claude/skills/_shared/project-context.md`, roadmap registry +> (`docs/roadmap/generated/ROADMAP_SUMMARY.md`, snapshot 2026-06-24), competitive-landscape +> reference, and July 2026 web research (sources at the end). +> +> **Roadmap state:** Waves 1–5X are done (data trust gate, paper cockpit, promotion evidence, +> reconciliation readiness, governed report packs, accounting records, multi-asset proof lane, +> financial-operations control center, shared record explorers). The two planned lanes are +> **W6-BTSTUDIO-001** (Backtesting Studio evidence loop) and **W7-LIVE-001** (live-readiness +> governance). The highest-value code either feeds those lanes directly or widens the moat the +> market research says buyers now pay for. + +--- + +## Market Research Summary + +Five signals from the July 2026 landscape shape this prioritization: + +1. **Fund-ops platforms are converging on AI-assisted reconciliation.** Enfusion (now part of + Clearwater Analytics), Arcesium, and SS&C all market front-to-back cloud platforms; Arcesium is + explicitly productizing a "Reconciliation Agent." Industry surveys report workflow automation + cutting manual reconciliation effort by ~51% and real-time compliance monitoring in use at ~47% + of firms. Reconciliation is described as the most demanded but messiest agentic use case. + Meridian already owns a `CanonicalReconciliationEngine`, a CoS approval-gated runtime, and an + MCP server layer — the ingredients no incumbent ships together in a self-hosted package. +2. **TCA is now mandatory infrastructure, not optional reporting.** The TCA market reached ~$2.8B + with MiFID II-style best-execution obligations spreading beyond Europe; vendors (ION LookOut, + ACA, Quod) sell 80+ metric engines, and quant funds credit TCA with 10–30 bps/yr. Meridian has + TCA models only on the backtest side (`PostSimulationTcaReporter`, `TcaReportModels`) — not on + the paper/live fill tape where the regulatory and operator value lives. +3. **Open-source quant stacks still lack an ops/accounting story.** QuantConnect LEAN self-hosting + is "significant DevOps work and bring-your-own-data"; NautilusTrader is execution-mechanics + strong but has no ledger, reconciliation, or governed reporting. Meridian's front-to-back + self-hosted platform is unique — and interop bridges capture those communities instead of + fighting them. +4. **Market-data economics favor self-hosted collection.** Databento live tiers run $1,399–$3,500/mo + and metered pricing "scales aggressively"; Polygon Stocks Advanced is $199/mo with no + structured local storage. Meridian's multi-provider, own-your-data collection amortizes to near + zero marginal cost — but the value story is invisible unless the product surfaces it and makes + the local store queryable. +5. **Buyers expect multi-asset, real-time consolidation, and open integration.** ~61% of hedge-fund + platform users need multi-asset coverage; 2026 platform-selection commentary emphasizes open + integration frameworks and AI-agent readiness as the reason firms leave legacy systems. + +--- + +## Ideas at a Glance + +| # | Idea | Effort | Audience | Impact | Depends On | +|---|------|--------|----------|--------|------------| +| 1 | Live/Paper TCA & best-execution report pack | M | I | High | — | +| 2 | Live execution hardening: wire `AlpacaBrokerageGateway` into promotion, reconciliation, and governance | M | H, I | High | — | +| 3 | Reconciliation break-resolution agent (MCP + CoS gates) | M | I | High | — | +| 4 | Custodian/broker statement connector library | M | I, H | High | — | +| 5 | DuckDB analytics workbench in the Data workspace | M | Q, H | High | — | +| 6 | Python `meridian` client package | M | H, Q | High | 5 (optional) | +| 7 | W6 Backtesting Studio evidence loop (run comparison + reproducibility manifest) | L | all | High | — | +| 8 | LEAN / NautilusTrader data-feed bridge | M | H, Q | Med-High | 6 (optional) | +| 9 | Governed investor report delivery ("portal-lite") | L | I | Med-High | — | +| 10 | Data-cost savings meter ("what this would have cost") | S | all | Med | — | + +Effort: **S** = days, **M** = 1–2 weeks, **L** = 1+ month. Audience: **H** = hobbyist quant, +**Q** = academic, **I** = institutional/professional. + +> Continuity note: ideas 1, 5, and 6 were surfaced in earlier sessions (2026-03 ledger) as +> promising; this session **promotes them with market evidence** — TCA regulatory demand, data-cost +> economics, and the Python-SDK competitive gap respectively. Ideas 2, 3, 4, 9, and 10 are new +> territory for the ledger. + +--- + +## The Ideas + +### 1. Live/Paper TCA & Best-Execution Report Pack + +Meridian already computes TCA for simulations — `PostSimulationTcaReporter` in +`src/Meridian.Backtesting/Metrics/` and the shared models in +`src/Meridian.Backtesting.Sdk/TcaReportModels.cs`. But the fill tape that operators actually +answer for — paper sessions today, live sessions under W7 — has no cost attribution at all. The +market research is unambiguous: TCA has shifted from optional to mandatory infrastructure, and it +is sold as a $2.8B standalone category. Meridian can ship it as a built-in. + +The move is to lift the TCA model out of the backtest pillar into a shared surface: run the same +implementation-shortfall / arrival-price / spread-capture decomposition over +`OrderManagementSystem` fill history from `PaperTradingGateway` (and any future live gateway), and +emit the result through the existing governed report-pack machinery from W4-RPT-001. The operator +moment: in the Trading workspace, a completed session shows a "Execution cost" chip; clicking it +opens per-order slippage decomposition with benchmark comparisons, and "Add to report pack" +produces the auditable best-execution artifact compliance asks for. + +Tradeoffs: benchmark prices need trustworthy market data at fill timestamps — the W1 data-trust +gate helps, but missing-tick handling must be explicit (grade the report by data confidence rather +than silently interpolating). This is also the single strongest tie-in between the Trading and +Reporting workspaces, so contract placement matters: models belong in a shared SDK location, not +duplicated from Backtesting.Sdk. + +**Why now:** feeds W7 live-readiness directly; institutional differentiator no open-source +competitor has; models already exist and are tested on the backtest side. + +### 2. Live Execution Hardening: Wire `AlpacaBrokerageGateway` Into Promotion, Reconciliation, and Governance + +A broker adapter already exists — `AlpacaBrokerageGateway` +(`src/Meridian.Infrastructure/Adapters/Alpaca/AlpacaBrokerageGateway.cs`) implements +`IBrokerageGateway : IExecutionGateway` (`src/Meridian.Execution.Sdk/IBrokerageGateway.cs`) with +account, position, open-order, and health queries, and the host registers it under the `alpaca` +key. What's missing is everything around it that makes live execution a governed product rather +than a wired-up capability: the promotion workflow (`BacktestToLivePromoter`, paper-first gate +pattern from ADR-015/016) still terminates at paper, and W7-LIVE-001 — live-readiness governance — +has no evidence surface over the real gateway. The competitive scan makes this the sharpest gap: +NautilusTrader's whole pitch is surviving the backtest→live jump. + +The work is hardening, not adapter-building: route the promotion workflow's final step to the +registered brokerage gateway behind `ExecutionMode.Live` plus the `CompositeRiskValidator` +pre-trade chain with an explicit operator sign-off (the approval-gate pattern the CoS runtime was +built for); add broker-state reconciliation that compares `GetPositionsAsync` / +`GetOpenOrdersAsync` broker truth against the local `OrderManagementSystem` view and surfaces +divergence as reconciliation breaks (fills arriving after disconnect are the hard 20%); and emit +live-readiness evidence (gateway health checks, capability declarations, reconciliation status) +into the W7 governance surface. The user moment: the promotion workflow's final step changes from +"paper only" to a governed live toggle whose evidence panel shows broker health and position parity +before the operator signs off. + +Tradeoffs: live order routing is the highest-blast-radius surface in the product. Scope tightly — +promotion wiring, reconciliation, and evidence first; order-type breadth and multi-account later — +and let the risk-rule chain and W7 governance carry the safety story. + +**Why now:** unlocks W7, completes the "collection → backtest → paper → live" value proposition +that the differentiation matrix already promises, and converts the promotion workflow from demo to +product. + +### 3. Reconciliation Break-Resolution Agent (MCP + CoS Approval Gates) + +Arcesium is marketing a "Reconciliation Agent"; surveys put reconciliation automation at the top of +fund-ops AI demand while calling it the messiest use case. Meridian is unusually well positioned: +`CanonicalReconciliationEngine` and `StatementReconciliationOrchestrator` +(`src/Meridian.FinancialOperations/Reconciliation/`) already produce structured breaks, the MCP +server layer (`src/Meridian.Mcp/`, `src/Meridian.McpServer/`) already exposes tools to AI agents, +and the CoS runtime already does approval-gated, evidence-retained orchestration. No competitor +ships this combination self-hosted. + +Build an MCP toolset over the reconciliation surface: `list_open_breaks`, `get_break_evidence` +(both sides, lineage, prior similar breaks), `propose_resolution` (match, adjust, journal-entry +draft), and `submit_for_approval` — the last routed through the CoS approval gate so a human +operator signs off in the accounting screen's close cockpit before anything posts to the `Ledger`. +The operator moment: the accounting workspace's break queue gains a "suggested resolution" column +with confidence and cited evidence; the operator approves, edits, or rejects, and every action +lands in the evidence timeline that already exists in the workstation shell. + +Tradeoffs: the agent must never write directly — the tool contract has to be propose-only with the +approval gate as the sole mutation path, and evaluation needs a replayable corpus of historical +breaks to measure suggestion quality before operators see it. Start with the highest-frequency +break classes (quantity/price mismatches on plain-vanilla positions), not the long tail. + +**Why now:** this is the single feature where Meridian's existing MCP moat converts into +fund-operations differentiation the market is actively naming and pricing. + +### 4. Custodian/Broker Statement Connector Library + +Reconciliation readiness (W4-RECON-001) shipped the engine; `StatementMappingProfiles` shows the +canonical mapping seam. What limits real-world adoption is the unglamorous part: getting actual +custodian and broker statements into canonical form. Every fund-ops platform selection guide +weights connector breadth heavily, and for small funds the connectors *are* the product. + +Ship a connector library as data, not code, wherever possible: an IB Flex Report connector (XML, +well-documented, huge overlap with Meridian's IB users), an Alpaca account-activity connector +(pairs with idea 2), and a declarative CSV/OFX mapping-profile format so operators can onboard a +new custodian format without a release. The operator moment: in the Accounting workspace, "Import +statement" accepts a file or a scheduled fetch, previews the canonical mapping with per-column +confidence, and drops straight into the existing reconciliation queue. A mapping-profile editor +with live preview turns a support burden into a self-service surface. + +Tradeoffs: statement formats drift; each connector needs golden-file regression tests and a +versioned profile schema (respect ADR-014 source-generated JSON and route persistence through +`AtomicFileWriter`). Scheduled fetching pulls in credential storage — reuse the desktop persistence +baseline rather than inventing a new secret store. + +**Why now:** multiplies the value of already-shipped W4/W5 lanes and of idea 3 (the agent is only +as good as statement coverage); high end-user value per unit effort for the fund-ops persona. + +### 5. DuckDB Analytics Workbench in the Data Workspace + +The data-economics research shows Meridian's core advantage — self-hosted, no per-query fees — but +that advantage is latent while the local Parquet/JSONL store is only reachable through files. +`PortableDataPackager` already generates DuckDB SQL scripts +(`src/Meridian.Storage/Packaging/PortableDataPackager.Scripts.Sql.cs`), so the schema mapping is +proven; the step is embedding DuckDB (via `DuckDB.NET`, added through central package management) +behind a read-only query service and a workbench panel in the Data workspace. + +The user moment: a "Query" tab in the Data screen with a SQL editor, schema browser listing the +catalog from `StorageCatalogService`, result grid, and one-click export to Parquet/CSV. A hobbyist +answers "show me AAPL spread by minute for March" without leaving the app; an academic validates a +dataset before export. Expose the same service as `/api/query` through +`WorkstationEndpoints` so the browser and WPF surfaces share it — and idea 6 gets a query +transport for free. + +Tradeoffs: unbounded SQL over large stores needs guardrails — read-only connection, query timeout, +row limits, and memory caps; queries must run out-of-process from the hot ingestion path so a bad +scan can't stall `EventPipeline` flushes. + +**Why now:** turns the "own your data" cost advantage into a daily-felt feature; validated demand +(Polygon now sells an SQL query option as a premium capability); prior-session idea now backed by +market evidence. + +### 6. Python `meridian` Client Package + +The competitive matrix has shown "Python SDK: No / Planned" against Bloomberg, Databento, and +Polygon all shipping one since the first landscape scan — and the 2026 backtesting-landscape +commentary confirms Python remains the community's center of gravity. Every quarter without it, +Meridian's collection and backtest evidence is invisible to the tools quants actually analyze in. + +Scope v1 as a thin, honest client over the existing HTTP surface (`WorkstationEndpoints` plus +`/api/query` from idea 5): `meridian.history(symbols, start, end)` returning a pandas DataFrame, +`meridian.live(symbols)` as an async iterator over the WebSocket feed, `meridian.runs()` / +`meridian.run(id)` fetching backtest and paper-session results with their metrics, and +`meridian.sql(query)` when the workbench lands. Publish to PyPI with a README notebook that goes +from `pip install meridian-client` to a plotted equity curve in ten lines. The user moment is the +hobbyist's first Jupyter cell showing live ticks from their own self-hosted collector. + +Tradeoffs: it's a second artifact with its own release cadence and API-compatibility surface — +version it against the workstation API contract and generate models from the OpenAPI spec rather +than hand-maintaining them. Resist scope creep toward a full ORM; the DataFrame boundary is the +product. + +**Why now:** cheapest way to widen the funnel to the largest persona; unlocks idea 8; repeatedly +deferred while remaining the most-cited competitive gap. + +### 7. W6 Backtesting Studio Evidence Loop — Run Comparison + Reproducibility Manifest + +This is the planned roadmap lane (W6-BTSTUDIO-001), and prior sessions already ideated its parts: +multi-run comparison, reproducibility manifests, promotion evidence. The market angle from this +session's research: LEAN's weakness is DevOps and data friction, NautilusTrader's is +research-to-ops workflow — a studio where every backtest is a *governed, reproducible evidence +artifact* is the differentiated version of "backtesting UI," consistent with Meridian's +evidence-led identity rather than a me-too chart page. + +Concretely: every run in `StrategyRunStore` gains a reproducibility manifest (code/strategy +version, parameter set, data-catalog snapshot hash, fill-model config, seed); the Strategy +workspace gains a run-comparison view (equity curves overlaid, metric deltas, parameter diff) built +on `StrategyRunReadService`; and the existing paper-promotion workflow consumes the manifest as its +evidence input, closing the loop W2/W3 opened. The operator moment: select two runs, see exactly +what changed and what it did to the numbers, and promote with the manifest attached — the +comparison view *is* the promotion justification. + +Tradeoffs: it's the largest item here; sequence it as manifest first (S–M, pure data model), then +comparison view (M), then promotion-evidence wiring (S). Data-snapshot hashing needs a cheap +catalog-level identity, not content-hashing terabytes. + +**Why now:** it is literally the next roadmap item, and manifest + comparison is the highest-value +slice of it; ideas 1 and 2 both strengthen it (TCA metrics appear in run comparison; live adapter +gives promotion somewhere to go). + +### 8. LEAN / NautilusTrader Data-Feed Bridge + +The open-source scan's clearest finding: LEAN self-hosters must "bring your own data," and that +pain is documented in their own community. Meridian can be the collector that feeds your +backtesting framework — an on-ramp that meets quants where they already are and makes Meridian's +data layer sticky even for users who never open the workstation. + +Ship two thin bridges: a LEAN-format exporter (LEAN's zip/csv data folder layout for equities, +minute/second resolution) runnable as `meridian export --format lean`, and a NautilusTrader loader +in the Python package (idea 6) mapping Meridian Parquet to Nautilus data objects. The user moment: +a LEAN user points their config at a Meridian-exported data folder and their existing algorithm +just runs — with better, gap-audited data than they had. + +Tradeoffs: format fidelity is the whole game — golden-file tests against LEAN's own sample data; +corporate-action handling differences need explicit documentation. Keep it export-shaped (batch) +rather than implementing LEAN's live `IDataQueueHandler` in v1. + +**Why now:** cheap growth surface with a defined target format; converts competitor communities +into Meridian data users rather than fighting for their whole stack. + +### 9. Governed Investor Report Delivery ("Portal-Lite") + +W4-RPT-001 and W5X shipped governed report packs and delivery-history UI plumbing +(`reporting-screen.delivery-history.tsx`, branding-access modules already exist in the dashboard). +The fund-ops market research shows client-facing transparency is a top selection criterion, and +"client portal" is already a sanctioned expansion lane in the project context. Portal-lite is the +minimal version: branded PDF/HTML render of an approved report pack, tokenized read-only share +links with expiry, and a delivery audit trail (who accessed what, when) folded into the existing +evidence timeline. + +The operator moment: from the Reporting workspace, "Deliver" on an approved pack generates the +branded artifact and a revocable link; the delivery-history panel shows access events. No investor +login system, no new top-level navigation — it stays inside the Reporting workspace and the +existing approval gates. + +Tradeoffs: anything outward-facing raises the security bar — signed URLs, expiry, revocation, and +rate limiting are table stakes; rendering fidelity for branded PDFs is a real time sink. Defer +investor identity/accounts entirely; that's the full-portal lane. + +**Why now:** high perceived value for the fund persona at modest incremental cost because the +approval, branding, and delivery-history seams already exist. + +### 10. Data-Cost Savings Meter + +Small, sharp, and directly from the pricing research: Meridian knows exactly what it collected +(symbols × days × depth via `StorageCatalogService` and Prometheus counters). Price that against +public rate cards (Databento ~$0.10–$1.00/symbol-day historical, live tiers $1,399+/mo; Polygon +$199/mo) and show a running "estimated replacement cost" of the user's self-hosted dataset — on the +Data workspace and in the governed report pack summary. + +The user moment: the Data screen header reads "Your local store: 214 GB · est. replacement cost +$3,120/mo on metered vendors," with a drill-down by asset class and a methodology note. For a fund +operator it quantifies the platform in budget language; for a hobbyist it's the screenshot they +post. It's also honest marketing: the rate-card table lives in a config file with citations and a +"last verified" date, not hardcoded claims. + +Tradeoffs: pricing comparisons go stale and must be conservative and clearly labeled as estimates; +keep the mapping simple (per symbol-day by resolution tier) rather than simulating vendor bills +precisely. + +**Why now:** days of effort, makes the platform's core economic moat visible every single session, +and strengthens every sales/adoption conversation. + +--- + +## Synthesis + +**Highest-leverage single item: #1, Live/Paper TCA.** The models exist, the report-pack machinery +exists, the market prices TCA as mandatory infrastructure, and it upgrades three surfaces at once +(Trading, Reporting, and the W6 run-comparison view). Best impact-per-effort in the set. + +**Platform bets:** #2 (live broker adapter) and #5 (DuckDB service). The adapter converts the +promotion workflow and W7 from governance-without-an-object into the completed +collection→backtest→paper→live story; the query service becomes the shared transport under the +workbench UI, the Python package, the cost meter's drill-downs, and future analytics surfaces. + +**Cross-cutting theme:** almost everything here is *evidence productization* — TCA artifacts, +reconciliation evidence, reproducibility manifests, delivery audit trails, even the cost meter. +Meridian's identity ("evidence-backed investment operations") is also its differentiation; a shared +evidence-artifact pattern (typed payload + lineage + approval state + timeline entry) would keep +these five features from inventing five formats. + +**Sequencing recommendation:** + +1. **Now (parallel lanes):** #10 cost meter (S, instant story) · #1 TCA on the paper fill tape (M) + · #4 statement connectors starting with IB Flex (M). +2. **Next:** #5 DuckDB service + workbench (M) → #6 Python package (M) riding the same API · #2 + Alpaca adapter (M) feeding W7. +3. **Then:** #7 W6 studio slice (manifest → comparison → promotion evidence), enriched by TCA + metrics from step 1 · #3 reconciliation agent once connector coverage (#4) gives it enough + break volume to learn from. +4. **Later:** #8 LEAN bridge (after #6) · #9 portal-lite (after a security review of outward + surfaces). + +**Competitive signals:** Bloomberg-tier incumbents and the fund-ops platforms (Enfusion/Clearwater, +Arcesium, SS&C) are selling exactly two things Meridian can ship in code this quarter — +AI-assisted reconciliation and best-execution evidence — but only as cloud services with +per-seat pricing. Databento/Polygon own developer mindshare through Python-first ergonomics and +now sell SQL-over-your-data as premium features; Meridian's self-hosted architecture delivers both +without metering, and the DuckDB + Python pair is the most adaptable pattern to borrow. The +open-source engines (LEAN, NautilusTrader) validate that no one else owns the governed +research→paper→live→books loop — which is why the W6/W7 lanes plus TCA are the moat-wideners, not +just roadmap chores. + +--- + +## Sources + +- [Limina — Enfusion (now part of Clearwater Analytics) competitors & alternatives](https://www.limina.com/enfusion) +- [Arcesium — The Arrival of the Reconciliation Agent](https://www.arcesium.com/blog/real-time-reconciliation-agent) +- [Arcesium — Data & operations approach for launching a new hedge fund](https://www.arcesium.com/blog/what-is-the-best-data-operations-approach-for-launching-a-new-hedge-fund) +- [FundCount — Best hedge fund portfolio management software (2026)](https://fundcount.com/best-hedge-fund-portfolio-management-software/) +- [360MarketUpdates — Hedge fund software market size and forecast 2026–2035](https://www.360marketupdates.com/market-reports/hedge-fund-software-market-400051) +- [StackAI — Agentic AI in multi-strategy hedge fund operations](https://www.stackai.com/insights/agentic-ai-in-multi-strategy-hedge-fund-operations-practical-use-cases-automation-and-governance-for-bam-style-platforms) +- [Digiqt — AI agents in hedge funds: use cases (2026)](https://digiqt.com/blog/ai-agents-in-hedge-funds/) +- [Finantrix — Buyer's guide: TCA tools for quantitative funds](https://www.finantrix.com/buyer-guides/transaction-cost-analysis-tools-quant-funds) +- [PR Newswire — ION LookOut TCA wins RegTech Insight Award Europe 2026](https://www.prnewswire.com/news-releases/ion-lookout-tca-wins-best-transaction-cost-analysis-solution-for-best-execution-at-regtech-insight-awards--europe-2026-302778146.html) +- [A-Team Insight — Top transaction cost analysis solutions](https://a-teaminsight.com/blog/the-top-transaction-cost-analysis-tca-solutions/) +- [python.financial — The Python backtesting landscape (2026)](https://python.financial/) +- [QuantConnect LEAN — GitHub](https://github.com/QuantConnect/Lean) +- [AI Fin Hub — Market data APIs compared: Databento vs Polygon 2026](https://aifinhub.io/articles/market-data-apis-compared-2026/) +- [edgeful — Futures data API comparison: Polygon vs Databento (2026)](https://www.edgeful.com/blog/posts/futures-data-api-polygon-databento-edgeful-comparison) +- [Databento — Pricing](https://databento.com/pricing) +- [Alphanume — Best market data APIs for algorithmic trading in 2026](https://www.alphanume.com/blog/best-market-data-apis-for-algorithmic-trading-in-2026) diff --git a/docs/status/TODO.md b/docs/status/TODO.md index 7728e0052a..0b83773efe 100644 --- a/docs/status/TODO.md +++ b/docs/status/TODO.md @@ -43,11 +43,11 @@ Total items: **200** | `src/Meridian.Ui.Services/Services/ProviderHealthService.cs` | 530 | `NOTE` | ❌ | // NOTE: ProviderComparison is defined in AdvancedAnalyticsModels.cs for cross-provider comparison | | `src/Meridian.Ui.Shared/Endpoints/ArchiveMaintenanceEndpoints.cs` | 32 | `NOTE` | ❌ | // NOTE: GET /schedules, GET /schedules/{id}, POST /schedules, POST /schedules/{id}/enable, | | `src/Meridian.Ui.Shared/Endpoints/ArchiveMaintenanceEndpoints.cs` | 117 | `NOTE` | ❌ | // NOTE: POST /schedules/{id}/enable and POST /schedules/{id}/disable are registered | -| `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs` | 4245 | `NOTE` | ❌ | note: "Paper adapter routing is available.", | -| `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs` | 4257 | `NOTE` | ❌ | note: "Realtime subscriptions are steady.", | -| `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs` | 4269 | `NOTE` | ❌ | note: "Replay queue is elevated but within tolerance.", | -| `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs` | 4381 | `NOTE` | ❌ | Note: note, | -| `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs` | 4464 | `NOTE` | ❌ | Note: note, | +| `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs` | 4332 | `NOTE` | ❌ | note: "Paper adapter routing is available.", | +| `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs` | 4344 | `NOTE` | ❌ | note: "Realtime subscriptions are steady.", | +| `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs` | 4356 | `NOTE` | ❌ | note: "Replay queue is elevated but within tolerance.", | +| `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs` | 4468 | `NOTE` | ❌ | Note: note, | +| `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs` | 4551 | `NOTE` | ❌ | Note: note, | | `src/Meridian.Ui.Shared/Services/ProviderLedgerReconciliationService.cs` | 1974 | `NOTE` | ❌ | Note: "Provider-ledger reconciliation break signed off.", | | `src/Meridian.Ui.Shared/Services/ReportPackDeliveryService.cs` | 240 | `NOTE` | ❌ | Note: NormalizeNullable(target.Note) ?? $"Scheduled delivery for {normalizedTemplateId}.", | | `src/Meridian.Ui.Shared/Services/ReportPackRunReadService.cs` | 2047 | `NOTE` | ❌ | Note: NormalizeOptional(target.Note), | @@ -174,10 +174,10 @@ Total items: **200** | `src/Meridian.Ui/dashboard/src/screens/w4-acceptance-parity.test.ts` | 429 | `NOTE` | ❌ | note: "Close evidence reviewed." | | `src/Meridian.Ui/dashboard/src/screens/w4-acceptance-parity.test.ts` | 437 | `NOTE` | ❌ | note: "Published to investor portal." | | `src/Meridian.Ui/dashboard/src/types.ts` | 3924 | `NOTE` | ❌ | note: string; | -| `src/Meridian.Ui/dashboard/src/types.ts` | 4895 | `NOTE` | ❌ | note: string \| null; | -| `src/Meridian.Ui/dashboard/src/types.ts` | 5248 | `NOTE` | ❌ | note: string \| null; | -| `src/Meridian.Ui/dashboard/src/types.ts` | 5297 | `NOTE` | ❌ | note: string \| null; | -| `src/Meridian.Ui/dashboard/src/types.ts` | 5494 | `NOTE` | ❌ | note: string \| null; | +| `src/Meridian.Ui/dashboard/src/types.ts` | 4910 | `NOTE` | ❌ | note: string \| null; | +| `src/Meridian.Ui/dashboard/src/types.ts` | 5263 | `NOTE` | ❌ | note: string \| null; | +| `src/Meridian.Ui/dashboard/src/types.ts` | 5312 | `NOTE` | ❌ | note: string \| null; | +| `src/Meridian.Ui/dashboard/src/types.ts` | 5509 | `NOTE` | ❌ | note: string \| null; | | `src/Meridian.Wpf/GlobalUsings.cs` | 7 | `NOTE` | ❌ | // NOTE: Type aliases and Contracts namespaces are NOT re-defined here because | | `src/Meridian.Wpf/ViewModels/SecurityPassportEditorViewModel.cs` | 262 | `NOTE` | ❌ | Note: null, | | `tests/Meridian.Tests/Application/Backfill/BackfillWorkerServiceTests.cs` | 28 | `NOTE` | ❌ | // NOTE: Using null! because validation throws before dependencies are accessed | diff --git a/docs/status/api-contract-coverage-dashboard.json b/docs/status/api-contract-coverage-dashboard.json index bf113ba3a8..0d2c7710fd 100644 --- a/docs/status/api-contract-coverage-dashboard.json +++ b/docs/status/api-contract-coverage-dashboard.json @@ -3533,13 +3533,13 @@ { "method": "GET", "path": "/api/strategies/{strategyId}/runs", - "source": "src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs:2528", + "source": "src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs:2576", "documented": true }, { "method": "GET", "path": "/api/strategies/runs/compare", - "source": "src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs:2649", + "source": "src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs:2697", "documented": true } ], diff --git a/docs/status/api-contract-coverage-dashboard.md b/docs/status/api-contract-coverage-dashboard.md index a726583009..1024f01265 100644 --- a/docs/status/api-contract-coverage-dashboard.md +++ b/docs/status/api-contract-coverage-dashboard.md @@ -579,8 +579,8 @@ Tracks whether mapped API routes and workstation DTO contracts are visible in th | `POST` | `/api/strategies/covered-call/runs/{runId}/cancel` | Documented | `src/Meridian.Ui.Shared/Endpoints/CoveredCallEndpoints.cs:135` | | `GET` | `/api/strategies/covered-call/runs/{runId}/result` | Documented | `src/Meridian.Ui.Shared/Endpoints/CoveredCallEndpoints.cs:93` | | `GET` | `/api/strategies/covered-call/runs/{runId}/status` | Documented | `src/Meridian.Ui.Shared/Endpoints/CoveredCallEndpoints.cs:70` | -| `GET` | `/api/strategies/runs/compare` | Documented | `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs:2649` | -| `GET` | `/api/strategies/{strategyId}/runs` | Documented | `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs:2528` | +| `GET` | `/api/strategies/runs/compare` | Documented | `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs:2697` | +| `GET` | `/api/strategies/{strategyId}/runs` | Documented | `src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs:2576` | | `GET` | `/api/subscriptions/active` | Documented | `src/Meridian.Ui.Shared/Endpoints/SubscriptionEndpoints.cs:21` | | `POST` | `/api/subscriptions/subscribe` | Documented | `src/Meridian.Ui.Shared/Endpoints/SubscriptionEndpoints.cs:43` | | `POST` | `/api/subscriptions/unsubscribe/{symbol}` | Documented | `src/Meridian.Ui.Shared/Endpoints/SubscriptionEndpoints.cs:72` | diff --git a/docs/status/coverage-report.md b/docs/status/coverage-report.md index 14b63018fe..d88c519907 100644 --- a/docs/status/coverage-report.md +++ b/docs/status/coverage-report.md @@ -5,7 +5,7 @@ ## Overall Coverage -**2598 / 7240** items documented (**35.9%**) — Grade: **F** +**2602 / 7258** items documented (**35.9%**) — Grade: **F** ```text [=======-------------] 35.9% @@ -15,15 +15,15 @@ | Category | Documented | Total | Coverage | Grade | | ---------- | ----------- | ------- | ---------- | ------- | -| Public Classes / Interfaces | 2477 | 6745 | 36.7% | F | -| API Endpoints | 107 | 348 | 30.7% | F | +| Public Classes / Interfaces | 2481 | 6762 | 36.7% | F | +| API Endpoints | 107 | 349 | 30.7% | F | | Configuration Options | 3 | 136 | 2.2% | F | | Provider Implementations | 0 | 0 | 100.0% | A | | ADR Implementations | 11 | 11 | 100.0% | A | ## Undocumented Items -### Public Classes / Interfaces (4268 undocumented) +### Public Classes / Interfaces (4281 undocumented) | Item | Location | | ------ | ---------- | @@ -77,9 +77,9 @@ | `SecurityMasterActiveImportStatus` | `src/Meridian.Application/SecurityMaster/SecurityMasterIngestStatusService.cs:13` | | `SecurityMasterCompletedImportStatus` | `src/Meridian.Application/SecurityMaster/SecurityMasterIngestStatusService.cs:26` | | `ISecurityMasterIngestStatusService` | `src/Meridian.Application/SecurityMaster/SecurityMasterIngestStatusService.cs:41` | -| ... and 4218 more | | +| ... and 4231 more | | -### API Endpoints (241 undocumented) +### API Endpoints (242 undocumented) | Item | Location | | ------ | ---------- | @@ -133,7 +133,7 @@ | `/{loanId:guid}/status-transitions` | `src/Meridian.Ui.Shared/Endpoints/DirectLendingEndpoints.cs:685` | | `/{loanId:guid}/pik` | `src/Meridian.Ui.Shared/Endpoints/DirectLendingEndpoints.cs:715` | | `/{loanId:guid}/restructures` | `src/Meridian.Ui.Shared/Endpoints/DirectLendingEndpoints.cs:745` | -| ... and 191 more | | +| ... and 192 more | | ### Configuration Options (133 undocumented) @@ -193,8 +193,8 @@ ## Recommendations -1. **Public Classes / Interfaces**: 4268 undocumented types. Consider generating API docs with DocFX (`docfx docfx.json`) to cover the long tail of public types automatically. -2. **API Endpoints**: 241 endpoint(s) missing from `docs/reference/api-reference.md`. Run the endpoint audit and update the API reference table. +1. **Public Classes / Interfaces**: 4281 undocumented types. Consider generating API docs with DocFX (`docfx docfx.json`) to cover the long tail of public types automatically. +2. **API Endpoints**: 242 endpoint(s) missing from `docs/reference/api-reference.md`. Run the endpoint audit and update the API reference table. 3. **Configuration Options**: 133 config key(s) not found in `docs/generated/configuration-schema.md`. Re-run the configuration schema generator to synchronise. --- diff --git a/docs/status/doc-health-dashboard.json b/docs/status/doc-health-dashboard.json index e5562fce58..f251df4cc6 100644 --- a/docs/status/doc-health-dashboard.json +++ b/docs/status/doc-health-dashboard.json @@ -1,11 +1,11 @@ { - "total_files": 534, - "total_lines": 87263, + "total_files": 535, + "total_lines": 87658, "orphaned_count": 204, "no_heading_count": 38, "stale_count": 0, "todo_count": 199, - "average_lines": 163.4, + "average_lines": 163.8, "health_score": 83, "orphaned_files": [ ".agents/skills/meridian-archive-organizer/SKILL.md", @@ -2129,7 +2129,7 @@ }, { "path": "docs/generated/repository-structure.md", - "line_count": 7663, + "line_count": 7675, "has_heading": true, "todo_count": 9, "last_modified_utc": "1970-01-01T00:00:00+00:00", @@ -3055,6 +3055,14 @@ "last_modified_utc": "1970-01-01T00:00:00+00:00", "stale": false }, + { + "path": "docs/product/high-value-code-brainstorm-2026-07.md", + "line_count": 380, + "has_heading": true, + "todo_count": 0, + "last_modified_utc": "1970-01-01T00:00:00+00:00", + "stale": false + }, { "path": "docs/product/implementation-todo-list.md", "line_count": 186, @@ -3073,7 +3081,7 @@ }, { "path": "docs/product/README.md", - "line_count": 198, + "line_count": 201, "has_heading": true, "todo_count": 2, "last_modified_utc": "1970-01-01T00:00:00+00:00", diff --git a/docs/status/doc-health-dashboard.md b/docs/status/doc-health-dashboard.md index 999c80adc1..534ca34228 100644 --- a/docs/status/doc-health-dashboard.md +++ b/docs/status/doc-health-dashboard.md @@ -19,9 +19,9 @@ Data sources: `repo markdown (*.md)`, `file modification metadata` | Metric | Value | | -------- | ------- | -| Total documentation files | 534 | -| Total lines | 87,263 | -| Average file size (lines) | 163.4 | +| Total documentation files | 535 | +| Total lines | 87,658 | +| Average file size (lines) | 163.8 | | Orphaned files | 204 | | Files without headings | 38 | | Stale files (>90 days) | 0 | @@ -93,7 +93,7 @@ These files are not linked from any other Markdown file in the repository: | Date | Score | Files | Orphans | Stale | | ------ | ------- | ------- | --------- | ------- | -| 1970-01-01 | 83 | 534 | 204 | 0 | +| 1970-01-01 | 83 | 535 | 204 | 0 | --- diff --git a/docs/status/example-validation.md b/docs/status/example-validation.md index c8f8906f91..170437edfc 100644 --- a/docs/status/example-validation.md +++ b/docs/status/example-validation.md @@ -2,7 +2,7 @@ > Auto-generated by `validate-examples.py`. Do not edit manually. > Scan time: 1970-01-01T00:00:00+00:00 -> Files scanned: 418 +> Files scanned: 419 ## Overall Summary diff --git a/src/Meridian.Application/Composition/Features/StorageFeatureRegistration.cs b/src/Meridian.Application/Composition/Features/StorageFeatureRegistration.cs index d5430a17e0..ad55e92264 100644 --- a/src/Meridian.Application/Composition/Features/StorageFeatureRegistration.cs +++ b/src/Meridian.Application/Composition/Features/StorageFeatureRegistration.cs @@ -55,6 +55,7 @@ using Meridian.Storage.Ledger; using Meridian.Storage.Maintenance; using Meridian.Storage.Policies; +using Meridian.Storage.Query; using Meridian.Storage.SecurityMaster; using Meridian.Storage.Services; using Meridian.Workflow.EnvironmentDesign; @@ -164,6 +165,9 @@ public IServiceCollection Register(IServiceCollection services, CompositionOptio }); services.TryAddSingleton(sp => sp.GetRequiredService()); services.TryAddEnumerable(ServiceDescriptor.Singleton()); + services.TryAddSingleton(sp => new DuckDbQueryService( + sp.GetRequiredService(), + sp.GetRequiredService())); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Meridian.Contracts/Api/UiApiRoutes.cs b/src/Meridian.Contracts/Api/UiApiRoutes.cs index 330ac7ac73..fb7b6cf503 100644 --- a/src/Meridian.Contracts/Api/UiApiRoutes.cs +++ b/src/Meridian.Contracts/Api/UiApiRoutes.cs @@ -606,6 +606,8 @@ public static class UiApiRoutes public const string WorkstationTrading = "/api/workstation/trading"; public const string WorkstationDataOperations = "/api/workstation/data-operations"; public const string WorkstationData = "/api/workstation/data"; + public const string WorkstationDataQuery = "/api/workstation/data/query"; + public const string WorkstationDataReplacementCost = "/api/workstation/data/replacement-cost"; public const string WorkstationDataUploadTemplates = "/api/workstation/data/uploads/templates"; public const string WorkstationDataUploadPreview = "/api/workstation/data/uploads/preview"; public const string WorkstationBankStatementImport = "/api/workstation/data/uploads/bank-statements/import"; diff --git a/src/Meridian.Execution/Services/SessionTcaReporter.cs b/src/Meridian.Execution/Services/SessionTcaReporter.cs new file mode 100644 index 0000000000..753e24deba --- /dev/null +++ b/src/Meridian.Execution/Services/SessionTcaReporter.cs @@ -0,0 +1,433 @@ +using Meridian.Execution.Sdk; + +namespace Meridian.Execution.Services; + +/// +/// Generates a Transaction Cost Analysis (TCA) report from a paper/live session fill tape. +/// Computes commission attribution, per-symbol cost breakdowns, outlier detection, and +/// execution-quality statistics (time-to-fill, limit-price improvement). This is a +/// post-processing pass over the session's persisted history; +/// it requires no market data beyond the fills and orders already recorded by the session. +/// +public static class SessionTcaReporter +{ + // Fills whose commission rate exceeds this multiplier × the median per-fill rate are flagged. + // The median (not the aggregate mean) is used so that a single outlier fill cannot inflate + // the baseline threshold enough to avoid detection. + private const double OutlierThresholdMultiplier = 3.0; + + // Suppress outlier flagging for fills below this minimum rate (bps) to avoid noise on + // zero- or near-zero-commission fills. + private const double OutlierMinimumRateBps = 1.0; + + /// + /// Generates a for from the + /// session's fill history. Only reports of type or + /// with a fill price and positive filled + /// quantity contribute. Report sequences whose per-order quantities are cumulative (as + /// emitted by brokers that report running totals, e.g. the IB gateway) are converted to + /// incremental quantities before aggregation. is optional and + /// enriches the report with time-to-fill and limit-price-improvement statistics. + /// Returns a zero-valued report when no usable fills exist. + /// + public static SessionTcaReport Generate( + string sessionId, + string strategyId, + IReadOnlyList fills, + IReadOnlyList? orders = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(sessionId); + ArgumentNullException.ThrowIfNull(fills); + + // Last-writer-wins on duplicate order updates: the latest OrderState for an order id + // carries the final limit price, creation timestamp, and executed quantity. + var ordersById = new Dictionary(StringComparer.Ordinal); + if (orders is not null) + { + foreach (var order in orders) + ordersById[order.OrderId] = order; + } + + var normalized = NormalizeFills(fills, ordersById); + + if (normalized.Count == 0) + { + return new SessionTcaReport( + sessionId, + strategyId, + DateTimeOffset.UtcNow, + new SessionTcaCostSummary(0m, 0m, 0m, 0m, 0.0, 0, 0, 0), + Array.Empty(), + Array.Empty(), + SessionTcaExecutionQuality.Empty); + } + + var costSummary = BuildCostSummary(normalized); + var symbolSummaries = BuildSymbolSummaries(normalized); + var outliers = DetectOutliers(normalized); + var executionQuality = BuildExecutionQuality(normalized, ordersById); + + return new SessionTcaReport( + sessionId, + strategyId, + DateTimeOffset.UtcNow, + costSummary, + symbolSummaries, + outliers, + executionQuality); + } + + /// A fill with its effective (incremental) executed quantity. + private sealed record TcaFill( + string OrderId, + string Symbol, + OrderSide Side, + decimal Quantity, + decimal Price, + decimal Commission, + DateTimeOffset Timestamp) + { + public decimal Notional => Quantity * Price; + } + + /// + /// Filters the raw report tape down to usable fills and converts cumulative per-order + /// quantity sequences into incremental ones. A per-order sequence is treated as cumulative + /// when it contains multiple reports whose quantities never decrease and whose sum exceeds + /// the executed quantity known for the order — either the order quantity (a 40-share + /// partial followed by a 100-share final report on a 100-share order describes 100 executed + /// shares, not 140) or, for orders still working, the executed quantity recorded in order + /// history (cumulative partials of 10 then 20 against a 100-share order describe 20 + /// executed shares, not 30). Incremental tapes (e.g. the paper gateway) sum exactly to the + /// executed quantity and are unaffected. When neither signal is available the sequence is + /// left as-is because the two encodings cannot be distinguished. + /// + private static List NormalizeFills( + IReadOnlyList fills, + IReadOnlyDictionary ordersById) + { + var usable = new List(fills.Count); + foreach (var fill in fills) + { + if (fill.ReportType is ExecutionReportType.Fill or ExecutionReportType.PartialFill + && fill.FillPrice is > 0m + && fill.FilledQuantity > 0m) + { + usable.Add(fill); + } + } + + var result = new List(usable.Count); + foreach (var orderGroup in usable.GroupBy(static f => f.OrderId, StringComparer.Ordinal)) + { + var sequence = orderGroup.OrderBy(static f => f.Timestamp).ToList(); + ordersById.TryGetValue(orderGroup.Key, out var orderState); + var isCumulative = IsCumulativeSequence(sequence, orderState); + + decimal previousCumulative = 0m; + foreach (var report in sequence) + { + var quantity = isCumulative + ? report.FilledQuantity - previousCumulative + : report.FilledQuantity; + previousCumulative = report.FilledQuantity; + + if (quantity <= 0m) + continue; + + result.Add(new TcaFill( + report.OrderId, + report.Symbol, + report.Side, + quantity, + report.FillPrice!.Value, + report.Commission ?? 0m, + report.Timestamp)); + } + } + + result.Sort(static (a, b) => a.Timestamp.CompareTo(b.Timestamp)); + return result; + } + + private static bool IsCumulativeSequence(List sequence, OrderState? orderState) + { + if (sequence.Count < 2) + return false; + + decimal sum = 0m; + decimal previous = 0m; + foreach (var report in sequence) + { + if (report.FilledQuantity < previous) + return false; + previous = report.FilledQuantity; + sum += report.FilledQuantity; + } + + var orderQuantity = sequence[^1].OrderQuantity; + if (orderQuantity > 0m && sum > orderQuantity) + return true; + + // Order still working: compare against the executed quantity the order history knows + // about. A cumulative tape always sums past it; an incremental tape sums exactly to it. + return orderState is { FilledQuantity: > 0m } && sum > orderState.FilledQuantity; + } + + private static SessionTcaCostSummary BuildCostSummary(List fills) + { + decimal totalCommissions = 0m; + decimal totalBuyNotional = 0m; + decimal totalSellNotional = 0m; + int buyFills = 0; + int sellFills = 0; + + foreach (var fill in fills) + { + totalCommissions += fill.Commission; + + if (fill.Side == OrderSide.Buy) + { + totalBuyNotional += fill.Notional; + buyFills++; + } + else + { + totalSellNotional += fill.Notional; + sellFills++; + } + } + + var totalNotional = totalBuyNotional + totalSellNotional; + var avgCommissionRateBps = totalNotional > 0 + ? (double)(totalCommissions / totalNotional) * 10_000.0 + : 0.0; + + return new SessionTcaCostSummary( + totalCommissions, + totalBuyNotional, + totalSellNotional, + totalNotional, + Math.Round(avgCommissionRateBps, 2), + fills.Count, + buyFills, + sellFills); + } + + private static IReadOnlyList BuildSymbolSummaries(List fills) + { + var grouped = new Dictionary>(StringComparer.Ordinal); + foreach (var fill in fills) + { + if (!grouped.TryGetValue(fill.Symbol, out var bucket)) + { + bucket = new List(); + grouped[fill.Symbol] = bucket; + } + bucket.Add(fill); + } + + var summaries = new List(grouped.Count); + foreach (var (symbol, symbolFills) in grouped) + { + decimal buyNotional = 0m, sellNotional = 0m, commission = 0m; + decimal buyQty = 0m, sellQty = 0m; + decimal buyWeighted = 0m, sellWeighted = 0m; + + foreach (var fill in symbolFills) + { + commission += fill.Commission; + + if (fill.Side == OrderSide.Buy) + { + buyNotional += fill.Notional; + buyQty += fill.Quantity; + buyWeighted += fill.Quantity * fill.Price; + } + else + { + sellNotional += fill.Notional; + sellQty += fill.Quantity; + sellWeighted += fill.Quantity * fill.Price; + } + } + + var symbolNotional = buyNotional + sellNotional; + var symbolBps = symbolNotional > 0 + ? (double)(commission / symbolNotional) * 10_000.0 + : 0.0; + + summaries.Add(new SessionSymbolTcaSummary( + symbol, + buyNotional, + sellNotional, + Math.Round(buyQty > 0 ? buyWeighted / buyQty : 0m, 4), + Math.Round(sellQty > 0 ? sellWeighted / sellQty : 0m, 4), + commission, + Math.Round(symbolBps, 2), + symbolFills.Count)); + } + + // Sort descending by total commission (highest cost symbols first). + summaries.Sort(static (a, b) => b.TotalCommission.CompareTo(a.TotalCommission)); + return summaries; + } + + private static IReadOnlyList DetectOutliers(List fills) + { + // The median (not the aggregate mean) is used as the baseline so that a single high-cost + // fill cannot inflate the threshold enough to avoid detection. + var perFillBps = new List(fills.Count); + foreach (var fill in fills) + { + if (fill.Notional > 0m) + perFillBps.Add((double)(fill.Commission / fill.Notional) * 10_000.0); + } + + double medianBps = 0.0; + if (perFillBps.Count > 0) + { + perFillBps.Sort(); + var mid = perFillBps.Count / 2; + medianBps = perFillBps.Count % 2 == 0 + ? (perFillBps[mid - 1] + perFillBps[mid]) / 2.0 + : perFillBps[mid]; + } + + var outliers = new List(); + foreach (var fill in fills) + { + if (fill.Notional <= 0m) + continue; + + var fillBps = (double)(fill.Commission / fill.Notional) * 10_000.0; + if (fillBps > medianBps * OutlierThresholdMultiplier + && fillBps > OutlierMinimumRateBps) + { + outliers.Add(new SessionTcaFillOutlier( + fill.OrderId, + fill.Symbol, + fill.Notional, + fill.Commission, + Math.Round(fillBps, 2), + fill.Timestamp)); + } + } + + // Sort descending by commission rate (worst outliers first). + outliers.Sort(static (a, b) => b.CommissionRateBps.CompareTo(a.CommissionRateBps)); + return outliers; + } + + private static SessionTcaExecutionQuality BuildExecutionQuality( + List fills, + IReadOnlyDictionary ordersById) + { + if (ordersById.Count == 0) + return SessionTcaExecutionQuality.Empty; + + var timesToFillSeconds = new List(); + decimal improvementWeightedBps = 0m; + decimal improvementNotional = 0m; + var limitOrderIds = new HashSet(StringComparer.Ordinal); + + foreach (var fill in fills) + { + if (!ordersById.TryGetValue(fill.OrderId, out var order)) + continue; + + var elapsed = (fill.Timestamp - order.CreatedAt).TotalSeconds; + if (elapsed >= 0) + timesToFillSeconds.Add(elapsed); + + if (order.LimitPrice is > 0m) + { + limitOrderIds.Add(order.OrderId); + var limit = order.LimitPrice.Value; + + // Positive = filled better than the limit (price improvement). + var improvementBps = fill.Side == OrderSide.Buy + ? (limit - fill.Price) / limit * 10_000m + : (fill.Price - limit) / limit * 10_000m; + + improvementWeightedBps += improvementBps * fill.Notional; + improvementNotional += fill.Notional; + } + } + + double medianTimeToFill = 0.0; + if (timesToFillSeconds.Count > 0) + { + timesToFillSeconds.Sort(); + var mid = timesToFillSeconds.Count / 2; + medianTimeToFill = timesToFillSeconds.Count % 2 == 0 + ? (timesToFillSeconds[mid - 1] + timesToFillSeconds[mid]) / 2.0 + : timesToFillSeconds[mid]; + } + + var avgImprovementBps = improvementNotional > 0m + ? (double)(improvementWeightedBps / improvementNotional) + : 0.0; + + return new SessionTcaExecutionQuality( + limitOrderIds.Count, + Math.Round(avgImprovementBps, 2), + Math.Round(medianTimeToFill, 3), + timesToFillSeconds.Count); + } +} + +/// TCA report over a paper/live session fill tape. +public sealed record SessionTcaReport( + string SessionId, + string StrategyId, + DateTimeOffset GeneratedAtUtc, + SessionTcaCostSummary CostSummary, + IReadOnlyList SymbolSummaries, + IReadOnlyList Outliers, + SessionTcaExecutionQuality ExecutionQuality); + +/// Aggregate session-level cost summary. +public sealed record SessionTcaCostSummary( + decimal TotalCommissions, + decimal TotalBuyNotional, + decimal TotalSellNotional, + decimal TotalNotional, + double CommissionRateBps, + int TotalFills, + int BuyFills, + int SellFills); + +/// Per-symbol cost breakdown, sorted by total commission descending. +public sealed record SessionSymbolTcaSummary( + string Symbol, + decimal TotalBuyNotional, + decimal TotalSellNotional, + decimal AvgBuyPrice, + decimal AvgSellPrice, + decimal TotalCommission, + double CommissionRateBps, + int TotalFills); + +/// A fill whose commission rate significantly exceeds the session median. +public sealed record SessionTcaFillOutlier( + string OrderId, + string Symbol, + decimal Notional, + decimal Commission, + double CommissionRateBps, + DateTimeOffset FilledAt); + +/// +/// Execution-quality statistics derived by joining fills to their originating orders. +/// is notional-weighted; positive values mean fills +/// executed better than their limit price. +/// +public sealed record SessionTcaExecutionQuality( + int OrdersWithLimitPrice, + double AvgLimitPriceImprovementBps, + double MedianTimeToFillSeconds, + int TimedFillCount) +{ + public static SessionTcaExecutionQuality Empty { get; } = new(0, 0.0, 0.0, 0); +} diff --git a/src/Meridian.FinancialOperations/Reconciliation/ReconciliationServiceRegistration.cs b/src/Meridian.FinancialOperations/Reconciliation/ReconciliationServiceRegistration.cs index a31e091dc3..c400e47c67 100644 --- a/src/Meridian.FinancialOperations/Reconciliation/ReconciliationServiceRegistration.cs +++ b/src/Meridian.FinancialOperations/Reconciliation/ReconciliationServiceRegistration.cs @@ -21,7 +21,7 @@ public static IServiceCollection AddStatementReconciliationServices(this IServic services.TryAddSingleton(sp => new JsonCanonicalStatementStore(sp.GetRequiredService().RootPath)); services.TryAddSingleton(sp => new JsonReconciliationCaseStore(sp.GetRequiredService().RootPath)); services.TryAddSingleton(sp => new JsonReconciliationBreakStore(sp.GetRequiredService().RootPath)); - services.TryAddSingleton(sp => new CsvBrokerStatementService(sp.GetRequiredService())); + AddBrokerStatementServices(services); return services; } @@ -38,10 +38,21 @@ public static IServiceCollection AddStatementReconciliationServices(this IServic services.TryAddSingleton(_ => new JsonCanonicalStatementStore(dataRoot)); services.TryAddSingleton(_ => new JsonReconciliationCaseStore(dataRoot)); services.TryAddSingleton(_ => new JsonReconciliationBreakStore(dataRoot)); - services.TryAddSingleton(sp => new CsvBrokerStatementService(sp.GetRequiredService())); + AddBrokerStatementServices(services); return services; } + private static void AddBrokerStatementServices(IServiceCollection services) + { + services.TryAddSingleton(sp => + new CsvBrokerStatementService(sp.GetRequiredService())); + services.TryAddSingleton(sp => + new IbFlexBrokerStatementService(sp.GetRequiredService())); + services.TryAddSingleton(sp => new RoutingBrokerStatementService( + sp.GetRequiredService(), + sp.GetRequiredService())); + } + private static void AddSharedServices(IServiceCollection services) { services.TryAddSingleton(); diff --git a/src/Meridian.FinancialOperations/Reconciliation/StatementMappingProfiles.cs b/src/Meridian.FinancialOperations/Reconciliation/StatementMappingProfiles.cs index 0f10786803..407c93ef07 100644 --- a/src/Meridian.FinancialOperations/Reconciliation/StatementMappingProfiles.cs +++ b/src/Meridian.FinancialOperations/Reconciliation/StatementMappingProfiles.cs @@ -54,6 +54,7 @@ public sealed class StatementMappingProfileRegistry { public const string CanonicalCsvV1ProfileId = "canonical-csv-v1"; public const string SampleBrokerCsvV1ProfileId = "sample-broker-csv-v1"; + public const string IbFlexV1ProfileId = "ib-flex-v1"; private readonly Dictionary _profiles; @@ -85,6 +86,7 @@ public StatementMappingProfile ResolveForSourceKind(string normalizedSourceKind, return Resolve(normalizedSourceKind switch { "sample-broker" => SampleBrokerCsvV1ProfileId, + "ib-flex" or "ibflex" or "ibkr" or "interactive-brokers" or "interactivebrokers" => IbFlexV1ProfileId, _ => CanonicalCsvV1ProfileId }); } @@ -149,6 +151,39 @@ private static IReadOnlyList CreateDefaultProfiles() => new("SELL", "trade"), new("FEE", "fee"), new("DIV", "dividend") + ]), + new StatementMappingProfile( + IbFlexV1ProfileId, + "Interactive Brokers Flex Query v1", + [ + // Source columns are Flex XML attribute names; the Flex importer emits the + // canonical activity types (trade/position/cash) directly per section. + new(StatementCanonicalField.Account, "accountId"), + new(StatementCanonicalField.SecurityIdentifier, "symbol"), + new(StatementCanonicalField.Quantity, "quantity"), + new(StatementCanonicalField.Price, "tradePrice"), + new(StatementCanonicalField.CashAmount, "netCash"), + new(StatementCanonicalField.ActivityType, "activityType"), + new(StatementCanonicalField.TradeDate, "tradeDate"), + new(StatementCanonicalField.SettlementDate, "settleDateTarget", Required: false), + new(StatementCanonicalField.Currency, "currency", Required: false), + new(StatementCanonicalField.FeesCommission, "ibCommission", Required: false), + new(StatementCanonicalField.ExternalTransactionId, "tradeID", Required: false), + new(StatementCanonicalField.MarketValue, "positionValue", Required: false), + new(StatementCanonicalField.Amount, "amount", Required: false) + ], + [ + new("BUY", "trade"), + new("SELL", "trade"), + new("trade", "trade"), + new("position", "position"), + new("cash", "cash"), + new("Dividends", "cash"), + new("Deposits/Withdrawals", "cash"), + new("Broker Interest Paid", "cash"), + new("Broker Interest Received", "cash"), + new("Withholding Tax", "cash"), + new("Other Fees", "fee") ]) ]; } diff --git a/src/Meridian.FinancialOperations/Reconciliation/StatementReconciliationService.cs b/src/Meridian.FinancialOperations/Reconciliation/StatementReconciliationService.cs index 08bf10aa4f..5cc5e6f4ee 100644 --- a/src/Meridian.FinancialOperations/Reconciliation/StatementReconciliationService.cs +++ b/src/Meridian.FinancialOperations/Reconciliation/StatementReconciliationService.cs @@ -22,7 +22,16 @@ public Task ValidateAsync(string sourceKind, string sourcePath, string? ct.ThrowIfCancellationRequested(); var normalizedSourceKind = ValidateSourceAccess(sourceKind, sourcePath); var profileId = mappingProfileId; - if (UsesCanonicalSchema(normalizedSourceKind, profileId)) + if (UsesFlexProcessing(normalizedSourceKind, sourcePath)) + { + // Flex reports are XML; the canonical CSV header check does not apply. Validate the + // document shape instead so the workflow rejects non-Flex files before import. + ValidateFlexDocument(sourcePath); + profileId = string.IsNullOrWhiteSpace(profileId) + ? StatementMappingProfileRegistry.IbFlexV1ProfileId + : profileId; + } + else if (UsesCanonicalSchema(normalizedSourceKind, profileId)) { var profile = ValidateStatementHeader(normalizedSourceKind, sourcePath, profileId); profileId = profile.ProfileId; @@ -39,6 +48,11 @@ public async Task ImportAsync(string sourceKind { var normalizedSourceKind = ValidateSourceAccess(sourceKind, sourcePath); ct.ThrowIfCancellationRequested(); + if (UsesFlexProcessing(normalizedSourceKind, sourcePath)) + { + return await ReadFlexStatementImportAsync(normalizedSourceKind, sourcePath, ct).ConfigureAwait(false); + } + if (UsesCanonicalSchema(normalizedSourceKind, mappingProfileId)) { return await ReadNormalizedStatementImportAsync(normalizedSourceKind, sourcePath, mappingProfileId, ct).ConfigureAwait(false); @@ -96,11 +110,17 @@ private static string ValidateSourceAccess(string sourceKind, string sourcePath) throw new ArgumentException("Statement source path is required.", nameof(sourcePath)); var normalizedSourceKind = sourceKind.Trim().ToLowerInvariant(); - if (!string.Equals(normalizedSourceKind, "local", StringComparison.Ordinal) + if (IsIbFlexSourceKind(normalizedSourceKind)) + { + normalizedSourceKind = IbFlexSourceKind; + } + else if (!string.Equals(normalizedSourceKind, "local", StringComparison.Ordinal) && !string.Equals(normalizedSourceKind, "broker", StringComparison.Ordinal) && !string.Equals(normalizedSourceKind, "custodian", StringComparison.Ordinal) && !string.Equals(normalizedSourceKind, "sample-broker", StringComparison.Ordinal)) - throw new NotSupportedException($"Statement source kind '{sourceKind}' is not supported. Use 'local', 'broker', 'custodian', or 'sample-broker'."); + { + throw new NotSupportedException($"Statement source kind '{sourceKind}' is not supported. Use 'local', 'broker', 'custodian', 'sample-broker', or 'ib-flex'."); + } if (!File.Exists(sourcePath)) throw new FileNotFoundException($"Statement source file '{sourcePath}' was not found.", sourcePath); @@ -108,6 +128,204 @@ private static string ValidateSourceAccess(string sourceKind, string sourcePath) return normalizedSourceKind; } + private const string IbFlexSourceKind = "ib-flex"; + + private static bool IsIbFlexSourceKind(string normalizedSourceKind) => + normalizedSourceKind is "ib-flex" or "ibflex" or "ibkr" or "interactive-brokers" or "interactivebrokers"; + + // Mirrors RoutingBrokerStatementService: an explicit Flex source kind always uses Flex + // processing, and a canonical broker/custodian kind with an .xml file routes to Flex too, + // so the workflow's validation stage agrees with where the import router will send the file. + private static bool UsesFlexProcessing(string normalizedSourceKind, string sourcePath) => + IsIbFlexSourceKind(normalizedSourceKind) + || (RequiresCanonicalStatementSchema(normalizedSourceKind) + && string.Equals(Path.GetExtension(sourcePath), ".xml", StringComparison.OrdinalIgnoreCase)); + + /// + /// Validates that is a well-formed IB Flex Query document + /// (root element FlexQueryResponse) without loading the full report. + /// + private static void ValidateFlexDocument(string sourcePath) + { + var settings = new System.Xml.XmlReaderSettings + { + DtdProcessing = System.Xml.DtdProcessing.Prohibit, + XmlResolver = null, + CloseInput = true + }; + + try + { + using var reader = System.Xml.XmlReader.Create(File.OpenRead(sourcePath), settings); + reader.MoveToContent(); + if (!string.Equals(reader.LocalName, "FlexQueryResponse", StringComparison.Ordinal)) + { + throw new InvalidDataException( + $"Statement source '{sourcePath}' is not an IB Flex Query report (root element '{reader.LocalName}')."); + } + } + catch (System.Xml.XmlException ex) + { + throw new InvalidDataException($"Statement source '{sourcePath}' is not well-formed XML: {ex.Message}", ex); + } + } + + /// + /// Reads an IB Flex report into normalized statement rows (one per Trade, OpenPosition, + /// and CashTransaction element) so case intake matches Flex statements with the same + /// engine as canonical CSV rows. Fee-like cash transactions keep fee row semantics via + /// . + /// + private static IReadOnlyList ReadFlexStatementRows( + string importId, + string normalizedSourceKind, + string sourcePath, + string content) + { + var document = System.Xml.Linq.XDocument.Parse(content); + var rows = new List(); + var rowNumber = 0; + + foreach (var statement in document.Descendants("FlexStatement")) + { + var statementToDate = Infrastructure.Reconciliation.IbFlexBrokerStatementService + .ParseFlexDate((string?)statement.Attribute("toDate")); + + foreach (var element in statement.Descendants() + .Where(static e => e.Name.LocalName is "Trade" or "OpenPosition" or "CashTransaction")) + { + rowNumber++; + var (activityType, quantity, amount, date) = element.Name.LocalName switch + { + "Trade" => ( + "trade", + FlexDecimal(element, "quantity"), + FlexFirstDecimal(element, "netCash", "proceeds") is var cash && cash != 0m + ? cash + : FlexDecimal(element, "quantity") * FlexDecimal(element, "tradePrice"), + FlexFirstDate(element, ["tradeDate"], statementToDate)), + "OpenPosition" => ( + "position", + FlexDecimal(element, "position"), + FlexFirstDecimal(element, "positionValue") is var value && value != 0m + ? value + : FlexDecimal(element, "position") * FlexFirstDecimal(element, "markPrice", "costBasisPrice"), + FlexFirstDate(element, ["reportDate"], statementToDate)), + _ => ( + Infrastructure.Reconciliation.IbFlexBrokerStatementService + .MapCashTransactionActivity((string?)element.Attribute("type")), + 0m, + FlexDecimal(element, "amount"), + FlexFirstDate(element, ["dateTime", "reportDate", "settleDate"], statementToDate)) + }; + + var snapshot = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["importId"] = importId, + ["sourceKind"] = normalizedSourceKind, + ["sourcePath"] = sourcePath, + ["elementName"] = element.Name.LocalName, + ["activityType"] = activityType, + ["rowNumber"] = rowNumber.ToString(CultureInfo.InvariantCulture) + }; + foreach (var attribute in element.Attributes()) + { + snapshot[attribute.Name.LocalName] = attribute.Value; + } + + var elementText = element.ToString(System.Xml.Linq.SaveOptions.DisableFormatting); + rows.Add(new NormalizedStatementRow( + $"{importId}:{rowNumber}", + ToStatementRowKind(activityType), + (string?)element.Attribute("symbol") ?? string.Empty, + quantity, + amount, + new DateTimeOffset(date.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero), + (string?)element.Attribute("currency") ?? "USD", + DeterministicFingerprint.Compute($"{importId}|{rowNumber}|{elementText}"), + snapshot)); + } + } + + return rows; + } + + private static decimal FlexDecimal(System.Xml.Linq.XElement element, string attribute) + { + var raw = (string?)element.Attribute(attribute); + return string.IsNullOrWhiteSpace(raw) + ? 0m + : decimal.Parse(raw, NumberStyles.Number, CultureInfo.InvariantCulture); + } + + private static decimal FlexFirstDecimal(System.Xml.Linq.XElement element, params string[] attributes) + { + foreach (var attribute in attributes) + { + var raw = (string?)element.Attribute(attribute); + if (!string.IsNullOrWhiteSpace(raw)) + return decimal.Parse(raw, NumberStyles.Number, CultureInfo.InvariantCulture); + } + + return 0m; + } + + private static DateOnly FlexFirstDate(System.Xml.Linq.XElement element, string[] attributes, DateOnly? fallback) + { + foreach (var attribute in attributes) + { + if (Infrastructure.Reconciliation.IbFlexBrokerStatementService + .ParseFlexDate((string?)element.Attribute(attribute)) is { } parsed) + return parsed; + } + + return fallback ?? throw new InvalidDataException( + "Flex row has no parseable date attribute and the statement has no toDate fallback."); + } + + /// + /// Reads an IB Flex report into source-row references (one per Trade, OpenPosition, and + /// CashTransaction element, with the element's attributes as the raw snapshot) so the + /// checkpointed ingestion stage reports real row counts. Canonical row construction for + /// the workflow path is owned by the Flex-aware broker statement importer. + /// + private static async Task ReadFlexStatementImportAsync( + string normalizedSourceKind, + string sourcePath, + CancellationToken ct) + { + var content = await File.ReadAllTextAsync(sourcePath, ct).ConfigureAwait(false); + var importId = DeterministicFingerprint.Compute($"{normalizedSourceKind}|{sourcePath}|{content}"); + + var document = System.Xml.Linq.XDocument.Parse(content); + var sourceRows = new List(); + var rowNumber = 0; + foreach (var element in document.Descendants() + .Where(static e => e.Name.LocalName is "Trade" or "OpenPosition" or "CashTransaction")) + { + ct.ThrowIfCancellationRequested(); + rowNumber++; + var snapshot = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["sourceKind"] = normalizedSourceKind, + ["sourcePath"] = sourcePath, + ["elementName"] = element.Name.LocalName + }; + foreach (var attribute in element.Attributes()) + { + snapshot[attribute.Name.LocalName] = attribute.Value; + } + + sourceRows.Add(CreateSourceRowReference( + importId, + rowNumber, + element.ToString(System.Xml.Linq.SaveOptions.DisableFormatting), + snapshot)); + } + + return new NormalizedStatementImportResult(importId, normalizedSourceKind, sourcePath, sourceRows.Count, [], [], [], [], sourceRows); + } + private static bool RequiresCanonicalStatementSchema(string normalizedSourceKind) => string.Equals(normalizedSourceKind, "broker", StringComparison.Ordinal) || string.Equals(normalizedSourceKind, "custodian", StringComparison.Ordinal) @@ -123,6 +341,24 @@ private static bool UsesCanonicalSchema(string normalizedSourceKind, string? map private ExternalStatementCaseIntakeResult CreateExternalStatementCases(string normalizedSourceKind, string sourcePath, string? mappingProfileId = null) { + // Flex XML never parses through the canonical CSV path, regardless of any mapping + // profile the caller selected; its rows are read from the XML sections and matched + // with the same engine so CLI/orchestrator intake reports real match/unresolved counts. + if (UsesFlexProcessing(normalizedSourceKind, sourcePath)) + { + var flexContent = File.ReadAllText(sourcePath); + var flexImportId = DeterministicFingerprint.Compute($"{normalizedSourceKind}|{sourcePath}|{flexContent}"); + var flexRows = ReadFlexStatementRows(flexImportId, normalizedSourceKind, sourcePath, flexContent); + var (flexMatches, flexCases) = MatchRows(flexRows); + return new ExternalStatementCaseIntakeResult( + flexImportId, + normalizedSourceKind, + sourcePath, + flexRows.Count, + flexMatches.Count, + flexCases); + } + if (!UsesCanonicalSchema(normalizedSourceKind, mappingProfileId)) { var content = File.ReadAllText(sourcePath); diff --git a/src/Meridian.Infrastructure/Reconciliation/IbFlexStatementService.cs b/src/Meridian.Infrastructure/Reconciliation/IbFlexStatementService.cs new file mode 100644 index 0000000000..464bdf3c46 --- /dev/null +++ b/src/Meridian.Infrastructure/Reconciliation/IbFlexStatementService.cs @@ -0,0 +1,328 @@ +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using System.Xml; +using System.Xml.Linq; +using Meridian.Domain.Reconciliation; + +namespace Meridian.Infrastructure.Reconciliation; + +/// +/// Imports Interactive Brokers Flex Query XML statements into canonical statement rows. +/// Supports the three Flex sections used by reconciliation: Trades/Trade (activity +/// type trade), OpenPositions/OpenPosition (position), and +/// CashTransactions/CashTransaction (cash). Follows the same duplicate-key, +/// checksum, and persistence flow as so downstream +/// matching and case intake treat both sources identically. +/// +public sealed class IbFlexBrokerStatementService(ICanonicalStatementStore store) : IBrokerStatementService +{ + private static readonly string[] SupportedBrokerAliases = + ["ibflex", "ib-flex", "ibkr", "interactivebrokers", "interactive-brokers"]; + + /// Returns whether names the IB Flex source. + public static bool IsIbFlexSource(string? broker) => + broker is not null + && SupportedBrokerAliases.Contains(broker.Trim(), StringComparer.OrdinalIgnoreCase); + + public async Task ValidateAsync( + BrokerStatementImportRequest request, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(request); + + var errors = new List(); + if (!File.Exists(request.SourcePath)) + { + errors.Add("Source file not found."); + return new BrokerStatementValidationResult(false, errors, 0); + } + + XDocument document; + try + { + document = await LoadDocumentAsync(request.SourcePath, ct).ConfigureAwait(false); + } + catch (XmlException ex) + { + errors.Add($"Source file is not well-formed XML: {ex.Message}"); + return new BrokerStatementValidationResult(false, errors, 0); + } + + if (!string.Equals(document.Root?.Name.LocalName, "FlexQueryResponse", StringComparison.Ordinal)) + { + errors.Add("Root element is not FlexQueryResponse; not an IB Flex Query report."); + return new BrokerStatementValidationResult(false, errors, 0); + } + + var statements = document.Root!.Descendants("FlexStatement").ToList(); + if (statements.Count == 0) + { + errors.Add("Flex report contains no FlexStatement elements."); + return new BrokerStatementValidationResult(false, errors, 0); + } + + var rowCount = statements.Sum(static statement => + statement.Descendants("Trade").Count() + + statement.Descendants("OpenPosition").Count() + + statement.Descendants("CashTransaction").Count()); + + if (rowCount == 0) + { + errors.Add("Flex report contains no Trade, OpenPosition, or CashTransaction rows; " + + "include those sections in the Flex Query definition."); + } + + return new BrokerStatementValidationResult(errors.Count == 0, errors, rowCount); + } + + public async Task ImportAsync( + BrokerStatementImportRequest request, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(request); + + var fileBytes = await File.ReadAllBytesAsync(request.SourcePath, ct).ConfigureAwait(false); + var sourceFileHash = string.IsNullOrWhiteSpace(request.SourceFileHash) + ? Convert.ToHexString(SHA256.HashData(fileBytes)) + : request.SourceFileHash.Trim().ToUpperInvariant(); + var duplicateKey = StatementDuplicateKey.Create( + request.FundAccountId, + request.StatementPeriodStart, + request.StatementPeriodEnd, + sourceFileHash); + + if (await store.ImportExistsByDuplicateKeyAsync(duplicateKey, ct).ConfigureAwait(false)) + throw new InvalidOperationException( + "Statement already imported (fund account, statement period, and source file hash match)."); + + var document = await LoadDocumentAsync(request.SourcePath, ct).ConfigureAwait(false); + if (!string.Equals(document.Root?.Name.LocalName, "FlexQueryResponse", StringComparison.Ordinal)) + throw new InvalidDataException("Root element is not FlexQueryResponse; not an IB Flex Query report."); + + var importId = duplicateKey; + var normalizedRequest = request.WithSourceFileHash(sourceFileHash); + var rows = ParseRows(document, importId).ToList(); + if (rows.Count == 0) + { + // A Flex query configured without the supported sections must fail loudly instead + // of being recorded as a clean zero-row run with no breaks or cases. + throw new InvalidDataException( + "Flex report contains no Trade, OpenPosition, or CashTransaction rows; " + + "include those sections in the Flex Query definition."); + } + + var import = new CanonicalStatementImport( + importId, + normalizedRequest.Broker, + normalizedRequest.StatementPeriodEnd, + DateTimeOffset.UtcNow, + normalizedRequest.SourcePath, + sourceFileHash, + rows.Count, + rows.Count) + { + SourceInstitution = normalizedRequest.SourceInstitution, + FundAccountId = normalizedRequest.FundAccountId, + ExternalAccountId = normalizedRequest.ExternalAccountId, + StatementPeriodStart = normalizedRequest.StatementPeriodStart, + StatementPeriodEnd = normalizedRequest.StatementPeriodEnd, + OriginalFileName = normalizedRequest.OriginalFileName, + MappingProfileId = normalizedRequest.MappingProfileId, + ToleranceProfileId = normalizedRequest.ToleranceProfileId, + ImportedBy = normalizedRequest.ImportedBy, + SourceFileHash = sourceFileHash, + DuplicateKey = duplicateKey + }; + + await store.SaveImportAsync(import, rows, ct).ConfigureAwait(false); + return new BrokerStatementImportResult(import, rows); + } + + private static async Task LoadDocumentAsync(string path, CancellationToken ct) + { + // DTD processing stays disabled: Flex reports never carry DTDs, and prohibiting them + // blocks XXE-style payloads in operator-supplied files. + var settings = new XmlReaderSettings + { + DtdProcessing = DtdProcessing.Prohibit, + XmlResolver = null, + Async = true, + CloseInput = true + }; + + using var reader = XmlReader.Create(File.OpenRead(path), settings); + return await XDocument.LoadAsync(reader, LoadOptions.None, ct).ConfigureAwait(false); + } + + private static IEnumerable ParseRows(XDocument document, string importId) + { + var rowNumber = 0; + foreach (var statement in document.Root!.Descendants("FlexStatement")) + { + var statementAccount = (string?)statement.Attribute("accountId") ?? string.Empty; + var statementToDate = ParseFlexDate((string?)statement.Attribute("toDate")); + + foreach (var trade in statement.Descendants("Trade")) + { + rowNumber++; + yield return new CanonicalStatementRow( + importId, + rowNumber, + Account(trade, statementAccount), + (string?)trade.Attribute("symbol") ?? string.Empty, + ParseDecimal(trade, "quantity"), + ParseDecimal(trade, "tradePrice"), + ParseFirstDecimal(trade, "netCash", "proceeds"), + "trade", + RequireDate(trade, "tradeDate", statementToDate, rowNumber), + HashElement(trade)); + } + + foreach (var position in statement.Descendants("OpenPosition")) + { + rowNumber++; + yield return new CanonicalStatementRow( + importId, + rowNumber, + Account(position, statementAccount), + (string?)position.Attribute("symbol") ?? string.Empty, + ParseDecimal(position, "position"), + ParseFirstDecimal(position, "markPrice", "costBasisPrice"), + 0m, + "position", + RequireDate(position, "reportDate", statementToDate, rowNumber), + HashElement(position)); + } + + foreach (var cash in statement.Descendants("CashTransaction")) + { + rowNumber++; + yield return new CanonicalStatementRow( + importId, + rowNumber, + Account(cash, statementAccount), + (string?)cash.Attribute("symbol") ?? string.Empty, + 0m, + 0m, + ParseDecimal(cash, "amount"), + MapCashTransactionActivity((string?)cash.Attribute("type")), + RequireFirstDate(cash, ["dateTime", "reportDate", "settleDate"], statementToDate, rowNumber), + HashElement(cash)); + } + } + } + + private static string Account(XElement element, string statementAccount) => + (string?)element.Attribute("accountId") is { Length: > 0 } account ? account : statementAccount; + + /// + /// Maps a Flex CashTransaction type to the canonical activity type so downstream + /// matching applies fee handling to fee-like rows ("Other Fees", "Advisor Fees", …) instead + /// of cash tolerance rules. All other cash transaction types (dividends, deposits, + /// withholding tax, interest) stay canonical cash. + /// + public static string MapCashTransactionActivity(string? flexType) => + flexType is not null && flexType.Contains("fee", StringComparison.OrdinalIgnoreCase) + ? "fee" + : "cash"; + + private static decimal ParseDecimal(XElement element, string attribute) + { + var raw = (string?)element.Attribute(attribute); + return string.IsNullOrWhiteSpace(raw) + ? 0m + : decimal.Parse(raw, NumberStyles.Number, CultureInfo.InvariantCulture); + } + + private static decimal ParseFirstDecimal(XElement element, params string[] attributes) + { + foreach (var attribute in attributes) + { + var raw = (string?)element.Attribute(attribute); + if (!string.IsNullOrWhiteSpace(raw)) + return decimal.Parse(raw, NumberStyles.Number, CultureInfo.InvariantCulture); + } + + return 0m; + } + + private static DateOnly RequireDate(XElement element, string attribute, DateOnly? fallback, int rowNumber) => + ParseFlexDate((string?)element.Attribute(attribute)) + ?? fallback + ?? throw new InvalidDataException( + $"Flex row {rowNumber} has no parseable '{attribute}' date and the statement has no toDate fallback."); + + private static DateOnly RequireFirstDate( + XElement element, + string[] attributes, + DateOnly? fallback, + int rowNumber) + { + foreach (var attribute in attributes) + { + if (ParseFlexDate((string?)element.Attribute(attribute)) is { } parsed) + return parsed; + } + + return fallback ?? throw new InvalidDataException( + $"Flex row {rowNumber} has no parseable date attribute and the statement has no toDate fallback."); + } + + /// + /// Parses the date formats Flex Queries emit depending on report configuration: + /// yyyyMMdd, yyyy-MM-dd, and datetime variants with a ;HHmmss or + /// HH:mm:ss suffix (only the date part is kept). + /// + public static DateOnly? ParseFlexDate(string? raw) + { + if (string.IsNullOrWhiteSpace(raw)) + return null; + + var datePart = raw.Split(';', ' ', 'T')[0]; + + if (DateOnly.TryParseExact(datePart, "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var compact)) + return compact; + if (DateOnly.TryParseExact(datePart, "yyyy-MM-dd", CultureInfo.InvariantCulture, DateTimeStyles.None, out var iso)) + return iso; + + return null; + } + + private static string HashElement(XElement element) => + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(element.ToString(SaveOptions.DisableFormatting)))); +} + +/// +/// Routes statement imports to the format-appropriate implementation: IB Flex XML sources go +/// to ; everything else keeps the existing CSV path. +/// Routing is by broker alias first, then by an .xml source extension so operators who +/// pick a generic broker kind with a Flex file still land on the XML parser. +/// +public sealed class RoutingBrokerStatementService( + CsvBrokerStatementService csvService, + IbFlexBrokerStatementService ibFlexService) : IBrokerStatementService +{ + public Task ValidateAsync( + BrokerStatementImportRequest request, + CancellationToken ct = default) + => Resolve(request).ValidateAsync(request, ct); + + public Task ImportAsync( + BrokerStatementImportRequest request, + CancellationToken ct = default) + => Resolve(request).ImportAsync(request, ct); + + private IBrokerStatementService Resolve(BrokerStatementImportRequest request) + { + ArgumentNullException.ThrowIfNull(request); + + if (IbFlexBrokerStatementService.IsIbFlexSource(request.Broker)) + return ibFlexService; + + if (string.Equals(Path.GetExtension(request.SourcePath), ".xml", StringComparison.OrdinalIgnoreCase)) + return ibFlexService; + + return csvService; + } +} diff --git a/src/Meridian.Storage/Meridian.Storage.csproj b/src/Meridian.Storage/Meridian.Storage.csproj index 1647a5b0d7..7702b55104 100644 --- a/src/Meridian.Storage/Meridian.Storage.csproj +++ b/src/Meridian.Storage/Meridian.Storage.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Meridian.Storage/Query/DuckDbQueryService.cs b/src/Meridian.Storage/Query/DuckDbQueryService.cs new file mode 100644 index 0000000000..fc404fe36e --- /dev/null +++ b/src/Meridian.Storage/Query/DuckDbQueryService.cs @@ -0,0 +1,281 @@ +using System.Diagnostics; +using System.Globalization; +using System.Text.RegularExpressions; +using DuckDB.NET.Data; +using Meridian.Storage.Interfaces; +using Microsoft.Extensions.Logging; + +namespace Meridian.Storage.Query; + +/// +/// Read-only SQL analytics over the local market-data store, backed by an in-memory DuckDB +/// session per query. The catalog is exposed as a meridian_files view (symbol, event +/// type, date, format, absolute path, size, event count) so operators can discover files and +/// feed paths into read_parquet/read_json_auto without leaving SQL. Statements +/// are guarded: single SELECT-family statement, keyword blocklist, row cap, and timeout — +/// a runaway query can never touch the ingestion hot path or mutate the store. File access +/// is sandboxed to the storage root via DuckDB's allowed_directories with external +/// access otherwise disabled and the configuration locked, so reader functions cannot reach +/// arbitrary host files. +/// +public sealed partial class DuckDbQueryService( + IStorageCatalogService catalogService, + StorageOptions storageOptions, + ILogger? logger = null) +{ + /// + /// Executes and returns a bounded, string-projected result set. + /// Guard violations and execution errors are returned as a failed result, not thrown, + /// so the endpoint can map them to a 400 with the message intact. + /// + public async Task ExecuteAsync( + string? sql, + DataQueryOptions options, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(options); + + var guardError = SqlStatementGuard.Validate(sql, options.MaxSqlLength); + if (guardError is not null) + return DataQueryResult.Failed(guardError); + + using var timeoutSource = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeoutSource.CancelAfter(TimeSpan.FromSeconds(options.TimeoutSeconds)); + + var stopwatch = Stopwatch.StartNew(); + try + { + using var connection = new DuckDBConnection("DataSource=:memory:"); + await connection.OpenAsync(timeoutSource.Token).ConfigureAwait(false); + + await ApplySessionLimitsAsync(connection, options, timeoutSource.Token).ConfigureAwait(false); + await CreateCatalogViewAsync(connection, timeoutSource.Token).ConfigureAwait(false); + + using var command = connection.CreateCommand(); + // Executing operator-supplied SQL is this workbench's purpose, so parameterization + // does not apply (CodeQL cs/sql-injection fires here by design). Containment instead + // of sanitization: SqlStatementGuard admits a single SELECT-family statement with + // write/config verbs blocked; the session is in-memory, capped (memory/threads/rows/ + // timeout), filesystem-sandboxed to the storage root via allowed_directories with + // external access otherwise disabled, and its configuration is locked; the endpoint + // is rate-limited. + command.CommandText = sql!; + + using var reader = await command.ExecuteReaderAsync(timeoutSource.Token).ConfigureAwait(false); + + var columns = new string[reader.FieldCount]; + var columnTypes = new string[reader.FieldCount]; + for (var i = 0; i < reader.FieldCount; i++) + { + columns[i] = reader.GetName(i); + columnTypes[i] = reader.GetDataTypeName(i); + } + + var rows = new List(); + var truncated = false; + while (await reader.ReadAsync(timeoutSource.Token).ConfigureAwait(false)) + { + if (rows.Count >= options.MaxRows) + { + truncated = true; + break; + } + + var row = new string?[reader.FieldCount]; + for (var i = 0; i < reader.FieldCount; i++) + { + row[i] = reader.IsDBNull(i) + ? null + : Convert.ToString(reader.GetValue(i), CultureInfo.InvariantCulture); + } + rows.Add(row); + } + + stopwatch.Stop(); + return new DataQueryResult( + Success: true, + Error: null, + Columns: columns, + ColumnTypes: columnTypes, + Rows: rows, + RowCount: rows.Count, + Truncated: truncated, + ElapsedMs: stopwatch.ElapsedMilliseconds); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return DataQueryResult.Failed( + $"Query exceeded the {options.TimeoutSeconds}s time limit and was cancelled."); + } + catch (Exception ex) when (ex is DuckDBException or InvalidOperationException or FormatException) + { + logger?.LogDebug(ex, "Data query failed"); + return DataQueryResult.Failed(ex.Message); + } + } + + private async Task ApplySessionLimitsAsync( + DuckDBConnection connection, + DataQueryOptions options, + CancellationToken ct) + { + using var command = connection.CreateCommand(); + // Session-scoped resource limits and the filesystem sandbox; these SET statements are + // issued by the service itself before the (blocklist-guarded) user statement runs. + // Reader functions (read_parquet/read_csv_auto/...) may only touch the storage root: + // allowed_directories confines file access, enable_external_access=false blocks + // everything else (other paths, remote endpoints, extension loading), and + // lock_configuration=true makes the sandbox immutable for the rest of the session. + var storageRoot = Path.GetFullPath(storageOptions.RootPath).Replace("'", "''", StringComparison.Ordinal); + command.CommandText = + $"SET memory_limit='{options.MemoryLimitMb}MB'; " + + $"SET threads={options.MaxThreads}; " + + $"SET allowed_directories=['{storageRoot}']; " + + "SET enable_external_access=false; " + + "SET lock_configuration=true;"; + await command.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + } + + private const int CatalogInsertBatchSize = 500; + + private async Task CreateCatalogViewAsync(DuckDBConnection connection, CancellationToken ct) + { + using var create = connection.CreateCommand(); + create.CommandText = + "CREATE TABLE meridian_files (symbol VARCHAR, event_type VARCHAR, source VARCHAR, " + + "date DATE, format VARCHAR, compressed BOOLEAN, path VARCHAR, size_bytes BIGINT, event_count BIGINT)"; + await create.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + + var batch = new List(CatalogInsertBatchSize); + foreach (var file in catalogService.SearchFiles(new CatalogSearchCriteria())) + { + var absolutePath = Path.GetFullPath(Path.Combine(storageOptions.RootPath, file.RelativePath)); + batch.Add( + $"({Quote(file.Symbol)}, {Quote(file.EventType)}, {Quote(file.Source)}, " + + $"{Quote(file.Date?.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture))}, " + + $"{Quote(file.Format)}, {(file.IsCompressed ? "true" : "false")}, {Quote(absolutePath)}, " + + $"{file.SizeBytes.ToString(CultureInfo.InvariantCulture)}, {file.EventCount.ToString(CultureInfo.InvariantCulture)})"); + + if (batch.Count >= CatalogInsertBatchSize) + { + await InsertBatchAsync(connection, batch, ct).ConfigureAwait(false); + batch.Clear(); + } + } + + if (batch.Count > 0) + await InsertBatchAsync(connection, batch, ct).ConfigureAwait(false); + } + + private static async Task InsertBatchAsync( + DuckDBConnection connection, + IReadOnlyList valueTuples, + CancellationToken ct) + { + using var insert = connection.CreateCommand(); + insert.CommandText = "INSERT INTO meridian_files VALUES " + string.Join(", ", valueTuples); + await insert.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + } + + private static string Quote(string? value) => + value is null ? "NULL" : "'" + value.Replace("'", "''", StringComparison.Ordinal) + "'"; +} + +/// +/// Validates that a statement is a single, read-only SELECT-family query. The first token +/// must be a query verb and no write/config verb may appear anywhere — DuckDB's in-memory +/// session cannot mutate the store through queries, but COPY/EXPORT/ATTACH/INSTALL can touch +/// the filesystem or network, so they are rejected outright. +/// +public static partial class SqlStatementGuard +{ + private static readonly string[] AllowedFirstTokens = + ["SELECT", "WITH", "FROM", "DESCRIBE", "SHOW", "EXPLAIN", "SUMMARIZE", "VALUES"]; + + [GeneratedRegex( + @"\b(COPY|EXPORT|IMPORT|INSTALL|LOAD|ATTACH|DETACH|CREATE|INSERT|UPDATE|DELETE|DROP|ALTER|TRUNCATE|PRAGMA|SET|RESET|CALL|BEGIN|COMMIT|ROLLBACK|CHECKPOINT|VACUUM|GRANT|USE)\b", + RegexOptions.IgnoreCase)] + private static partial Regex BlockedKeywords(); + + [GeneratedRegex(@"^(\s*(--[^\n]*\n|/\*.*?\*/))*\s*", RegexOptions.Singleline)] + private static partial Regex LeadingCommentsAndWhitespace(); + + /// Returns an error message, or when the statement is allowed. + public static string? Validate(string? sql, int maxSqlLength) + { + if (string.IsNullOrWhiteSpace(sql)) + return "A SQL statement is required."; + + if (sql.Length > maxSqlLength) + return $"Statement exceeds the {maxSqlLength}-character limit."; + + var body = LeadingCommentsAndWhitespace().Replace(sql, string.Empty, 1).TrimEnd(); + if (body.Length == 0) + return "A SQL statement is required."; + + // Single statement only: a trailing semicolon is fine, embedded ones are not. + var withoutTrailing = body.TrimEnd(';', ' ', '\t', '\r', '\n'); + if (withoutTrailing.Contains(';', StringComparison.Ordinal)) + return "Only a single statement is allowed."; + + var firstToken = FirstToken(withoutTrailing); + if (!AllowedFirstTokens.Contains(firstToken, StringComparer.OrdinalIgnoreCase)) + return $"Only read-only queries are allowed ({string.Join(", ", AllowedFirstTokens)}); got '{firstToken}'."; + + var blocked = BlockedKeywords().Match(withoutTrailing); + if (blocked.Success) + return $"Statement contains the blocked keyword '{blocked.Value.ToUpperInvariant()}'."; + + return null; + } + + private static string FirstToken(string body) + { + var end = 0; + while (end < body.Length && (char.IsLetter(body[end]) || body[end] == '_')) + end++; + return end == 0 ? body[..Math.Min(body.Length, 12)] : body[..end]; + } +} + +/// +/// Limits for the data query workbench, bound from the DataQuery configuration section. +/// +public sealed class DataQueryOptions +{ + public const string SectionName = "DataQuery"; + + public bool Enabled { get; set; } = true; + + /// Maximum rows returned to the client; additional rows set Truncated. + public int MaxRows { get; set; } = 10_000; + + /// Wall-clock limit for a single query. + public int TimeoutSeconds { get; set; } = 30; + + /// DuckDB session memory limit. + public int MemoryLimitMb { get; set; } = 512; + + /// DuckDB session thread cap, keeping analytics off the ingestion cores. + public int MaxThreads { get; set; } = 2; + + public int MaxSqlLength { get; set; } = 20_000; +} + +/// Request body for the data query endpoint. +public sealed record DataQueryRequest(string? Sql); + +/// Bounded, string-projected query result. +public sealed record DataQueryResult( + bool Success, + string? Error, + IReadOnlyList Columns, + IReadOnlyList ColumnTypes, + IReadOnlyList Rows, + int RowCount, + bool Truncated, + long ElapsedMs) +{ + public static DataQueryResult Failed(string error) => new( + false, error, Array.Empty(), Array.Empty(), Array.Empty(), 0, false, 0); +} diff --git a/src/Meridian.Storage/Services/DataReplacementCostEstimator.cs b/src/Meridian.Storage/Services/DataReplacementCostEstimator.cs new file mode 100644 index 0000000000..7ef2e9ee86 --- /dev/null +++ b/src/Meridian.Storage/Services/DataReplacementCostEstimator.cs @@ -0,0 +1,206 @@ +using Meridian.Contracts.Catalog; + +namespace Meridian.Storage.Services; + +/// +/// Estimates what the locally collected market-data store would cost to repurchase from +/// commercial metered vendors, priced against a configurable rate card. Estimates are +/// deliberately conservative: two independent bases are computed (per symbol-day and per +/// gigabyte) and the lower one is reported as the headline figure. This is a pure +/// post-processing pass over the ; it never touches the data files. +/// +public static class DataReplacementCostEstimator +{ + /// + /// Computes a replacement-cost estimate for using + /// . Returns a zero-valued estimate when the catalog is empty + /// or the meter is disabled. + /// + public static DataReplacementCostEstimate Estimate(StorageCatalog catalog, DataReplacementCostOptions options) + { + ArgumentNullException.ThrowIfNull(catalog); + ArgumentNullException.ThrowIfNull(options); + + if (!options.Enabled || catalog.Symbols.Count == 0) + { + return DataReplacementCostEstimate.Empty(options); + } + + // ── Symbol-day basis ─────────────────────────────────────────────── + // Each (symbol, event type) pair is priced per covered trading day, matching how + // metered vendors price historical downloads. Trading days are preferred; calendar + // days are the fallback when the catalog has not computed them. + var byEventType = new Dictionary(StringComparer.OrdinalIgnoreCase); + long totalSymbolDays = 0; + decimal symbolDayBasisUsd = 0m; + + foreach (var entry in catalog.Symbols.Values) + { + var days = ResolveCoveredDays(entry); + if (days <= 0) + continue; + + var eventTypes = entry.EventTypes is { Length: > 0 } + ? entry.EventTypes + : UnclassifiedEventTypes; + + foreach (var eventType in eventTypes) + { + var rate = ResolveRate(options, eventType); + var cost = days * rate; + + symbolDayBasisUsd += cost; + totalSymbolDays += days; + + byEventType.TryGetValue(eventType, out var bucket); + byEventType[eventType] = (bucket.SymbolDays + days, bucket.Usd + cost); + } + } + + // ── Volume basis ─────────────────────────────────────────────────── + // Uncompressed size is what a vendor would meter; fall back to on-disk size when the + // raw figure is unavailable so the estimate stays conservative. + var meteredBytes = catalog.Statistics.TotalBytesRaw > 0 + ? catalog.Statistics.TotalBytesRaw + : catalog.Statistics.TotalBytesCompressed; + var totalGigabytes = meteredBytes / 1_000_000_000m; + var volumeBasisUsd = totalGigabytes * options.RatePerGigabyteUsd; + + // ── Conservative headline ────────────────────────────────────────── + var positiveBases = new List(2); + if (symbolDayBasisUsd > 0m) + positiveBases.Add(symbolDayBasisUsd); + if (volumeBasisUsd > 0m) + positiveBases.Add(volumeBasisUsd); + var conservative = positiveBases.Count > 0 ? positiveBases.Min() : 0m; + + var breakdown = byEventType + .Select(static kvp => new DataReplacementCostByEventType( + kvp.Key, + kvp.Value.SymbolDays, + Math.Round(kvp.Value.Usd, 2))) + .OrderByDescending(static row => row.EstimatedUsd) + .ToArray(); + + return new DataReplacementCostEstimate( + GeneratedAtUtc: DateTimeOffset.UtcNow, + Currency: options.Currency, + ConservativeEstimateUsd: Math.Round(conservative, 2), + SymbolDayBasisUsd: Math.Round(symbolDayBasisUsd, 2), + VolumeBasisUsd: Math.Round(volumeBasisUsd, 2), + TotalSymbols: catalog.Symbols.Count, + TotalSymbolDays: totalSymbolDays, + TotalGigabytes: Math.Round(totalGigabytes, 2), + EventTypeBreakdown: breakdown, + RateCardSource: options.RateCardSource, + RateCardLastVerified: options.RateCardLastVerified, + MethodologyNote: MethodologyNote); + } + + private const string MethodologyNote = + "Lower bound of two independent bases: per-symbol-day rates applied to covered trading days " + + "per event type, and a per-gigabyte rate applied to uncompressed stored volume. Rates are " + + "conservative low-end list prices from the configured rate card, not a vendor quote."; + + private static readonly string[] UnclassifiedEventTypes = ["Unclassified"]; + + private static int ResolveCoveredDays(SymbolCatalogEntry entry) + { + if (entry.DateRange is { } range) + { + if (range.TradingDays > 0) + return range.TradingDays; + if (range.CalendarDays > 0) + return range.CalendarDays; + } + + // A symbol with recorded events but no computed range still covers at least one day. + return entry.EventCount > 0 ? 1 : 0; + } + + private static decimal ResolveRate(DataReplacementCostOptions options, string eventType) + { + if (options.HistoricalRatePerSymbolDayUsd is { } rates + && rates.TryGetValue(eventType, out var rate) + && rate >= 0m) + { + return rate; + } + + return options.DefaultRatePerSymbolDayUsd; + } +} + +/// +/// Rate card and toggles for the data replacement-cost meter, bound from the +/// DataReplacementCost configuration section. Defaults are conservative low-end +/// list prices as of the recorded verification date; operators should refresh them from +/// current vendor rate cards rather than treating them as quotes. +/// +public sealed class DataReplacementCostOptions +{ + public const string SectionName = "DataReplacementCost"; + + public bool Enabled { get; set; } = true; + + public string Currency { get; set; } = "USD"; + + /// Historical per-symbol-day rates keyed by event type (e.g. Trade, Quote, Depth). + public Dictionary? HistoricalRatePerSymbolDayUsd { get; set; } = + new(StringComparer.OrdinalIgnoreCase) + { + ["Trade"] = 0.10m, + ["Quote"] = 0.10m, + ["Depth"] = 0.25m, + ["OrderBook"] = 0.25m, + }; + + /// Fallback rate applied to event types not present in the rate card. + public decimal DefaultRatePerSymbolDayUsd { get; set; } = 0.10m; + + /// Metered per-gigabyte rate applied to uncompressed stored volume. + public decimal RatePerGigabyteUsd { get; set; } = 5.00m; + + /// Provenance note for the configured rates. + public string RateCardSource { get; set; } = + "Conservative low-end of published metered vendor list prices (per-symbol-day historical and per-GB tick data)."; + + /// Date the rate card values were last checked against vendor pricing pages. + public string RateCardLastVerified { get; set; } = "2026-07-01"; +} + +/// Replacement-cost estimate for the local market-data store. +public sealed record DataReplacementCostEstimate( + DateTimeOffset GeneratedAtUtc, + string Currency, + decimal ConservativeEstimateUsd, + decimal SymbolDayBasisUsd, + decimal VolumeBasisUsd, + int TotalSymbols, + long TotalSymbolDays, + decimal TotalGigabytes, + IReadOnlyList EventTypeBreakdown, + string RateCardSource, + string RateCardLastVerified, + string MethodologyNote) +{ + public static DataReplacementCostEstimate Empty(DataReplacementCostOptions options) => new( + DateTimeOffset.UtcNow, + options.Currency, + 0m, + 0m, + 0m, + 0, + 0, + 0m, + Array.Empty(), + options.RateCardSource, + options.RateCardLastVerified, + "No catalogued data or the meter is disabled."); +} + +/// Per-event-type slice of the symbol-day cost basis. +public sealed record DataReplacementCostByEventType( + string EventType, + long SymbolDays, + decimal EstimatedUsd); diff --git a/src/Meridian.Ui.Shared/Endpoints/ExecutionEndpoints.cs b/src/Meridian.Ui.Shared/Endpoints/ExecutionEndpoints.cs index f94b7581dd..153e3fa3d1 100644 --- a/src/Meridian.Ui.Shared/Endpoints/ExecutionEndpoints.cs +++ b/src/Meridian.Ui.Shared/Endpoints/ExecutionEndpoints.cs @@ -581,6 +581,28 @@ public static void MapExecutionEndpoints(this WebApplication app, JsonSerializer .Produces(200) .Produces(404); + group.MapGet("/sessions/{sessionId}/tca", async (string sessionId, HttpContext context) => + { + var persistence = context.RequestServices.GetService(); + if (persistence is null) + return Results.NotFound(); + + await persistence.InitialiseAsync(context.RequestAborted).ConfigureAwait(false); + var session = persistence.GetSession(sessionId); + if (session is null) + return Results.NotFound(); + + var report = SessionTcaReporter.Generate( + sessionId, + session.Summary.StrategyId, + session.FillHistory ?? Array.Empty(), + session.OrderHistory); + return Results.Json(report, jsonOptions); + }) + .WithName("GetExecutionSessionTcaReport") + .Produces(200) + .Produces(404); + group.MapPost("/sessions/create", async (CreatePaperSessionRequest request, HttpContext context) => { if (!HasExecutionTradingPermission(context, UserPermission.ExecuteTrades)) diff --git a/src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs b/src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs index 35b735904d..4c7c270cb0 100644 --- a/src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs +++ b/src/Meridian.Ui.Shared/Endpoints/WorkstationEndpoints.cs @@ -23,6 +23,9 @@ using Meridian.Instruments.AssetOperations; using Meridian.QuantScript.Compilation; using Meridian.Storage.Export; +using Meridian.Storage.Interfaces; +using Meridian.Storage.Query; +using Meridian.Storage.Services; using Meridian.Strategies.Interfaces; using Meridian.Strategies.Models; using Meridian.Strategies.Promotions; @@ -36,6 +39,7 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using ContractSecurityMasterQueryService = Meridian.Contracts.SecurityMaster.ISecurityMasterQueryService; namespace Meridian.Ui.Shared.Endpoints; @@ -385,6 +389,50 @@ public static void MapWorkstationEndpoints(this WebApplication app, JsonSerializ }) .WithName("GetWorkstationData"); + group.MapPost(WorkstationSubroute(UiApiRoutes.WorkstationDataQuery), async (DataQueryRequest request, HttpContext context) => + { + var queryService = context.RequestServices.GetService(); + if (queryService is null) + { + return Results.Problem( + "The data query service is not available.", + statusCode: StatusCodes.Status503ServiceUnavailable); + } + + var options = context.RequestServices + .GetService>()?.CurrentValue + ?? new DataQueryOptions(); + if (!options.Enabled) + { + return Results.Problem( + "The data query workbench is disabled.", + statusCode: StatusCodes.Status503ServiceUnavailable); + } + + // Guard violations and SQL errors are part of the result payload (Success=false) + // so the workbench can render them inline without parsing problem responses. + var result = await queryService.ExecuteAsync(request.Sql, options, context.RequestAborted) + .ConfigureAwait(false); + return Results.Ok(result); + }) + .WithName("PostWorkstationDataQuery") + .Produces(200) + .Produces(503) + .RequireRateLimiting(UiEndpoints.MutationRateLimitPolicy); + + group.MapGet(WorkstationSubroute(UiApiRoutes.WorkstationDataReplacementCost), (HttpContext context) => + { + var estimate = TryBuildDataReplacementCostEstimate(context); + return estimate is null + ? Results.Problem( + "Storage catalog is not available.", + statusCode: StatusCodes.Status503ServiceUnavailable) + : Results.Ok(estimate); + }) + .WithName("GetWorkstationDataReplacementCost") + .Produces(200) + .Produces(503); + group.MapGet(WorkstationSubroute(UiApiRoutes.WorkstationGovernance), async (HttpContext context) => { return await BuildAccountingPayloadAsync(context).ConfigureAwait(false); @@ -4119,15 +4167,28 @@ private static async Task BuildDataPayloadAsync(HttpCont backfills = []; } + var metrics = new List + { + new("providers-healthy", "Providers Healthy", healthyProviderCount.ToString(CultureInfo.InvariantCulture), "0", healthyProviderCount > 0 ? "success" : "default"), + new("backfills-running", "Backfills Running", activeRuns.ToString(CultureInfo.InvariantCulture), activeRuns == 0 ? "0" : $"+{activeRuns}", activeRuns > 0 ? "default" : "success"), + new("exports-ready", "Exports Ready", "0", "0", "default"), + new("ops-review", "Needs Review", reviewRuns.ToString(CultureInfo.InvariantCulture), reviewRuns == 0 ? "0" : $"+{reviewRuns}", reviewRuns == 0 ? "default" : "warning"), + new("kernel-critical-jumps", "Kernel Jump Alerts", GetKernelActiveAlertCount(kernelObservability).ToString(CultureInfo.InvariantCulture), FormatKernelJumpAlertDelta(kernelObservability), GetKernelJumpAlertTone(kernelObservability)) + }; + + var replacementCost = TryBuildDataReplacementCostEstimate(context); + if (replacementCost is { ConservativeEstimateUsd: > 0m }) + { + metrics.Add(new WorkstationMetricCard( + "data-replacement-cost", + "Est. Replacement Cost", + FormatCompactUsd(replacementCost.ConservativeEstimateUsd), + $"{replacementCost.TotalGigabytes.ToString("0.#", CultureInfo.InvariantCulture)} GB local", + "success")); + } + return new WorkstationDataPayload( - Metrics: - [ - new WorkstationMetricCard("providers-healthy", "Providers Healthy", healthyProviderCount.ToString(CultureInfo.InvariantCulture), "0", healthyProviderCount > 0 ? "success" : "default"), - new WorkstationMetricCard("backfills-running", "Backfills Running", activeRuns.ToString(CultureInfo.InvariantCulture), activeRuns == 0 ? "0" : $"+{activeRuns}", activeRuns > 0 ? "default" : "success"), - new WorkstationMetricCard("exports-ready", "Exports Ready", "0", "0", "default"), - new WorkstationMetricCard("ops-review", "Needs Review", reviewRuns.ToString(CultureInfo.InvariantCulture), reviewRuns == 0 ? "0" : $"+{reviewRuns}", reviewRuns == 0 ? "default" : "warning"), - new WorkstationMetricCard("kernel-critical-jumps", "Kernel Jump Alerts", GetKernelActiveAlertCount(kernelObservability).ToString(CultureInfo.InvariantCulture), FormatKernelJumpAlertDelta(kernelObservability), GetKernelJumpAlertTone(kernelObservability)) - ], + Metrics: metrics, Providers: providers, Backfills: backfills, Exports: [], @@ -4135,6 +4196,32 @@ private static async Task BuildDataPayloadAsync(HttpCont KernelObservability: BuildKernelObservabilityPayload(kernelObservability)); } + /// + /// Builds the replacement-cost estimate from the storage catalog, or returns + /// when the catalog service is unavailable or the meter is disabled. + /// + private static DataReplacementCostEstimate? TryBuildDataReplacementCostEstimate(HttpContext context) + { + var catalogService = context.RequestServices.GetService(); + if (catalogService is null) + return null; + + var options = context.RequestServices + .GetService>()?.CurrentValue + ?? new DataReplacementCostOptions(); + if (!options.Enabled) + return null; + + return DataReplacementCostEstimator.Estimate(catalogService.GetCatalog(), options); + } + + private static string FormatCompactUsd(decimal usd) => usd switch + { + >= 1_000_000m => "$" + (usd / 1_000_000m).ToString("0.0", CultureInfo.InvariantCulture) + "M", + >= 1_000m => "$" + (usd / 1_000m).ToString("0.0", CultureInfo.InvariantCulture) + "k", + _ => "$" + usd.ToString("0", CultureInfo.InvariantCulture) + }; + private static bool MatchesLedgerDimensionFilter( LedgerDimensionSetDto? dimensions, string? fundId, diff --git a/src/Meridian.Ui.Shared/Services/WorkstationServiceCollectionExtensions.cs b/src/Meridian.Ui.Shared/Services/WorkstationServiceCollectionExtensions.cs index e02fbd8789..d0d88c1aba 100644 --- a/src/Meridian.Ui.Shared/Services/WorkstationServiceCollectionExtensions.cs +++ b/src/Meridian.Ui.Shared/Services/WorkstationServiceCollectionExtensions.cs @@ -80,6 +80,14 @@ public static IServiceCollection AddWorkstationSharedServices(this IServiceColle services.AddHttpClient(); services.AddHttpContextAccessor(); services.AddMemoryCache(); + + // Options consumed by workstation endpoints must bind in the shared registration so + // every host of these endpoints (UiServer and AddUiSharedServices alike) honors + // operator configuration rather than compiled defaults. + services.AddOptions() + .BindConfiguration(Meridian.Storage.Services.DataReplacementCostOptions.SectionName); + services.AddOptions() + .BindConfiguration(Meridian.Storage.Query.DataQueryOptions.SectionName); services.TryAddScoped(); // SEC-005 slice 4c-ii: ambient caller-tenant accessor consumed by the singleton Postgres ledger // store for tenant read predicates. Singleton + IHttpContextAccessor-backed (no captive scope). diff --git a/src/Meridian.Ui/dashboard/src/lib/api.ts b/src/Meridian.Ui/dashboard/src/lib/api.ts index 3681b8a8ef..cc7d5e5a30 100644 --- a/src/Meridian.Ui/dashboard/src/lib/api.ts +++ b/src/Meridian.Ui/dashboard/src/lib/api.ts @@ -50,6 +50,8 @@ import type { ExternalGlMappingProfile, DataFetchRequest, DataFetchResult, + DataQueryRequest, + DataQueryResult, DataUploadPreviewResult, DataUploadTemplateCatalog, DataWorkspaceResponse, @@ -1843,6 +1845,10 @@ export function getDataWorkspace(options: ApiRequestOptions = {}) { return getJson(WORKSTATION_API_ENDPOINTS.data, options); } +export function runDataQuery(request: DataQueryRequest, options: ApiRequestOptions = {}) { + return postJson(WORKSTATION_API_ENDPOINTS.dataQuery, request, options); +} + export function getDataUploadTemplates(options: ApiRequestOptions = {}) { return getJson(WORKSTATION_API_ENDPOINTS.dataUploadTemplates, options); } diff --git a/src/Meridian.Ui/dashboard/src/lib/ui-api-routes.generated.ts b/src/Meridian.Ui/dashboard/src/lib/ui-api-routes.generated.ts index 9d280d318d..092842399d 100644 --- a/src/Meridian.Ui/dashboard/src/lib/ui-api-routes.generated.ts +++ b/src/Meridian.Ui/dashboard/src/lib/ui-api-routes.generated.ts @@ -488,6 +488,8 @@ export const UI_API_ROUTES = { WorkstationTrading: "/api/workstation/trading", WorkstationDataOperations: "/api/workstation/data-operations", WorkstationData: "/api/workstation/data", + WorkstationDataQuery: "/api/workstation/data/query", + WorkstationDataReplacementCost: "/api/workstation/data/replacement-cost", WorkstationDataUploadTemplates: "/api/workstation/data/uploads/templates", WorkstationDataUploadPreview: "/api/workstation/data/uploads/preview", WorkstationBankStatementImport: "/api/workstation/data/uploads/bank-statements/import", diff --git a/src/Meridian.Ui/dashboard/src/lib/workstation-endpoints.ts b/src/Meridian.Ui/dashboard/src/lib/workstation-endpoints.ts index ad37b9c7b0..1ee84c8134 100644 --- a/src/Meridian.Ui/dashboard/src/lib/workstation-endpoints.ts +++ b/src/Meridian.Ui/dashboard/src/lib/workstation-endpoints.ts @@ -16,6 +16,7 @@ export const WORKSTATION_API_ENDPOINTS = { financialRecordExplorerRecord: UI_API_ROUTES.WorkstationFinancialRecordExplorerRecord, financialRecordExplorerSavedViews: UI_API_ROUTES.WorkstationFinancialRecordExplorerSavedViews, data: UI_API_ROUTES.WorkstationData, + dataQuery: UI_API_ROUTES.WorkstationDataQuery, dataUploadTemplates: UI_API_ROUTES.WorkstationDataUploadTemplates, dataUploadPreview: UI_API_ROUTES.WorkstationDataUploadPreview, accounting: UI_API_ROUTES.WorkstationAccounting, diff --git a/src/Meridian.Ui/dashboard/src/screens/data-screen.query-panel.view-model.test.ts b/src/Meridian.Ui/dashboard/src/screens/data-screen.query-panel.view-model.test.ts new file mode 100644 index 0000000000..737155032c --- /dev/null +++ b/src/Meridian.Ui/dashboard/src/screens/data-screen.query-panel.view-model.test.ts @@ -0,0 +1,76 @@ +import { act, renderHook } from "@testing-library/react"; +import { describe, expect, it, vi } from "vitest"; +import { + DEFAULT_DATA_QUERY_SQL, + useDataQueryPanel, + type DataQueryRunner, +} from "@/screens/data-screen.query-panel.view-model"; +import type { DataQueryResult } from "@/types"; + +function successResult(overrides: Partial = {}): DataQueryResult { + return { + success: true, + error: null, + columns: ["symbol", "event_count"], + columnTypes: ["VARCHAR", "BIGINT"], + rows: [["SPY", "5000"]], + rowCount: 1, + truncated: false, + elapsedMs: 12, + ...overrides, + }; +} + +describe("data query panel view model", () => { + it("starts with the discovery query and no result", () => { + const { result } = renderHook(() => useDataQueryPanel(vi.fn() as DataQueryRunner)); + + expect(result.current.sql).toBe(DEFAULT_DATA_QUERY_SQL); + expect(result.current.busy).toBe(false); + expect(result.current.result).toBeNull(); + expect(result.current.error).toBeNull(); + }); + + it("runs the current sql and stores the result", async () => { + const runQuery = vi.fn().mockResolvedValue(successResult()); + const { result } = renderHook(() => useDataQueryPanel(runQuery)); + + act(() => { + result.current.setSql("SELECT 1"); + }); + await act(async () => { + await result.current.run(); + }); + + expect(runQuery).toHaveBeenCalledWith({ sql: "SELECT 1" }); + expect(result.current.result?.rowCount).toBe(1); + expect(result.current.error).toBeNull(); + expect(result.current.busy).toBe(false); + }); + + it("surfaces guard rejections from success=false payloads", async () => { + const runQuery = vi.fn().mockResolvedValue( + successResult({ success: false, error: "Only read-only queries are allowed.", rows: [], rowCount: 0 }), + ); + const { result } = renderHook(() => useDataQueryPanel(runQuery)); + + await act(async () => { + await result.current.run(); + }); + + expect(result.current.result).toBeNull(); + expect(result.current.error).toContain("read-only"); + }); + + it("surfaces transport failures as errors", async () => { + const runQuery = vi.fn().mockRejectedValue(new Error("network down")); + const { result } = renderHook(() => useDataQueryPanel(runQuery)); + + await act(async () => { + await result.current.run(); + }); + + expect(result.current.result).toBeNull(); + expect(result.current.error).toBe("network down"); + }); +}); diff --git a/src/Meridian.Ui/dashboard/src/screens/data-screen.query-panel.view-model.ts b/src/Meridian.Ui/dashboard/src/screens/data-screen.query-panel.view-model.ts new file mode 100644 index 0000000000..9739cad149 --- /dev/null +++ b/src/Meridian.Ui/dashboard/src/screens/data-screen.query-panel.view-model.ts @@ -0,0 +1,49 @@ +import { useCallback, useState } from "react"; +import { runDataQuery } from "@/lib/api"; +import type { DataQueryResult } from "@/types"; + +export const DEFAULT_DATA_QUERY_SQL = + "SELECT symbol, event_type, date, format, size_bytes, event_count\nFROM meridian_files\nORDER BY symbol\nLIMIT 100"; + +export type DataQueryRunner = typeof runDataQuery; + +export interface DataQueryPanelViewModel { + sql: string; + setSql: (sql: string) => void; + busy: boolean; + result: DataQueryResult | null; + error: string | null; + run: () => Promise; +} + +/** + * View-model for the Data workspace SQL query panel. Guard violations and SQL errors arrive + * as a successful response with `success: false`, so they render inline; only transport + * failures surface through the catch path. + */ +export function useDataQueryPanel(runQuery: DataQueryRunner = runDataQuery): DataQueryPanelViewModel { + const [sql, setSql] = useState(DEFAULT_DATA_QUERY_SQL); + const [busy, setBusy] = useState(false); + const [result, setResult] = useState(null); + const [error, setError] = useState(null); + + const run = useCallback(async () => { + if (busy) { + return; + } + setBusy(true); + setError(null); + try { + const response = await runQuery({ sql }); + setResult(response.success ? response : null); + setError(response.success ? null : (response.error ?? "Query failed.")); + } catch (err) { + setResult(null); + setError(err instanceof Error ? err.message : "Query failed."); + } finally { + setBusy(false); + } + }, [busy, runQuery, sql]); + + return { sql, setSql, busy, result, error, run }; +} diff --git a/src/Meridian.Ui/dashboard/src/screens/data-screen.tsx b/src/Meridian.Ui/dashboard/src/screens/data-screen.tsx index 122b631dc8..78be1d5cea 100644 --- a/src/Meridian.Ui/dashboard/src/screens/data-screen.tsx +++ b/src/Meridian.Ui/dashboard/src/screens/data-screen.tsx @@ -33,6 +33,7 @@ import { StatusBanner } from "@/components/ui/status-banner"; import { TabPanel, Tabs } from "@/components/ui/tabs"; import { cn } from "@/lib/utils"; import { workspaceForPath } from "@/lib/workspace"; +import { useDataQueryPanel } from "@/screens/data-screen.query-panel.view-model"; import { DATA_BACKFILL_DETAIL_PANEL_ID, DATA_EXPORT_DETAIL_PANEL_ID, @@ -257,6 +258,7 @@ export function DataScreen({ providerRoutingTrustSnapshots ]); const vm = useDataViewModel(data, pathname, undefined, providerSetupLifecycle, providerEvidence); + const queryPanel = useDataQueryPanel(); if (!data) { return ; @@ -316,6 +318,7 @@ export function DataScreen({ Provider catalog Backfill queue Export packages + SQL query Watchlist Live quotes @@ -550,6 +553,66 @@ export function DataScreen({ + +
+ + SQL query + + Read-only DuckDB analytics over the local store. Start from the meridian_files catalog + view, then feed paths into read_parquet or read_json_auto. + + + +
+