diff --git a/libs/datatug/main/package.json b/libs/datatug/main/package.json index b2f1f17d..6d90a466 100644 --- a/libs/datatug/main/package.json +++ b/libs/datatug/main/package.json @@ -33,7 +33,9 @@ "@sneat/random": "*", "@sneat/api": "0.27.6", "firebase": ">=12.0.0 <13.0.0", - "vitest": "^4.0.18" + "vitest": "^4.0.18", + "ajv": "8.20.0", + "yaml": "2.9.0" }, "sideEffects": false } diff --git a/libs/datatug/main/src/lib/models/definition/query-def.ts b/libs/datatug/main/src/lib/models/definition/query-def.ts index e8b619c7..c17f4f5d 100644 --- a/libs/datatug/main/src/lib/models/definition/query-def.ts +++ b/libs/datatug/main/src/lib/models/definition/query-def.ts @@ -3,6 +3,8 @@ import { IParameterDef } from './parameter'; import { IRecordsetDef } from './recordset'; import { HttpMethod } from './command-definition'; import { IWidgetRef } from './widget'; +import type { BoundedFederation } from '../../queries/public-data/bounded-federation'; +import type { PublicDataScenario } from '../../queries/public-data/public-data-scenario'; export enum QueryType { HTTP = 'HTTP', @@ -39,6 +41,7 @@ export interface IQueryDef extends IQueryItem { recordsets?: IRecordsetDef[]; federation?: { readonly ovdbBaseUrl: string; + readonly bounds?: BoundedFederation; readonly tables: readonly { readonly name: string; readonly database?: string; readonly schema?: string; readonly fields: readonly string[] }[]; readonly lookups?: readonly { readonly database: string; @@ -48,6 +51,7 @@ export interface IQueryDef extends IQueryItem { readonly concurrency?: number; }[]; }; + publicData?: PublicDataScenario; widgets?: IWidgetRef[]; } diff --git a/libs/datatug/main/src/lib/pages/signed-in/project/project-page.component.html b/libs/datatug/main/src/lib/pages/signed-in/project/project-page.component.html index 866bdbc1..bdb58f7e 100644 --- a/libs/datatug/main/src/lib/pages/signed-in/project/project-page.component.html +++ b/libs/datatug/main/src/lib/pages/signed-in/project/project-page.component.html @@ -27,6 +27,7 @@ Boards Environments Queries + Public data lookup @if (enableEmptyShellPages) { Tags } diff --git a/libs/datatug/main/src/lib/queries/federated-query-executor.ts b/libs/datatug/main/src/lib/queries/federated-query-executor.ts index 2966a54b..e100043d 100644 --- a/libs/datatug/main/src/lib/queries/federated-query-executor.ts +++ b/libs/datatug/main/src/lib/queries/federated-query-executor.ts @@ -6,6 +6,8 @@ import { IndexedDbDatabase } from '@dalgo/indexeddb'; import type { RunQueryResponse, TypedValue } from '@sneat/datatug-semantic'; import type { IQueryDef, ITextQueryRequest } from '../models/definition/query-def'; import { deleteQueryDatabase, queryStorageError } from './federated-query-storage'; +import { createBoundedFederationFetch } from './public-data/bounded-federation'; +import { publicDataExceptions, SAVED_SCENARIO_PUBLICATION_BLOCKER, type PublicDataExceptions } from './public-data/public-data-scenario'; type Data = Record; interface OvdbRecord { readonly key: string; readonly data: Data } @@ -60,7 +62,7 @@ export interface FederatedQueryObserver { readonly onSourceLoaded?: (event: FederatedSourceLoaded) => void; } -export type FederatedQueryResult = RunQueryResponse & { readonly totalRows?: number; readonly hasMore?: boolean }; +export type FederatedQueryResult = RunQueryResponse & { readonly totalRows?: number; readonly hasMore?: boolean; readonly publicDataExceptions?: PublicDataExceptions; readonly publicDataBytes?: number }; export type FederatedOutputPage = (rows: readonly (readonly TypedValue[])[]) => Promise; export type FederatedQueryMode = 'full' | 'visible'; @@ -93,6 +95,33 @@ function retryDelay(attempt: number, signal?: AbortSignal): Promise { /** Runs each leaf against OVDB directly and merges/aggregates in this runtime. */ export async function runFederatedQuery(definition: IQueryDef, onProgress?: (progress: FederatedQueryProgress) => void, token = '', onOutputPage?: FederatedOutputPage, signal?: AbortSignal, mode: FederatedQueryMode = 'full', onPageReady?: (result: FederatedQueryResult) => void, waitForNextPage?: () => Promise, observer?: FederatedQueryObserver): Promise { + if (definition.publicData) throw new Error(SAVED_SCENARIO_PUBLICATION_BLOCKER); + const bounds = definition.federation?.bounds; + if (!bounds) return runFederatedQueryInternal(definition, onProgress, token, onOutputPage, signal, mode, onPageReady, waitForNextPage, observer); + if (mode !== 'full') throw new Error('Bounded public-data runs calculate one selected page. Browse result pages after it finishes.'); + const deadline = new AbortController(); + const timer = setTimeout(() => deadline.abort(new Error('The bounded lookup exceeded its deadline.')), bounds.timeoutMs); + const combined = signal ? AbortSignal.any([signal, deadline.signal]) : deadline.signal; + try { + const transport = createBoundedFederationFetch(definition.federation?.ovdbBaseUrl ?? '', bounds, observer?.fetch ?? fetch, combined); + let outputRows = 0; + const collected: TypedValue[][] = []; + const result = await runFederatedQueryInternal(definition, onProgress, token, async (rows) => { + combined.throwIfAborted(); + outputRows += rows.length; + if (outputRows > bounds.resultRows) throw new Error('The lookup exceeds the result-row bound; ambiguous or multiplied matches require a narrower query.'); + if (onOutputPage) await onOutputPage(rows); + else collected.push(...rows.map((row) => [...row])); + }, combined, 'full', undefined, undefined, { ...observer, fetch: transport.fetch }); + combined.throwIfAborted(); + if (result.recordset.rows.length > bounds.resultRows) throw new Error('The lookup exceeds the result-row bound.'); + // Nested joins use DALgo's batch result rather than the flat join callback. + if (!outputRows && result.recordset.rows.length && onOutputPage) await onOutputPage(result.recordset.rows); + return { ...result, ...(!onOutputPage && outputRows ? { recordset: { ...result.recordset, rows: collected } } : {}), publicDataExceptions: publicDataExceptions(bounds, transport.receipt.sources), publicDataBytes: transport.receipt.bytes(), limitations: [...result.limitations, { rowsFiltered: false, hiddenColumns: [], policy: `Selected user page: at most ${bounds.userRows} rows from offset ${bounds.userOffset}; ${bounds.identifierLimit} identifiers, ${bounds.resultRows} results, ${bounds.bytes} bytes, ${bounds.timeoutMs}ms.` }] }; + } finally { clearTimeout(timer); } +} + +async function runFederatedQueryInternal(definition: IQueryDef, onProgress?: (progress: FederatedQueryProgress) => void, token = '', onOutputPage?: FederatedOutputPage, signal?: AbortSignal, mode: FederatedQueryMode = 'full', onPageReady?: (result: FederatedQueryResult) => void, waitForNextPage?: () => Promise, observer?: FederatedQueryObserver): Promise { // Late-bound so a stubbed global fetch is honoured; an observer's fetch replaces it for this run only. const httpFetch: typeof fetch = observer?.fetch ?? ((input, init) => fetch(input, init)); const config = definition.federation; diff --git a/libs/datatug/main/src/lib/queries/federated-query.service.spec.ts b/libs/datatug/main/src/lib/queries/federated-query.service.spec.ts index 92712a40..7e56dd29 100644 --- a/libs/datatug/main/src/lib/queries/federated-query.service.spec.ts +++ b/libs/datatug/main/src/lib/queries/federated-query.service.spec.ts @@ -22,7 +22,7 @@ describe('federated query worker boundary', () => { postMessage(message: unknown): void { this.posted.push(message); } addEventListener(): void { /* not needed: no dispose in these runs */ } removeEventListener(): void { /* not needed */ } - terminate(): void { /* not needed */ } + terminate = vi.fn(); send(data: unknown): void { this.onmessage?.({ data } as MessageEvent); } } const definition = { id: 'query', title: 'Query', request: { queryType: QueryType.DTQL, text: '{}' } } as IQueryDef; @@ -63,5 +63,18 @@ describe('federated query worker boundary', () => { withSource.worker.send({ type: 'result', result }); await withSource.run; }); + it('terminates a bounded worker that never answers and rejects instead of keeping synchronous work alive', async () => { + vi.stubGlobal('Worker', FakeWorker); + const service = new FederatedQueryService(); + const bounded = { ...definition, federation: { ovdbBaseUrl: 'https://demodb.dev/ovdb', tables: [], bounds: { userRows: 100, userOffset: 0, identifierKind: 'place' as const, identifierLimit: 100, resultRows: 5000, bytes: 5242880, timeoutMs: 100, sources: [{ database: 'user', name: 'Customer', keyField: 'Country' }, { database: 'geo', name: 'countries', keyField: 'id', parent: { database: 'user', name: 'Customer', field: 'Country' } }] } } }; + FakeWorker.last = undefined; + const run = service.run(bounded); + const rejected = expect(run).rejects.toThrow(/exceeded its deadline/); + await vi.waitFor(() => expect(FakeWorker.last?.posted.length).toBe(1)); + const worker = FakeWorker.last; + await rejected; + expect(worker?.terminate).toHaveBeenCalledOnce(); + await expect(service.dispose()).resolves.toBeUndefined(); + }); }); }); diff --git a/libs/datatug/main/src/lib/queries/federated-query.service.ts b/libs/datatug/main/src/lib/queries/federated-query.service.ts index 452f81d9..3ac70d02 100644 --- a/libs/datatug/main/src/lib/queries/federated-query.service.ts +++ b/libs/datatug/main/src/lib/queries/federated-query.service.ts @@ -28,6 +28,16 @@ export class FederatedQueryService { return new Promise((resolve, reject) => { const worker = new Worker(new URL('./federated-query.worker.ts', import.meta.url), { type: 'module' }); this.worker = worker; + // A worker can be busy in synchronous DALgo work: enforce the wall-clock + // bound from the UI thread as well as aborting requests inside the worker. + const deadline = definition.federation?.bounds + ? setTimeout(() => { + worker.terminate(); + if (this.worker === worker) this.worker = undefined; + reject(new Error('The bounded lookup exceeded its deadline. Temporary storage is cleaned before the next run.')); + }, definition.federation.bounds.timeoutMs) + : undefined; + const clearDeadline = (): void => { if (deadline !== undefined) clearTimeout(deadline); }; worker.onmessage = (event: MessageEvent< | { type: 'progress'; progress: FederatedQueryProgress } | { type: 'source'; event: FederatedSourceLoaded } @@ -45,7 +55,7 @@ export class FederatedQueryService { if (message.type === 'source') { extras.onSourceLoaded?.(message.event); return; } if (message.type === 'finished') { onFinished?.(message.totalRows); return; } if (message.type === 'cleanup-error') return; - if (message.type === 'cancelled') { reject(new Error('The query was cancelled.')); return; } + if (message.type === 'cancelled') { clearDeadline(); reject(new Error('The query was cancelled.')); return; } if (message.type === 'page' || message.type === 'page-error') { const pending = this.pageRequests.get(message.requestId); this.pageRequests.delete(message.requestId); @@ -53,16 +63,18 @@ export class FederatedQueryService { else pending?.reject(new Error(message.message)); return; } - if (message.type === 'closed') { worker.terminate(); if (this.worker === worker) this.worker = undefined; return; } + if (message.type === 'closed') { clearDeadline(); worker.terminate(); if (this.worker === worker) this.worker = undefined; return; } if (message.type === 'result') { + clearDeadline(); if (mode === 'full' && message.result.totalRows === undefined) void this.dispose().catch(() => undefined); resolve(message.result); } else { + clearDeadline(); void this.dispose().catch(() => undefined); reject(new Error(message.message)); } }; - worker.onerror = (event) => { void this.dispose().catch(() => undefined); reject(new Error(event.message || 'The query worker failed.')); }; + worker.onerror = (event) => { clearDeadline(); void this.dispose().catch(() => undefined); reject(new Error(event.message || 'The query worker failed.')); }; worker.postMessage({ type: 'run', definition, token, mode, ...(extras.staticSource ? { staticSource: extras.staticSource } : {}) }); }); } diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/README.md b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/README.md new file mode 100644 index 00000000..19204068 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/README.md @@ -0,0 +1,26 @@ +# Hypothetical real-ROR user input + +These eight input rows declare a hypothetical `affiliations.Affiliation.ror_id` +string property. They are not a native field of any DemoDB database. The two +published URL IDs occur in the full ROR release dated 2026-09-22 (ROR release +v2.13, JSON member SHA256 +`6d032ff473f771e015da0837fe28b881a8f85822ee7815effefe11dc49c4346c`), +verified against `ingitdb/ror-ingitdb` source output at +`bbbec903248680caea04e68f94b9a957b6efc55b`. + +Proposed representation is native ROR URL (`ROR:URL`), UTF8 byte-exact identity, +with no trimming, case folding, URL shortening or organization-name inference. +`a1` and `a3` are two distinct affiliations to one organization. Multiple ROR +locations would remain ordered details of that organization, without multiplying +the affiliation denominator. NULL, empty, malformed case/space and synthetic +name inputs remain separate exceptions, preserving their raw strings. + +The exact immutable app commit and the schema path/hash identify this proposed +user mapping for a fresh independent reconciler. These files do not grant +semantic acceptance, canonical registration, native-id runtime support, +production eligibility or live journey acceptance. A committed declaration is +still pending until the dedicated decision and canonical companions land. + +Fixture rows and schema are app test material under the app repository licence; +referenced ROR metadata is CC0-1.0. Embedded ROR GeoNames location data, when +displayed, carries CC-BY-4.0 attribution separately. diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/affiliations.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/affiliations.json new file mode 100644 index 00000000..60841094 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/affiliations.json @@ -0,0 +1,10 @@ +[ + { "affiliation_id": "a1", "ror_id": "https://ror.org/000025p04" }, + { "affiliation_id": "a2", "ror_id": "https://ror.org/00002d369" }, + { "affiliation_id": "a3", "ror_id": "https://ror.org/000025p04" }, + { "affiliation_id": "null", "ror_id": null }, + { "affiliation_id": "empty", "ror_id": "" }, + { "affiliation_id": "space", "ror_id": " https://ror.org/000025p04" }, + { "affiliation_id": "case", "ror_id": "https://ROR.org/000025p04" }, + { "affiliation_id": "synthetic", "ror_id": "Synthetic organization" } +] diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/affiliations.modelspec.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/affiliations.modelspec.json new file mode 100644 index 00000000..b25e6de9 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/affiliations.modelspec.json @@ -0,0 +1,13 @@ +{ + "modelspec": "1.0-draft", + "module": { "name": "affiliations", "id": "github.com/datatug/datatug-apps/affiliations", "version": "0.1.0" }, + "entities": { + "Affiliation": { + "properties": { + "affiliation_id": { "type": "string", "required": true }, + "ror_id": { "type": "string", "required": false } + }, + "key": ["affiliation_id"] + } + } +} diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/README.md b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/README.md new file mode 100644 index 00000000..e8cc1511 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/README.md @@ -0,0 +1,7 @@ +# Synthetic consumer compatibility fixture + +Copied byte-for-byte from `openvaultdb/ovdb` commit `aec07d17be1e074210acefddb858363ce7777704`, `publisher/representation/testdata`, after independent review of the schema1 helper. These example repositories and labels are hypothetical test inputs. They do not register a provider or admit a semantic decision. Tests mutate copies and retain the original pinned bytes here. + +The consumer remains closed pending real Directory/provider companions, independent scoped semantic acceptance and runtime publication. Direct native identifier mode is a separate, unfrozen schema proposal; this fixture only exercises the reviewed label bridge shape. + +`directory.pinned.json` and `models.pinned.json` are exact canonical index bytes from Directory commit `02db362144d7924c6081dd6768cc0d7187ac9bc3` and ModelSpec registry commit `48b30b250a61385d94d46a70968677870750ea35`. Their SHA256 values match INITIAL_CANONICAL_PINS. They test connection/provider/entity/property coordinates and never grant semantic eligibility. The configured adapter test uses an explicit synthetic representation namespace; its source schema hash is the actual Chinook model JSON at Directory provider commit `26e852cca00101f53a84ef8ee1f1ae389067f5cf`. diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/bridge.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/bridge.json new file mode 100644 index 00000000..299fc266 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/bridge.json @@ -0,0 +1,13 @@ +{ + "table": "CustomerCountries", + "rows": [ + { + "raw_label": "USA", + "target_key": "US" + }, + { + "raw_label": "United Kingdom", + "target_key": "GB" + } + ] +} diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/contract.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/contract.json new file mode 100644 index 00000000..6addd13d --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/contract.json @@ -0,0 +1,82 @@ +{ + "format": "ovdb-representation-contract/1", + "contracts": [ + { + "source": { + "schema": { + "path": "source.modelspec.json", + "sha256": "7043520817a957c3c3e175fc09877238c320eae236f1ebbb5e80957024551905", + "repository": "https://github.com/example/source", + "revision": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "module": "sample", + "entity": "Customer", + "property": "Country", + "datatype": "string", + "namespace": "sample-country-labels" + }, + "target": { + "snapshot": { + "path": "snapshot.json", + "sha256": "6f4ec14c1ba7adad094ec6d95d889c1387a570eb110d86f15c005af7ad8cf447" + }, + "keys": { + "path": "keys.json", + "sha256": "5aa000ef1e2c824aa4dd6315256e5e181ba0a334ed94b43aca252d17434f222e" + }, + "model": { + "path": "target.modelspec.json", + "sha256": "aa5b9ea22bf18fb6f81b3ab537431c05a99a54384f1773edc4a890ff17bd410d" + }, + "module": "geo", + "entity": "Countries", + "property": "iso", + "datatype": "string", + "namespace": "iso-3166-1-alpha-2", + "binding": { + "document": { + "path": "target.meaning.json", + "sha256": "a447619f70c4de82848091c7eed913b54fd6bbc0264ff65c15568a9a27d1d954" + }, + "concept": "native-country", + "role": "identifier", + "meaning": { + "document": { + "path": "core.meaning.json", + "sha256": "af890560ed133e6a4aaa40d22c82027fc329e7e9f9cf8da2886b5415650292a2", + "repository": "https://github.com/example/source", + "revision": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "concept": "country" + } + } + }, + "bridge": { + "artifact": { + "path": "bridge.json", + "sha256": "fd225cf26ca051471e983c770e4af49edd8bdce8d429729e53b4722d677a5bc9" + }, + "table": "CustomerCountries", + "raw_label_column": "raw_label", + "target_key_column": "target_key", + "serving_identity_column": "serving_id" + }, + "policy": { + "transform": "identity", + "equality": "utf8-byte-exact", + "cardinality": "zero-or-one", + "unmatched": "exception", + "collision": "ineligible" + }, + "decision": { + "document": { + "path": "decision.md", + "sha256": "ad838e4f5321fdc424a2ae16a4f3288a1e901e11a2968a697c539df2e2ac0f66", + "repository": "https://github.com/example/source", + "revision": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "scope": "fixture-only/Customer.Country" + } + } + ] +} diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/core.meaning.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/core.meaning.json new file mode 100644 index 00000000..886fddaa --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/core.meaning.json @@ -0,0 +1,8 @@ +{ + "format": "meaning/draft-1", + "concepts": [ + { + "id": "country" + } + ] +} diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/decision.md b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/decision.md new file mode 100644 index 00000000..b6601c56 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/decision.md @@ -0,0 +1,3 @@ +# Synthetic acceptance fixture + +Fixture-only Customer.Country bridge scope; never production admission. diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/directory.pinned.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/directory.pinned.json new file mode 100644 index 00000000..6d09fa1d --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/directory.pinned.json @@ -0,0 +1,3012 @@ +{ + "format": "ovdb-directory/draft-1", + "checksum": "sha256:a40f66f811e9606e47efbd9c29a067b2e19f749fb3d8274c3b20436f38a04865", + "databases": [ + { + "id": "https://demodb.dev/chinook/", + "recordId": "chinook", + "localId": "chinook", + "directoryPath": "/ovdb/demodb.dev/chinook/", + "serverId": "https://demodb.dev/ovdb", + "serverDbBaseUrl": "https://demodb.dev/ovdb/db/chinook/", + "apiUrl": "https://demodb.dev/ovdb/v1/databases/chinook", + "title": "Chinook music store", + "description": "The Chinook sample database: a digital music store with artists, albums, tracks, playlists, customers, employees and invoices. Read-only, served from a pinned copy of the upstream SQLite fixture. Its recordsets and fields carry the meanings of the chinook meaning graph.", + "status": "draft", + "url": "https://demodb.dev/chinook/", + "deployment": { + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook", + "engine": "sqlite" + }, + "homepage": "https://chinook.demodb.dev/", + "repository": "https://github.com/demo-db/chinook", + "commit": "26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "manifest": "ovdb.yaml", + "licence": "MIT", + "model": { + "name": "chinook", + "path": "model/chinook.modelspec.hcl", + "address": "modelspec://github.com/demo-db/chinook/chinook" + }, + "meaning_graph": { + "id": "chinook", + "address": "meaning://github.com/demo-db/chinook" + }, + "recordsets": [ + { + "name": "Album", + "modelEntity": "Album", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook/collections/Album", + "meanings": [ + { + "graph": "chinook", + "concept": "album", + "label": "Album", + "role": "entity", + "address": "meaning://github.com/demo-db/chinook/album?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ], + "fields": [ + { + "name": "AlbumId", + "type": "int", + "meanings": [] + }, + { + "name": "Title", + "type": "string", + "meanings": [ + { + "graph": "chinook", + "concept": "album", + "label": "Album", + "role": "display-name", + "address": "meaning://github.com/demo-db/chinook/album?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + }, + { + "name": "ArtistId", + "type": "reference", + "references": "Artist", + "meanings": [ + { + "graph": "chinook", + "concept": "artist", + "label": "Artist", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/chinook/artist?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + } + ] + }, + { + "name": "Artist", + "modelEntity": "Artist", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook/collections/Artist", + "meanings": [ + { + "graph": "chinook", + "concept": "artist", + "label": "Artist", + "role": "entity", + "address": "meaning://github.com/demo-db/chinook/artist?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ], + "fields": [ + { + "name": "ArtistId", + "type": "int", + "meanings": [ + { + "graph": "chinook", + "concept": "artist", + "label": "Artist", + "role": "identifier", + "address": "meaning://github.com/demo-db/chinook/artist?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + }, + { + "name": "Name", + "type": "string", + "meanings": [ + { + "graph": "chinook", + "concept": "artist", + "label": "Artist", + "role": "display-name", + "address": "meaning://github.com/demo-db/chinook/artist?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + } + ] + }, + { + "name": "Customer", + "modelEntity": "Customer", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook/collections/Customer", + "meanings": [ + { + "graph": "chinook", + "concept": "customer", + "label": "Customer", + "role": "entity", + "address": "meaning://github.com/demo-db/chinook/customer?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "customer", + "label": "Customer", + "address": "meaning://github.com/meaninggraph/core/customer?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "CustomerId", + "type": "int", + "meanings": [] + }, + { + "name": "FirstName", + "type": "string", + "meanings": [] + }, + { + "name": "LastName", + "type": "string", + "meanings": [] + }, + { + "name": "Company", + "type": "string", + "meanings": [] + }, + { + "name": "Address", + "type": "string", + "meanings": [] + }, + { + "name": "City", + "type": "string", + "meanings": [] + }, + { + "name": "State", + "type": "string", + "meanings": [] + }, + { + "name": "Country", + "type": "string", + "meanings": [ + { + "graph": "chinook", + "concept": "customer-country", + "label": "Customer country", + "role": "value", + "address": "meaning://github.com/demo-db/chinook/customer-country?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [], + "values_of": { + "graph": "core", + "concept": "country", + "label": "Country", + "address": "meaning://github.com/meaninggraph/core/country?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97", + "extends": [] + } + } + ] + }, + { + "name": "PostalCode", + "type": "string", + "meanings": [] + }, + { + "name": "Phone", + "type": "string", + "meanings": [] + }, + { + "name": "Fax", + "type": "string", + "meanings": [] + }, + { + "name": "Email", + "type": "string", + "meanings": [] + }, + { + "name": "SupportRepId", + "type": "reference", + "references": "Employee", + "meanings": [ + { + "graph": "chinook", + "concept": "support-rep", + "label": "Support representative", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/chinook/support-rep?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [], + "values_of": { + "graph": "chinook", + "concept": "employee", + "label": "Employee", + "address": "meaning://github.com/demo-db/chinook/employee?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "employee", + "label": "Employee", + "address": "meaning://github.com/meaninggraph/core/employee?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "person", + "label": "Person", + "address": "meaning://github.com/meaninggraph/core/person?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + } + ] + } + ] + }, + { + "name": "Employee", + "modelEntity": "Employee", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook/collections/Employee", + "meanings": [ + { + "graph": "chinook", + "concept": "employee", + "label": "Employee", + "role": "entity", + "address": "meaning://github.com/demo-db/chinook/employee?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "employee", + "label": "Employee", + "address": "meaning://github.com/meaninggraph/core/employee?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "person", + "label": "Person", + "address": "meaning://github.com/meaninggraph/core/person?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "EmployeeId", + "type": "int", + "meanings": [] + }, + { + "name": "LastName", + "type": "string", + "meanings": [] + }, + { + "name": "FirstName", + "type": "string", + "meanings": [] + }, + { + "name": "Title", + "type": "string", + "meanings": [] + }, + { + "name": "ReportsTo", + "type": "reference", + "references": "Employee", + "meanings": [ + { + "graph": "chinook", + "concept": "manager", + "label": "Manager", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/chinook/manager?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "manager", + "label": "Manager", + "address": "meaning://github.com/meaninggraph/core/manager?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ], + "values_of": { + "graph": "chinook", + "concept": "employee", + "label": "Employee", + "address": "meaning://github.com/demo-db/chinook/employee?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "employee", + "label": "Employee", + "address": "meaning://github.com/meaninggraph/core/employee?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "person", + "label": "Person", + "address": "meaning://github.com/meaninggraph/core/person?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + } + ] + }, + { + "name": "BirthDate", + "type": "datetime", + "meanings": [] + }, + { + "name": "HireDate", + "type": "datetime", + "meanings": [] + }, + { + "name": "Address", + "type": "string", + "meanings": [] + }, + { + "name": "City", + "type": "string", + "meanings": [] + }, + { + "name": "State", + "type": "string", + "meanings": [] + }, + { + "name": "Country", + "type": "string", + "meanings": [] + }, + { + "name": "PostalCode", + "type": "string", + "meanings": [] + }, + { + "name": "Phone", + "type": "string", + "meanings": [] + }, + { + "name": "Fax", + "type": "string", + "meanings": [] + }, + { + "name": "Email", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "Genre", + "modelEntity": "Genre", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook/collections/Genre", + "meanings": [ + { + "graph": "chinook", + "concept": "genre", + "label": "Genre", + "role": "entity", + "address": "meaning://github.com/demo-db/chinook/genre?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ], + "fields": [ + { + "name": "GenreId", + "type": "int", + "meanings": [] + }, + { + "name": "Name", + "type": "string", + "meanings": [ + { + "graph": "chinook", + "concept": "genre", + "label": "Genre", + "role": "display-name", + "address": "meaning://github.com/demo-db/chinook/genre?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + } + ] + }, + { + "name": "Invoice", + "modelEntity": "Invoice", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook/collections/Invoice", + "meanings": [ + { + "graph": "chinook", + "concept": "invoice", + "label": "Invoice", + "role": "entity", + "address": "meaning://github.com/demo-db/chinook/invoice?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "invoice", + "label": "Invoice", + "address": "meaning://github.com/meaninggraph/core/invoice?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "InvoiceId", + "type": "int", + "meanings": [] + }, + { + "name": "CustomerId", + "type": "reference", + "references": "Customer", + "meanings": [ + { + "graph": "chinook", + "concept": "customer", + "label": "Customer", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/chinook/customer?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "customer", + "label": "Customer", + "address": "meaning://github.com/meaninggraph/core/customer?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + }, + { + "name": "InvoiceDate", + "type": "datetime", + "meanings": [ + { + "graph": "chinook", + "concept": "invoice-date", + "label": "Invoice date", + "role": "value", + "address": "meaning://github.com/demo-db/chinook/invoice-date?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "date", + "label": "Date", + "address": "meaning://github.com/meaninggraph/core/date?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + }, + { + "name": "BillingAddress", + "type": "string", + "meanings": [] + }, + { + "name": "BillingCity", + "type": "string", + "meanings": [] + }, + { + "name": "BillingState", + "type": "string", + "meanings": [] + }, + { + "name": "BillingCountry", + "type": "string", + "meanings": [ + { + "graph": "chinook", + "concept": "billing-country", + "label": "Country", + "role": "value", + "address": "meaning://github.com/demo-db/chinook/billing-country?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [], + "values_of": { + "graph": "core", + "concept": "country", + "label": "Country", + "address": "meaning://github.com/meaninggraph/core/country?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97", + "extends": [] + } + } + ] + }, + { + "name": "BillingPostalCode", + "type": "string", + "meanings": [] + }, + { + "name": "Total", + "type": "decimal", + "meanings": [ + { + "graph": "chinook", + "concept": "invoice-total", + "label": "Invoice total", + "role": "value", + "address": "meaning://github.com/demo-db/chinook/invoice-total?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "invoice-total", + "label": "Invoice total", + "address": "meaning://github.com/meaninggraph/core/invoice-total?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "money-amount", + "label": "Money amount", + "address": "meaning://github.com/meaninggraph/core/money-amount?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + }, + { + "graph": "chinook", + "concept": "music-sales", + "label": "Music sales", + "role": "value", + "address": "meaning://github.com/demo-db/chinook/music-sales?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "revenue", + "label": "Revenue", + "address": "meaning://github.com/meaninggraph/core/revenue?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + } + ] + }, + { + "name": "InvoiceLine", + "modelEntity": "InvoiceLine", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook/collections/InvoiceLine", + "meanings": [ + { + "graph": "chinook", + "concept": "invoice-line", + "label": "Invoice line", + "role": "entity", + "address": "meaning://github.com/demo-db/chinook/invoice-line?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "invoice-line", + "label": "Invoice line", + "address": "meaning://github.com/meaninggraph/core/invoice-line?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "commercial-line-item", + "label": "Commercial line item", + "address": "meaning://github.com/meaninggraph/core/commercial-line-item?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "InvoiceLineId", + "type": "int", + "meanings": [] + }, + { + "name": "InvoiceId", + "type": "reference", + "references": "Invoice", + "meanings": [ + { + "graph": "chinook", + "concept": "invoice", + "label": "Invoice", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/chinook/invoice?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "invoice", + "label": "Invoice", + "address": "meaning://github.com/meaninggraph/core/invoice?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + }, + { + "name": "TrackId", + "type": "reference", + "references": "Track", + "meanings": [ + { + "graph": "chinook", + "concept": "track", + "label": "Track", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/chinook/track?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + }, + { + "name": "UnitPrice", + "type": "decimal", + "meanings": [ + { + "graph": "chinook", + "concept": "unit-price", + "label": "Unit price", + "role": "value", + "address": "meaning://github.com/demo-db/chinook/unit-price?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "unit-price", + "label": "Unit price", + "address": "meaning://github.com/meaninggraph/core/unit-price?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "price", + "label": "Price", + "address": "meaning://github.com/meaninggraph/core/price?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "money-amount", + "label": "Money amount", + "address": "meaning://github.com/meaninggraph/core/money-amount?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + }, + { + "name": "Quantity", + "type": "int", + "meanings": [ + { + "graph": "chinook", + "concept": "quantity", + "label": "Quantity", + "role": "value", + "address": "meaning://github.com/demo-db/chinook/quantity?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "quantity", + "label": "Quantity", + "address": "meaning://github.com/meaninggraph/core/quantity?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + } + ] + }, + { + "name": "MediaType", + "modelEntity": "MediaType", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook/collections/MediaType", + "meanings": [ + { + "graph": "chinook", + "concept": "media-type", + "label": "Media type", + "role": "entity", + "address": "meaning://github.com/demo-db/chinook/media-type?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ], + "fields": [ + { + "name": "MediaTypeId", + "type": "int", + "meanings": [] + }, + { + "name": "Name", + "type": "string", + "meanings": [ + { + "graph": "chinook", + "concept": "media-type", + "label": "Media type", + "role": "display-name", + "address": "meaning://github.com/demo-db/chinook/media-type?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + } + ] + }, + { + "name": "Playlist", + "modelEntity": "Playlist", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook/collections/Playlist", + "meanings": [ + { + "graph": "chinook", + "concept": "playlist", + "label": "Playlist", + "role": "entity", + "address": "meaning://github.com/demo-db/chinook/playlist?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ], + "fields": [ + { + "name": "PlaylistId", + "type": "int", + "meanings": [] + }, + { + "name": "Name", + "type": "string", + "meanings": [ + { + "graph": "chinook", + "concept": "playlist", + "label": "Playlist", + "role": "display-name", + "address": "meaning://github.com/demo-db/chinook/playlist?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + } + ] + }, + { + "name": "PlaylistTrack", + "modelEntity": "PlaylistTrack", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook/collections/PlaylistTrack", + "meanings": [], + "fields": [ + { + "name": "PlaylistId", + "type": "reference", + "references": "Playlist", + "meanings": [ + { + "graph": "chinook", + "concept": "playlist", + "label": "Playlist", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/chinook/playlist?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + }, + { + "name": "TrackId", + "type": "reference", + "references": "Track", + "meanings": [] + } + ] + }, + { + "name": "Track", + "modelEntity": "Track", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/chinook/collections/Track", + "meanings": [ + { + "graph": "chinook", + "concept": "track", + "label": "Track", + "role": "entity", + "address": "meaning://github.com/demo-db/chinook/track?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ], + "fields": [ + { + "name": "TrackId", + "type": "int", + "meanings": [] + }, + { + "name": "Name", + "type": "string", + "meanings": [ + { + "graph": "chinook", + "concept": "track", + "label": "Track", + "role": "display-name", + "address": "meaning://github.com/demo-db/chinook/track?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + }, + { + "name": "AlbumId", + "type": "reference", + "references": "Album", + "meanings": [ + { + "graph": "chinook", + "concept": "album", + "label": "Album", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/chinook/album?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + }, + { + "name": "MediaTypeId", + "type": "reference", + "references": "MediaType", + "meanings": [ + { + "graph": "chinook", + "concept": "media-type", + "label": "Media type", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/chinook/media-type?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + }, + { + "name": "GenreId", + "type": "reference", + "references": "Genre", + "meanings": [ + { + "graph": "chinook", + "concept": "genre", + "label": "Genre", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/chinook/genre?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + }, + { + "name": "Composer", + "type": "string", + "meanings": [] + }, + { + "name": "Milliseconds", + "type": "int", + "meanings": [ + { + "graph": "chinook", + "concept": "track-length", + "label": "Track length", + "role": "value", + "address": "meaning://github.com/demo-db/chinook/track-length?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [] + } + ] + }, + { + "name": "Bytes", + "type": "int", + "meanings": [] + }, + { + "name": "UnitPrice", + "type": "decimal", + "meanings": [ + { + "graph": "chinook", + "concept": "list-price", + "label": "List price", + "role": "value", + "address": "meaning://github.com/demo-db/chinook/list-price?ref=26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "extends": [ + { + "graph": "core", + "concept": "price", + "label": "Price", + "address": "meaning://github.com/meaninggraph/core/price?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "money-amount", + "label": "Money amount", + "address": "meaning://github.com/meaninggraph/core/money-amount?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + } + ] + } + ] + }, + { + "id": "https://demodb.dev/northwind/", + "recordId": "northwind", + "localId": "northwind", + "directoryPath": "/ovdb/demodb.dev/northwind/", + "serverId": "https://demodb.dev/ovdb", + "serverDbBaseUrl": "https://demodb.dev/ovdb/db/northwind/", + "apiUrl": "https://demodb.dev/ovdb/v1/databases/northwind", + "title": "Northwind small-business sample database", + "description": "The classic Northwind sample database for customers, orders, products, suppliers, shippers, territories and employees. The pinned SQLite build is read-only; the native schema includes 13 tables and 17 views.", + "status": "draft", + "url": "https://demodb.dev/northwind/", + "deployment": { + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind", + "engine": "sqlite" + }, + "homepage": "https://northwind.demodb.dev/", + "repository": "https://github.com/demo-db/northwind", + "commit": "e74726515c3833620b54b7a50d1d273276dd23c1", + "manifest": "ovdb.yaml", + "licence": "MIT", + "model": { + "name": "northwind", + "path": "model/northwind.modelspec.hcl", + "address": "modelspec://github.com/demo-db/northwind/northwind" + }, + "meaning_graph": { + "id": "northwind", + "address": "meaning://github.com/demo-db/northwind" + }, + "recordsets": [ + { + "name": "Categories", + "modelEntity": "Categories", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/Categories", + "meanings": [], + "fields": [ + { + "name": "CategoryID", + "type": "int", + "meanings": [] + }, + { + "name": "CategoryName", + "type": "string", + "meanings": [] + }, + { + "name": "Description", + "type": "string", + "meanings": [] + }, + { + "name": "Picture", + "type": "document", + "meanings": [] + } + ] + }, + { + "name": "CustomerCustomerDemo", + "modelEntity": "CustomerCustomerDemo", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/CustomerCustomerDemo", + "meanings": [], + "fields": [ + { + "name": "CustomerID", + "type": "reference", + "references": "Customers", + "meanings": [] + }, + { + "name": "CustomerTypeID", + "type": "reference", + "references": "CustomerDemographics", + "meanings": [] + } + ] + }, + { + "name": "CustomerDemographics", + "modelEntity": "CustomerDemographics", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/CustomerDemographics", + "meanings": [], + "fields": [ + { + "name": "CustomerTypeID", + "type": "string", + "meanings": [] + }, + { + "name": "CustomerDesc", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "Customers", + "modelEntity": "Customers", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/Customers", + "meanings": [ + { + "graph": "northwind", + "concept": "customer", + "label": "customer", + "role": "entity", + "address": "meaning://github.com/demo-db/northwind/customer?ref=e74726515c3833620b54b7a50d1d273276dd23c1", + "extends": [ + { + "graph": "core", + "concept": "customer", + "label": "Customer", + "address": "meaning://github.com/meaninggraph/core/customer?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "CustomerID", + "type": "string", + "meanings": [] + }, + { + "name": "CompanyName", + "type": "string", + "meanings": [] + }, + { + "name": "ContactName", + "type": "string", + "meanings": [] + }, + { + "name": "ContactTitle", + "type": "string", + "meanings": [] + }, + { + "name": "Address", + "type": "string", + "meanings": [] + }, + { + "name": "City", + "type": "string", + "meanings": [] + }, + { + "name": "Region", + "type": "string", + "meanings": [] + }, + { + "name": "PostalCode", + "type": "string", + "meanings": [] + }, + { + "name": "Country", + "type": "string", + "meanings": [] + }, + { + "name": "Phone", + "type": "string", + "meanings": [] + }, + { + "name": "Fax", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "EmployeeTerritories", + "modelEntity": "EmployeeTerritories", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/EmployeeTerritories", + "meanings": [], + "fields": [ + { + "name": "EmployeeID", + "type": "reference", + "references": "Employees", + "meanings": [] + }, + { + "name": "TerritoryID", + "type": "reference", + "references": "Territories", + "meanings": [] + } + ] + }, + { + "name": "Employees", + "modelEntity": "Employees", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/Employees", + "meanings": [ + { + "graph": "northwind", + "concept": "employee", + "label": "employee", + "role": "entity", + "address": "meaning://github.com/demo-db/northwind/employee?ref=e74726515c3833620b54b7a50d1d273276dd23c1", + "extends": [ + { + "graph": "core", + "concept": "employee", + "label": "Employee", + "address": "meaning://github.com/meaninggraph/core/employee?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "person", + "label": "Person", + "address": "meaning://github.com/meaninggraph/core/person?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "EmployeeID", + "type": "int", + "meanings": [] + }, + { + "name": "LastName", + "type": "string", + "meanings": [] + }, + { + "name": "FirstName", + "type": "string", + "meanings": [] + }, + { + "name": "Title", + "type": "string", + "meanings": [] + }, + { + "name": "TitleOfCourtesy", + "type": "string", + "meanings": [] + }, + { + "name": "BirthDate", + "type": "date", + "meanings": [] + }, + { + "name": "HireDate", + "type": "date", + "meanings": [] + }, + { + "name": "Address", + "type": "string", + "meanings": [] + }, + { + "name": "City", + "type": "string", + "meanings": [] + }, + { + "name": "Region", + "type": "string", + "meanings": [] + }, + { + "name": "PostalCode", + "type": "string", + "meanings": [] + }, + { + "name": "Country", + "type": "string", + "meanings": [] + }, + { + "name": "HomePhone", + "type": "string", + "meanings": [] + }, + { + "name": "Extension", + "type": "string", + "meanings": [] + }, + { + "name": "Photo", + "type": "document", + "meanings": [] + }, + { + "name": "Notes", + "type": "string", + "meanings": [] + }, + { + "name": "ReportsTo", + "type": "reference", + "references": "Employees", + "meanings": [] + }, + { + "name": "PhotoPath", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "Order Details", + "modelEntity": "OrderDetails", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/Order%20Details", + "meanings": [ + { + "graph": "northwind", + "concept": "order-line", + "label": "sales order line", + "role": "entity", + "address": "meaning://github.com/demo-db/northwind/order-line?ref=e74726515c3833620b54b7a50d1d273276dd23c1", + "extends": [ + { + "graph": "core", + "concept": "order-line", + "label": "Order line", + "address": "meaning://github.com/meaninggraph/core/order-line?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "commercial-line-item", + "label": "Commercial line item", + "address": "meaning://github.com/meaninggraph/core/commercial-line-item?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "OrderID", + "type": "reference", + "references": "Orders", + "meanings": [] + }, + { + "name": "ProductID", + "type": "reference", + "references": "Products", + "meanings": [] + }, + { + "name": "UnitPrice", + "type": "decimal", + "meanings": [] + }, + { + "name": "Quantity", + "type": "int", + "meanings": [] + }, + { + "name": "Discount", + "type": "decimal", + "meanings": [] + } + ] + }, + { + "name": "Orders", + "modelEntity": "Orders", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/Orders", + "meanings": [ + { + "graph": "northwind", + "concept": "order", + "label": "sales order", + "role": "entity", + "address": "meaning://github.com/demo-db/northwind/order?ref=e74726515c3833620b54b7a50d1d273276dd23c1", + "extends": [ + { + "graph": "core", + "concept": "order", + "label": "Order", + "address": "meaning://github.com/meaninggraph/core/order?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "OrderID", + "type": "int", + "meanings": [] + }, + { + "name": "CustomerID", + "type": "reference", + "references": "Customers", + "meanings": [] + }, + { + "name": "EmployeeID", + "type": "reference", + "references": "Employees", + "meanings": [] + }, + { + "name": "OrderDate", + "type": "datetime", + "meanings": [] + }, + { + "name": "RequiredDate", + "type": "datetime", + "meanings": [] + }, + { + "name": "ShippedDate", + "type": "datetime", + "meanings": [] + }, + { + "name": "ShipVia", + "type": "reference", + "references": "Shippers", + "meanings": [] + }, + { + "name": "Freight", + "type": "decimal", + "meanings": [] + }, + { + "name": "ShipName", + "type": "string", + "meanings": [] + }, + { + "name": "ShipAddress", + "type": "string", + "meanings": [] + }, + { + "name": "ShipCity", + "type": "string", + "meanings": [] + }, + { + "name": "ShipRegion", + "type": "string", + "meanings": [] + }, + { + "name": "ShipPostalCode", + "type": "string", + "meanings": [] + }, + { + "name": "ShipCountry", + "type": "string", + "meanings": [ + { + "graph": "northwind", + "concept": "shipping-country", + "label": "shipping country", + "role": "value", + "address": "meaning://github.com/demo-db/northwind/shipping-country?ref=e74726515c3833620b54b7a50d1d273276dd23c1", + "extends": [], + "values_of": { + "graph": "core", + "concept": "country", + "label": "Country", + "address": "meaning://github.com/meaninggraph/core/country?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97", + "extends": [] + } + } + ] + } + ] + }, + { + "name": "Products", + "modelEntity": "Products", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/Products", + "meanings": [], + "fields": [ + { + "name": "ProductID", + "type": "int", + "meanings": [] + }, + { + "name": "ProductName", + "type": "string", + "meanings": [] + }, + { + "name": "SupplierID", + "type": "reference", + "references": "Suppliers", + "meanings": [] + }, + { + "name": "CategoryID", + "type": "reference", + "references": "Categories", + "meanings": [] + }, + { + "name": "QuantityPerUnit", + "type": "string", + "meanings": [] + }, + { + "name": "UnitPrice", + "type": "decimal", + "meanings": [] + }, + { + "name": "UnitsInStock", + "type": "int", + "meanings": [] + }, + { + "name": "UnitsOnOrder", + "type": "int", + "meanings": [] + }, + { + "name": "ReorderLevel", + "type": "int", + "meanings": [] + }, + { + "name": "Discontinued", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "Regions", + "modelEntity": "Regions", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/Regions", + "meanings": [], + "fields": [ + { + "name": "RegionID", + "type": "int", + "meanings": [] + }, + { + "name": "RegionDescription", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "Shippers", + "modelEntity": "Shippers", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/Shippers", + "meanings": [], + "fields": [ + { + "name": "ShipperID", + "type": "int", + "meanings": [] + }, + { + "name": "CompanyName", + "type": "string", + "meanings": [] + }, + { + "name": "Phone", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "Suppliers", + "modelEntity": "Suppliers", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/Suppliers", + "meanings": [], + "fields": [ + { + "name": "SupplierID", + "type": "int", + "meanings": [] + }, + { + "name": "CompanyName", + "type": "string", + "meanings": [] + }, + { + "name": "ContactName", + "type": "string", + "meanings": [] + }, + { + "name": "ContactTitle", + "type": "string", + "meanings": [] + }, + { + "name": "Address", + "type": "string", + "meanings": [] + }, + { + "name": "City", + "type": "string", + "meanings": [] + }, + { + "name": "Region", + "type": "string", + "meanings": [] + }, + { + "name": "PostalCode", + "type": "string", + "meanings": [] + }, + { + "name": "Country", + "type": "string", + "meanings": [] + }, + { + "name": "Phone", + "type": "string", + "meanings": [] + }, + { + "name": "Fax", + "type": "string", + "meanings": [] + }, + { + "name": "HomePage", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "Territories", + "modelEntity": "Territories", + "url": "https://cloud.openvaultdb.com/ovdb/dbs/northwind/collections/Territories", + "meanings": [], + "fields": [ + { + "name": "TerritoryID", + "type": "string", + "meanings": [] + }, + { + "name": "TerritoryDescription", + "type": "string", + "meanings": [] + }, + { + "name": "RegionID", + "type": "reference", + "references": "Regions", + "meanings": [] + } + ] + } + ] + }, + { + "id": "https://demodb.dev/pubs/", + "recordId": "pubs", + "localId": "pubs", + "directoryPath": "/ovdb/demodb.dev/pubs/", + "serverId": "https://demodb.dev/ovdb", + "serverDbBaseUrl": "https://demodb.dev/ovdb/db/pubs/", + "apiUrl": "https://demodb.dev/ovdb/v1/databases/pubs", + "title": "Microsoft Pubs publishing sample", + "description": "The classic Microsoft Pubs sample database for authors, publications, publishers, stores, sales and publisher-logo images. Its read-only SQLite source preserves all eleven native tables, including two tables without primary or unique keys.", + "status": "draft", + "url": "https://demodb.dev/pubs/", + "deployment": { + "url": "https://cloud.openvaultdb.com/ovdb/dbs/pubs", + "engine": "sqlite" + }, + "homepage": "https://pubs.demodb.dev/", + "repository": "https://github.com/demo-db/pubs", + "commit": "6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "manifest": "ovdb.yaml", + "licence": "MIT", + "model": { + "name": "pubs", + "path": "model/pubs.modelspec.hcl", + "address": "modelspec://github.com/demo-db/pubs/pubs" + }, + "meaning_graph": { + "id": "pubs", + "address": "meaning://github.com/demo-db/pubs" + }, + "recordsets": [ + { + "name": "authors", + "modelEntity": "authors", + "meanings": [ + { + "graph": "pubs", + "concept": "author", + "label": "Author", + "role": "entity", + "address": "meaning://github.com/demo-db/pubs/author?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [ + { + "graph": "core", + "concept": "person", + "label": "Person", + "address": "meaning://github.com/meaninggraph/core/person?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "au_id", + "type": "string", + "meanings": [] + }, + { + "name": "au_lname", + "type": "string", + "meanings": [] + }, + { + "name": "au_fname", + "type": "string", + "meanings": [] + }, + { + "name": "phone", + "type": "string", + "meanings": [] + }, + { + "name": "address", + "type": "string", + "meanings": [] + }, + { + "name": "city", + "type": "string", + "meanings": [] + }, + { + "name": "state", + "type": "string", + "meanings": [] + }, + { + "name": "zip", + "type": "string", + "meanings": [] + }, + { + "name": "contract", + "type": "int", + "meanings": [] + } + ] + }, + { + "name": "discounts", + "modelEntity": "discounts", + "meanings": [], + "fields": [ + { + "name": "discounttype", + "type": "string", + "meanings": [] + }, + { + "name": "stor_id", + "type": "reference", + "references": "stores", + "meanings": [ + { + "graph": "pubs", + "concept": "store", + "label": "Store", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/pubs/store?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [] + } + ] + }, + { + "name": "lowqty", + "type": "int", + "meanings": [] + }, + { + "name": "highqty", + "type": "int", + "meanings": [] + }, + { + "name": "discount", + "type": "decimal", + "meanings": [] + } + ] + }, + { + "name": "employee", + "modelEntity": "employee", + "meanings": [ + { + "graph": "pubs", + "concept": "employee", + "label": "Employee", + "role": "entity", + "address": "meaning://github.com/demo-db/pubs/employee?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [ + { + "graph": "core", + "concept": "employee", + "label": "Employee", + "address": "meaning://github.com/meaninggraph/core/employee?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "person", + "label": "Person", + "address": "meaning://github.com/meaninggraph/core/person?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "emp_id", + "type": "string", + "meanings": [] + }, + { + "name": "fname", + "type": "string", + "meanings": [] + }, + { + "name": "minit", + "type": "string", + "meanings": [] + }, + { + "name": "lname", + "type": "string", + "meanings": [] + }, + { + "name": "job_id", + "type": "reference", + "references": "jobs", + "meanings": [] + }, + { + "name": "job_lvl", + "type": "int", + "meanings": [] + }, + { + "name": "pub_id", + "type": "reference", + "references": "publishers", + "meanings": [ + { + "graph": "pubs", + "concept": "publisher", + "label": "Publisher", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/pubs/publisher?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [ + { + "graph": "core", + "concept": "organization", + "label": "Organization", + "address": "meaning://github.com/meaninggraph/core/organization?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + }, + { + "name": "hire_date", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "jobs", + "modelEntity": "jobs", + "meanings": [], + "fields": [ + { + "name": "job_id", + "type": "int", + "meanings": [] + }, + { + "name": "job_desc", + "type": "string", + "meanings": [] + }, + { + "name": "min_lvl", + "type": "int", + "meanings": [] + }, + { + "name": "max_lvl", + "type": "int", + "meanings": [] + } + ] + }, + { + "name": "pub_info", + "modelEntity": "pub_info", + "meanings": [], + "fields": [ + { + "name": "pub_id", + "type": "reference", + "references": "publishers", + "meanings": [ + { + "graph": "pubs", + "concept": "publisher", + "label": "Publisher", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/pubs/publisher?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [ + { + "graph": "core", + "concept": "organization", + "label": "Organization", + "address": "meaning://github.com/meaninggraph/core/organization?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + }, + { + "name": "logo", + "type": "document", + "meanings": [] + }, + { + "name": "pr_info", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "publishers", + "modelEntity": "publishers", + "meanings": [ + { + "graph": "pubs", + "concept": "publisher", + "label": "Publisher", + "role": "entity", + "address": "meaning://github.com/demo-db/pubs/publisher?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [ + { + "graph": "core", + "concept": "organization", + "label": "Organization", + "address": "meaning://github.com/meaninggraph/core/organization?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "pub_id", + "type": "string", + "meanings": [] + }, + { + "name": "pub_name", + "type": "string", + "meanings": [] + }, + { + "name": "city", + "type": "string", + "meanings": [] + }, + { + "name": "state", + "type": "string", + "meanings": [] + }, + { + "name": "country", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "roysched", + "modelEntity": "roysched", + "meanings": [], + "fields": [ + { + "name": "title_id", + "type": "reference", + "references": "titles", + "meanings": [ + { + "graph": "pubs", + "concept": "book", + "label": "Book", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/pubs/book?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [] + } + ] + }, + { + "name": "lorange", + "type": "int", + "meanings": [] + }, + { + "name": "hirange", + "type": "int", + "meanings": [] + }, + { + "name": "royalty", + "type": "int", + "meanings": [] + } + ] + }, + { + "name": "sales", + "modelEntity": "sales", + "meanings": [ + { + "graph": "pubs", + "concept": "sale", + "label": "Sale", + "role": "entity", + "address": "meaning://github.com/demo-db/pubs/sale?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [] + } + ], + "fields": [ + { + "name": "stor_id", + "type": "reference", + "references": "stores", + "meanings": [ + { + "graph": "pubs", + "concept": "store", + "label": "Store", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/pubs/store?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [] + } + ] + }, + { + "name": "ord_num", + "type": "string", + "meanings": [] + }, + { + "name": "ord_date", + "type": "string", + "meanings": [] + }, + { + "name": "qty", + "type": "int", + "meanings": [ + { + "graph": "pubs", + "concept": "sale-quantity", + "label": "Sale quantity", + "role": "value", + "address": "meaning://github.com/demo-db/pubs/sale-quantity?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [] + } + ] + }, + { + "name": "payterms", + "type": "string", + "meanings": [] + }, + { + "name": "title_id", + "type": "reference", + "references": "titles", + "meanings": [ + { + "graph": "pubs", + "concept": "book", + "label": "Book", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/pubs/book?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [] + } + ] + } + ] + }, + { + "name": "stores", + "modelEntity": "stores", + "meanings": [ + { + "graph": "pubs", + "concept": "store", + "label": "Store", + "role": "entity", + "address": "meaning://github.com/demo-db/pubs/store?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [] + } + ], + "fields": [ + { + "name": "stor_id", + "type": "string", + "meanings": [] + }, + { + "name": "stor_name", + "type": "string", + "meanings": [] + }, + { + "name": "stor_address", + "type": "string", + "meanings": [] + }, + { + "name": "city", + "type": "string", + "meanings": [] + }, + { + "name": "state", + "type": "string", + "meanings": [] + }, + { + "name": "zip", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "titleauthor", + "modelEntity": "titleauthor", + "meanings": [], + "fields": [ + { + "name": "au_id", + "type": "reference", + "references": "authors", + "meanings": [ + { + "graph": "pubs", + "concept": "author", + "label": "Author", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/pubs/author?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [ + { + "graph": "core", + "concept": "person", + "label": "Person", + "address": "meaning://github.com/meaninggraph/core/person?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + }, + { + "name": "title_id", + "type": "reference", + "references": "titles", + "meanings": [ + { + "graph": "pubs", + "concept": "book", + "label": "Book", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/pubs/book?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [] + } + ] + }, + { + "name": "au_ord", + "type": "int", + "meanings": [] + }, + { + "name": "royaltyper", + "type": "int", + "meanings": [] + } + ] + }, + { + "name": "titles", + "modelEntity": "titles", + "meanings": [ + { + "graph": "pubs", + "concept": "book", + "label": "Book", + "role": "entity", + "address": "meaning://github.com/demo-db/pubs/book?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [] + } + ], + "fields": [ + { + "name": "title_id", + "type": "string", + "meanings": [] + }, + { + "name": "title", + "type": "string", + "meanings": [] + }, + { + "name": "type", + "type": "string", + "meanings": [] + }, + { + "name": "pub_id", + "type": "reference", + "references": "publishers", + "meanings": [ + { + "graph": "pubs", + "concept": "publisher", + "label": "Publisher", + "role": "foreign-key", + "address": "meaning://github.com/demo-db/pubs/publisher?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [ + { + "graph": "core", + "concept": "organization", + "label": "Organization", + "address": "meaning://github.com/meaninggraph/core/organization?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + }, + { + "name": "price", + "type": "decimal", + "meanings": [ + { + "graph": "pubs", + "concept": "book-price", + "label": "Book price", + "role": "value", + "address": "meaning://github.com/demo-db/pubs/book-price?ref=6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "extends": [ + { + "graph": "core", + "concept": "price", + "label": "Price", + "address": "meaning://github.com/meaninggraph/core/price?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "money-amount", + "label": "Money amount", + "address": "meaning://github.com/meaninggraph/core/money-amount?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ] + }, + { + "name": "advance", + "type": "decimal", + "meanings": [] + }, + { + "name": "royalty", + "type": "int", + "meanings": [] + }, + { + "name": "ytd_sales", + "type": "int", + "meanings": [] + }, + { + "name": "notes", + "type": "string", + "meanings": [] + }, + { + "name": "pubdate", + "type": "string", + "meanings": [] + } + ] + } + ] + }, + { + "id": "https://demodb.dev/sakila/", + "recordId": "sakila", + "localId": "sakila", + "directoryPath": "/ovdb/demodb.dev/sakila/", + "serverId": "https://demodb.dev/ovdb", + "serverDbBaseUrl": "https://demodb.dev/ovdb/db/sakila/", + "apiUrl": "https://demodb.dev/ovdb/v1/databases/sakila", + "title": "Sakila video rental sample", + "description": "MySQL's Sakila video rental sample, published as a read-only SQLite fixture with native tables, views, keys, foreign keys, and BLOB values preserved.", + "status": "draft", + "url": "https://demodb.dev/sakila/", + "deployment": { + "url": "https://cloud.openvaultdb.com/ovdb/dbs/sakila", + "engine": "sqlite" + }, + "homepage": "https://sakila.demodb.dev/", + "repository": "https://github.com/demo-db/sakila", + "commit": "6567d30aec1592fe0917934a8bbe74ff70b04b01", + "manifest": "ovdb.yaml", + "licence": "BSD-3-Clause", + "model": { + "name": "sakila", + "path": "model/sakila.modelspec.hcl", + "address": "modelspec://github.com/demo-db/sakila/sakila" + }, + "meaning_graph": { + "id": "sakila", + "address": "meaning://github.com/demo-db/sakila" + }, + "recordsets": [ + { + "name": "actor", + "modelEntity": "actor", + "meanings": [ + { + "graph": "sakila", + "concept": "actor", + "label": "Actor", + "role": "entity", + "address": "meaning://github.com/demo-db/sakila/actor?ref=6567d30aec1592fe0917934a8bbe74ff70b04b01", + "extends": [ + { + "graph": "core", + "concept": "person", + "label": "Person", + "address": "meaning://github.com/meaninggraph/core/person?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "actor_id", + "type": "int", + "meanings": [] + }, + { + "name": "first_name", + "type": "string", + "meanings": [] + }, + { + "name": "last_name", + "type": "string", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "address", + "modelEntity": "address", + "meanings": [], + "fields": [ + { + "name": "address_id", + "type": "int", + "meanings": [] + }, + { + "name": "address", + "type": "string", + "meanings": [] + }, + { + "name": "address2", + "type": "string", + "meanings": [] + }, + { + "name": "district", + "type": "string", + "meanings": [] + }, + { + "name": "city_id", + "type": "reference", + "references": "city", + "meanings": [] + }, + { + "name": "postal_code", + "type": "string", + "meanings": [] + }, + { + "name": "phone", + "type": "string", + "meanings": [] + }, + { + "name": "location", + "type": "document", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "category", + "modelEntity": "category", + "meanings": [], + "fields": [ + { + "name": "category_id", + "type": "int", + "meanings": [] + }, + { + "name": "name", + "type": "string", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "city", + "modelEntity": "city", + "meanings": [], + "fields": [ + { + "name": "city_id", + "type": "int", + "meanings": [] + }, + { + "name": "city", + "type": "string", + "meanings": [] + }, + { + "name": "country_id", + "type": "reference", + "references": "country", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "country", + "modelEntity": "country", + "meanings": [], + "fields": [ + { + "name": "country_id", + "type": "int", + "meanings": [] + }, + { + "name": "country", + "type": "string", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "customer", + "modelEntity": "customer", + "meanings": [ + { + "graph": "sakila", + "concept": "customer", + "label": "Customer", + "role": "entity", + "address": "meaning://github.com/demo-db/sakila/customer?ref=6567d30aec1592fe0917934a8bbe74ff70b04b01", + "extends": [ + { + "graph": "core", + "concept": "customer", + "label": "Customer", + "address": "meaning://github.com/meaninggraph/core/customer?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "customer_id", + "type": "int", + "meanings": [] + }, + { + "name": "store_id", + "type": "reference", + "references": "store", + "meanings": [] + }, + { + "name": "first_name", + "type": "string", + "meanings": [] + }, + { + "name": "last_name", + "type": "string", + "meanings": [] + }, + { + "name": "email", + "type": "string", + "meanings": [] + }, + { + "name": "address_id", + "type": "reference", + "references": "address", + "meanings": [] + }, + { + "name": "active", + "type": "int", + "meanings": [] + }, + { + "name": "create_date", + "type": "string", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "film", + "modelEntity": "film", + "meanings": [ + { + "graph": "sakila", + "concept": "film", + "label": "Film", + "role": "entity", + "address": "meaning://github.com/demo-db/sakila/film?ref=6567d30aec1592fe0917934a8bbe74ff70b04b01", + "extends": [] + } + ], + "fields": [ + { + "name": "film_id", + "type": "int", + "meanings": [] + }, + { + "name": "title", + "type": "string", + "meanings": [] + }, + { + "name": "description", + "type": "string", + "meanings": [] + }, + { + "name": "release_year", + "type": "string", + "meanings": [] + }, + { + "name": "language_id", + "type": "reference", + "references": "language", + "meanings": [] + }, + { + "name": "original_language_id", + "type": "reference", + "references": "language", + "meanings": [] + }, + { + "name": "rental_duration", + "type": "int", + "meanings": [] + }, + { + "name": "rental_rate", + "type": "decimal", + "meanings": [] + }, + { + "name": "length", + "type": "int", + "meanings": [] + }, + { + "name": "replacement_cost", + "type": "decimal", + "meanings": [] + }, + { + "name": "rating", + "type": "string", + "meanings": [] + }, + { + "name": "special_features", + "type": "string", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "film_actor", + "modelEntity": "film_actor", + "meanings": [], + "fields": [ + { + "name": "actor_id", + "type": "reference", + "references": "actor", + "meanings": [] + }, + { + "name": "film_id", + "type": "reference", + "references": "film", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "film_category", + "modelEntity": "film_category", + "meanings": [], + "fields": [ + { + "name": "film_id", + "type": "reference", + "references": "film", + "meanings": [] + }, + { + "name": "category_id", + "type": "reference", + "references": "category", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "film_text", + "modelEntity": "film_text", + "meanings": [], + "fields": [ + { + "name": "film_id", + "type": "int", + "meanings": [] + }, + { + "name": "title", + "type": "string", + "meanings": [] + }, + { + "name": "description", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "inventory", + "modelEntity": "inventory", + "meanings": [], + "fields": [ + { + "name": "inventory_id", + "type": "int", + "meanings": [] + }, + { + "name": "film_id", + "type": "reference", + "references": "film", + "meanings": [] + }, + { + "name": "store_id", + "type": "reference", + "references": "store", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "language", + "modelEntity": "language", + "meanings": [], + "fields": [ + { + "name": "language_id", + "type": "int", + "meanings": [] + }, + { + "name": "name", + "type": "string", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "payment", + "modelEntity": "payment", + "meanings": [ + { + "graph": "sakila", + "concept": "payment", + "label": "Payment", + "role": "entity", + "address": "meaning://github.com/demo-db/sakila/payment?ref=6567d30aec1592fe0917934a8bbe74ff70b04b01", + "extends": [] + } + ], + "fields": [ + { + "name": "payment_id", + "type": "int", + "meanings": [] + }, + { + "name": "customer_id", + "type": "reference", + "references": "customer", + "meanings": [] + }, + { + "name": "staff_id", + "type": "reference", + "references": "staff", + "meanings": [] + }, + { + "name": "rental_id", + "type": "reference", + "references": "rental", + "meanings": [] + }, + { + "name": "amount", + "type": "decimal", + "meanings": [] + }, + { + "name": "payment_date", + "type": "string", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "rental", + "modelEntity": "rental", + "meanings": [ + { + "graph": "sakila", + "concept": "rental", + "label": "Rental", + "role": "entity", + "address": "meaning://github.com/demo-db/sakila/rental?ref=6567d30aec1592fe0917934a8bbe74ff70b04b01", + "extends": [] + } + ], + "fields": [ + { + "name": "rental_id", + "type": "int", + "meanings": [] + }, + { + "name": "rental_date", + "type": "string", + "meanings": [] + }, + { + "name": "inventory_id", + "type": "reference", + "references": "inventory", + "meanings": [] + }, + { + "name": "customer_id", + "type": "reference", + "references": "customer", + "meanings": [] + }, + { + "name": "return_date", + "type": "string", + "meanings": [] + }, + { + "name": "staff_id", + "type": "reference", + "references": "staff", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "staff", + "modelEntity": "staff", + "meanings": [ + { + "graph": "sakila", + "concept": "employee", + "label": "Staff member", + "role": "entity", + "address": "meaning://github.com/demo-db/sakila/employee?ref=6567d30aec1592fe0917934a8bbe74ff70b04b01", + "extends": [ + { + "graph": "core", + "concept": "employee", + "label": "Employee", + "address": "meaning://github.com/meaninggraph/core/employee?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + }, + { + "graph": "core", + "concept": "person", + "label": "Person", + "address": "meaning://github.com/meaninggraph/core/person?ref=982916d73f0a35ff2558b0062f58aa3ac4f24d97" + } + ] + } + ], + "fields": [ + { + "name": "staff_id", + "type": "int", + "meanings": [] + }, + { + "name": "first_name", + "type": "string", + "meanings": [] + }, + { + "name": "last_name", + "type": "string", + "meanings": [] + }, + { + "name": "address_id", + "type": "reference", + "references": "address", + "meanings": [] + }, + { + "name": "picture", + "type": "document", + "meanings": [] + }, + { + "name": "email", + "type": "string", + "meanings": [] + }, + { + "name": "store_id", + "type": "reference", + "references": "store", + "meanings": [] + }, + { + "name": "active", + "type": "int", + "meanings": [] + }, + { + "name": "username", + "type": "string", + "meanings": [] + }, + { + "name": "password", + "type": "string", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + }, + { + "name": "store", + "modelEntity": "store", + "meanings": [ + { + "graph": "sakila", + "concept": "store", + "label": "Store", + "role": "entity", + "address": "meaning://github.com/demo-db/sakila/store?ref=6567d30aec1592fe0917934a8bbe74ff70b04b01", + "extends": [] + } + ], + "fields": [ + { + "name": "store_id", + "type": "int", + "meanings": [] + }, + { + "name": "manager_staff_id", + "type": "reference", + "references": "staff", + "meanings": [] + }, + { + "name": "address_id", + "type": "reference", + "references": "address", + "meanings": [] + }, + { + "name": "last_update", + "type": "string", + "meanings": [] + } + ] + } + ] + } + ] +} diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/keys.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/keys.json new file mode 100644 index 00000000..a3b68e57 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/keys.json @@ -0,0 +1,7 @@ +{ + "namespace": "iso-3166-1-alpha-2", + "keys": [ + "US", + "GB" + ] +} diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/models.pinned.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/models.pinned.json new file mode 100644 index 00000000..34e96660 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/models.pinned.json @@ -0,0 +1,6398 @@ +{ + "format": "modelspec-registry/draft-1", + "checksum": "sha256:6943b507f49b2b3987d75c8f16b76d3ed2bbf1a9e04a96575386875117723391", + "models": [ + { + "id": "adventureworks", + "title": "AdventureWorks OLTP", + "description": "The ModelSpec for Microsoft's AdventureWorks OLTP sample, derived from the pinned SQL Server 2025 sample installer fixture. It describes the source tables and properties as a reusable model for copies of this database.", + "status": "draft", + "homepage": "https://demodb.dev/adventureworks/", + "address": "modelspec://github.com/demo-db/adventureworks/adventureworks", + "repository": "https://github.com/demo-db/adventureworks", + "commit": "cd8dcdf2079fe31480ad6d6c024b8c17cb91beea", + "module": "adventureworks", + "module_id": "github.com/demo-db/adventureworks/adventureworks", + "module_version": "0.1.0", + "modelspec": "1.0-draft", + "licence": "MIT", + "files": { + "source": "model/adventureworks.modelspec.hcl", + "json": "model/adventureworks.modelspec.json" + }, + "maintainers": [ + "trakhimenok" + ], + "entities": [ + { + "name": "HumanResources_Department", + "key": [ + "DepartmentID" + ], + "use": [], + "properties": [ + { + "name": "DepartmentID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "GroupName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "HumanResources_Employee", + "key": [ + "BusinessEntityID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Person_Person", + "required": true, + "key": true + }, + { + "name": "NationalIDNumber", + "type": "string", + "required": true, + "key": false + }, + { + "name": "LoginID", + "type": "string", + "required": true, + "key": false + }, + { + "name": "OrganizationNode", + "type": "document", + "required": false, + "key": false + }, + { + "name": "OrganizationLevel", + "type": "int", + "required": false, + "key": false + }, + { + "name": "JobTitle", + "type": "string", + "required": true, + "key": false + }, + { + "name": "BirthDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "MaritalStatus", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Gender", + "type": "string", + "required": true, + "key": false + }, + { + "name": "HireDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "SalariedFlag", + "type": "int", + "required": true, + "key": false + }, + { + "name": "VacationHours", + "type": "int", + "required": true, + "key": false + }, + { + "name": "SickLeaveHours", + "type": "int", + "required": true, + "key": false + }, + { + "name": "CurrentFlag", + "type": "int", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "HumanResources_EmployeeDepartmentHistory", + "key": [ + "BusinessEntityID", + "StartDate", + "DepartmentID", + "ShiftID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "HumanResources_Employee", + "required": true, + "key": true + }, + { + "name": "DepartmentID", + "type": "reference", + "references": "HumanResources_Department", + "required": true, + "key": true + }, + { + "name": "ShiftID", + "type": "reference", + "references": "HumanResources_Shift", + "required": true, + "key": true + }, + { + "name": "StartDate", + "type": "string", + "required": true, + "key": true + }, + { + "name": "EndDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "HumanResources_EmployeePayHistory", + "key": [ + "BusinessEntityID", + "RateChangeDate" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "HumanResources_Employee", + "required": true, + "key": true + }, + { + "name": "RateChangeDate", + "type": "string", + "required": true, + "key": true + }, + { + "name": "Rate", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "PayFrequency", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "HumanResources_JobCandidate", + "key": [ + "JobCandidateID" + ], + "use": [], + "properties": [ + { + "name": "JobCandidateID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "BusinessEntityID", + "type": "reference", + "references": "HumanResources_Employee", + "required": false, + "key": false + }, + { + "name": "Resume", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "HumanResources_Shift", + "key": [ + "ShiftID" + ], + "use": [], + "properties": [ + { + "name": "ShiftID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "StartTime", + "type": "string", + "required": true, + "key": false + }, + { + "name": "EndTime", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_Address", + "key": [ + "AddressID" + ], + "use": [], + "properties": [ + { + "name": "AddressID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "AddressLine1", + "type": "string", + "required": true, + "key": false + }, + { + "name": "AddressLine2", + "type": "string", + "required": false, + "key": false + }, + { + "name": "City", + "type": "string", + "required": true, + "key": false + }, + { + "name": "StateProvinceID", + "type": "reference", + "references": "Person_StateProvince", + "required": true, + "key": false + }, + { + "name": "PostalCode", + "type": "string", + "required": true, + "key": false + }, + { + "name": "SpatialLocation", + "type": "document", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_AddressType", + "key": [ + "AddressTypeID" + ], + "use": [], + "properties": [ + { + "name": "AddressTypeID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_BusinessEntity", + "key": [ + "BusinessEntityID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_BusinessEntityAddress", + "key": [ + "BusinessEntityID", + "AddressID", + "AddressTypeID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Person_BusinessEntity", + "required": true, + "key": true + }, + { + "name": "AddressID", + "type": "reference", + "references": "Person_Address", + "required": true, + "key": true + }, + { + "name": "AddressTypeID", + "type": "reference", + "references": "Person_AddressType", + "required": true, + "key": true + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_BusinessEntityContact", + "key": [ + "BusinessEntityID", + "PersonID", + "ContactTypeID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Person_BusinessEntity", + "required": true, + "key": true + }, + { + "name": "PersonID", + "type": "reference", + "references": "Person_Person", + "required": true, + "key": true + }, + { + "name": "ContactTypeID", + "type": "reference", + "references": "Person_ContactType", + "required": true, + "key": true + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_ContactType", + "key": [ + "ContactTypeID" + ], + "use": [], + "properties": [ + { + "name": "ContactTypeID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_CountryRegion", + "key": [ + "CountryRegionCode" + ], + "use": [], + "properties": [ + { + "name": "CountryRegionCode", + "type": "string", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_EmailAddress", + "key": [ + "BusinessEntityID", + "EmailAddressID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Person_Person", + "required": true, + "key": true + }, + { + "name": "EmailAddressID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "EmailAddress", + "type": "string", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_Password", + "key": [ + "BusinessEntityID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Person_Person", + "required": true, + "key": true + }, + { + "name": "PasswordHash", + "type": "string", + "required": true, + "key": false + }, + { + "name": "PasswordSalt", + "type": "string", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_Person", + "key": [ + "BusinessEntityID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Person_BusinessEntity", + "required": true, + "key": true + }, + { + "name": "PersonType", + "type": "string", + "required": true, + "key": false + }, + { + "name": "NameStyle", + "type": "int", + "required": true, + "key": false + }, + { + "name": "Title", + "type": "string", + "required": false, + "key": false + }, + { + "name": "FirstName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "MiddleName", + "type": "string", + "required": false, + "key": false + }, + { + "name": "LastName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Suffix", + "type": "string", + "required": false, + "key": false + }, + { + "name": "EmailPromotion", + "type": "int", + "required": true, + "key": false + }, + { + "name": "AdditionalContactInfo", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Demographics", + "type": "string", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_PersonPhone", + "key": [ + "BusinessEntityID", + "PhoneNumber", + "PhoneNumberTypeID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Person_Person", + "required": true, + "key": true + }, + { + "name": "PhoneNumber", + "type": "string", + "required": true, + "key": true + }, + { + "name": "PhoneNumberTypeID", + "type": "reference", + "references": "Person_PhoneNumberType", + "required": true, + "key": true + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_PhoneNumberType", + "key": [ + "PhoneNumberTypeID" + ], + "use": [], + "properties": [ + { + "name": "PhoneNumberTypeID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Person_StateProvince", + "key": [ + "StateProvinceID" + ], + "use": [], + "properties": [ + { + "name": "StateProvinceID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "StateProvinceCode", + "type": "string", + "required": true, + "key": false + }, + { + "name": "CountryRegionCode", + "type": "reference", + "references": "Person_CountryRegion", + "required": true, + "key": false + }, + { + "name": "IsOnlyStateProvinceFlag", + "type": "int", + "required": true, + "key": false + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "TerritoryID", + "type": "reference", + "references": "Sales_SalesTerritory", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_BillOfMaterials", + "key": [ + "BillOfMaterialsID" + ], + "use": [], + "properties": [ + { + "name": "BillOfMaterialsID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "ProductAssemblyID", + "type": "reference", + "references": "Production_Product", + "required": false, + "key": false + }, + { + "name": "ComponentID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": false + }, + { + "name": "StartDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "EndDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "UnitMeasureCode", + "type": "reference", + "references": "Production_UnitMeasure", + "required": true, + "key": false + }, + { + "name": "BOMLevel", + "type": "int", + "required": true, + "key": false + }, + { + "name": "PerAssemblyQty", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_Culture", + "key": [ + "CultureID" + ], + "use": [], + "properties": [ + { + "name": "CultureID", + "type": "string", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_Document", + "key": [ + "DocumentNode" + ], + "use": [], + "properties": [ + { + "name": "DocumentNode", + "type": "document", + "required": true, + "key": true + }, + { + "name": "DocumentLevel", + "type": "int", + "required": false, + "key": false + }, + { + "name": "Title", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Owner", + "type": "reference", + "references": "HumanResources_Employee", + "required": true, + "key": false + }, + { + "name": "FolderFlag", + "type": "int", + "required": true, + "key": false + }, + { + "name": "FileName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "FileExtension", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Revision", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ChangeNumber", + "type": "int", + "required": true, + "key": false + }, + { + "name": "Status", + "type": "int", + "required": true, + "key": false + }, + { + "name": "DocumentSummary", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Document", + "type": "document", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_Illustration", + "key": [ + "IllustrationID" + ], + "use": [], + "properties": [ + { + "name": "IllustrationID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Diagram", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_Location", + "key": [ + "LocationID" + ], + "use": [], + "properties": [ + { + "name": "LocationID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "CostRate", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "Availability", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_Product", + "key": [ + "ProductID" + ], + "use": [], + "properties": [ + { + "name": "ProductID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ProductNumber", + "type": "string", + "required": true, + "key": false + }, + { + "name": "MakeFlag", + "type": "int", + "required": true, + "key": false + }, + { + "name": "FinishedGoodsFlag", + "type": "int", + "required": true, + "key": false + }, + { + "name": "Color", + "type": "string", + "required": false, + "key": false + }, + { + "name": "SafetyStockLevel", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ReorderPoint", + "type": "int", + "required": true, + "key": false + }, + { + "name": "StandardCost", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "ListPrice", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "Size", + "type": "string", + "required": false, + "key": false + }, + { + "name": "SizeUnitMeasureCode", + "type": "reference", + "references": "Production_UnitMeasure", + "required": false, + "key": false + }, + { + "name": "WeightUnitMeasureCode", + "type": "reference", + "references": "Production_UnitMeasure", + "required": false, + "key": false + }, + { + "name": "Weight", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "DaysToManufacture", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ProductLine", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Class", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Style", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ProductSubcategoryID", + "type": "reference", + "references": "Production_ProductSubcategory", + "required": false, + "key": false + }, + { + "name": "ProductModelID", + "type": "reference", + "references": "Production_ProductModel", + "required": false, + "key": false + }, + { + "name": "SellStartDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "SellEndDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "DiscontinuedDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductCategory", + "key": [ + "ProductCategoryID" + ], + "use": [], + "properties": [ + { + "name": "ProductCategoryID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductCostHistory", + "key": [ + "ProductID", + "StartDate" + ], + "use": [], + "properties": [ + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": true + }, + { + "name": "StartDate", + "type": "string", + "required": true, + "key": true + }, + { + "name": "EndDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "StandardCost", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductDescription", + "key": [ + "ProductDescriptionID" + ], + "use": [], + "properties": [ + { + "name": "ProductDescriptionID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Description", + "type": "string", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductDocument", + "key": [ + "ProductID", + "DocumentNode" + ], + "use": [], + "properties": [ + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": true + }, + { + "name": "DocumentNode", + "type": "reference", + "references": "Production_Document", + "required": true, + "key": true + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductInventory", + "key": [ + "ProductID", + "LocationID" + ], + "use": [], + "properties": [ + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": true + }, + { + "name": "LocationID", + "type": "reference", + "references": "Production_Location", + "required": true, + "key": true + }, + { + "name": "Shelf", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Bin", + "type": "int", + "required": true, + "key": false + }, + { + "name": "Quantity", + "type": "int", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductListPriceHistory", + "key": [ + "ProductID", + "StartDate" + ], + "use": [], + "properties": [ + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": true + }, + { + "name": "StartDate", + "type": "string", + "required": true, + "key": true + }, + { + "name": "EndDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ListPrice", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductModel", + "key": [ + "ProductModelID" + ], + "use": [], + "properties": [ + { + "name": "ProductModelID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "CatalogDescription", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Instructions", + "type": "string", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductModelIllustration", + "key": [ + "ProductModelID", + "IllustrationID" + ], + "use": [], + "properties": [ + { + "name": "ProductModelID", + "type": "reference", + "references": "Production_ProductModel", + "required": true, + "key": true + }, + { + "name": "IllustrationID", + "type": "reference", + "references": "Production_Illustration", + "required": true, + "key": true + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductModelProductDescriptionCulture", + "key": [ + "ProductModelID", + "ProductDescriptionID", + "CultureID" + ], + "use": [], + "properties": [ + { + "name": "ProductModelID", + "type": "reference", + "references": "Production_ProductModel", + "required": true, + "key": true + }, + { + "name": "ProductDescriptionID", + "type": "reference", + "references": "Production_ProductDescription", + "required": true, + "key": true + }, + { + "name": "CultureID", + "type": "reference", + "references": "Production_Culture", + "required": true, + "key": true + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductPhoto", + "key": [ + "ProductPhotoID" + ], + "use": [], + "properties": [ + { + "name": "ProductPhotoID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "ThumbNailPhoto", + "type": "document", + "required": false, + "key": false + }, + { + "name": "ThumbnailPhotoFileName", + "type": "string", + "required": false, + "key": false + }, + { + "name": "LargePhoto", + "type": "document", + "required": false, + "key": false + }, + { + "name": "LargePhotoFileName", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductProductPhoto", + "key": [ + "ProductID", + "ProductPhotoID" + ], + "use": [], + "properties": [ + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": true + }, + { + "name": "ProductPhotoID", + "type": "reference", + "references": "Production_ProductPhoto", + "required": true, + "key": true + }, + { + "name": "Primary", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductReview", + "key": [ + "ProductReviewID" + ], + "use": [], + "properties": [ + { + "name": "ProductReviewID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": false + }, + { + "name": "ReviewerName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ReviewDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "EmailAddress", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Rating", + "type": "int", + "required": true, + "key": false + }, + { + "name": "Comments", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ProductSubcategory", + "key": [ + "ProductSubcategoryID" + ], + "use": [], + "properties": [ + { + "name": "ProductSubcategoryID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "ProductCategoryID", + "type": "reference", + "references": "Production_ProductCategory", + "required": true, + "key": false + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_ScrapReason", + "key": [ + "ScrapReasonID" + ], + "use": [], + "properties": [ + { + "name": "ScrapReasonID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_TransactionHistory", + "key": [ + "TransactionID" + ], + "use": [], + "properties": [ + { + "name": "TransactionID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": false + }, + { + "name": "ReferenceOrderID", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ReferenceOrderLineID", + "type": "int", + "required": true, + "key": false + }, + { + "name": "TransactionDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "TransactionType", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Quantity", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ActualCost", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_TransactionHistoryArchive", + "key": [ + "TransactionID" + ], + "use": [], + "properties": [ + { + "name": "TransactionID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "ProductID", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ReferenceOrderID", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ReferenceOrderLineID", + "type": "int", + "required": true, + "key": false + }, + { + "name": "TransactionDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "TransactionType", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Quantity", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ActualCost", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_UnitMeasure", + "key": [ + "UnitMeasureCode" + ], + "use": [], + "properties": [ + { + "name": "UnitMeasureCode", + "type": "string", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_WorkOrder", + "key": [ + "WorkOrderID" + ], + "use": [], + "properties": [ + { + "name": "WorkOrderID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": false + }, + { + "name": "OrderQty", + "type": "int", + "required": true, + "key": false + }, + { + "name": "StockedQty", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "ScrappedQty", + "type": "int", + "required": true, + "key": false + }, + { + "name": "StartDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "EndDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "DueDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ScrapReasonID", + "type": "reference", + "references": "Production_ScrapReason", + "required": false, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Production_WorkOrderRouting", + "key": [ + "WorkOrderID", + "ProductID", + "OperationSequence" + ], + "use": [], + "properties": [ + { + "name": "WorkOrderID", + "type": "reference", + "references": "Production_WorkOrder", + "required": true, + "key": true + }, + { + "name": "ProductID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "OperationSequence", + "type": "int", + "required": true, + "key": true + }, + { + "name": "LocationID", + "type": "reference", + "references": "Production_Location", + "required": true, + "key": false + }, + { + "name": "ScheduledStartDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ScheduledEndDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ActualStartDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ActualEndDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ActualResourceHrs", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "PlannedCost", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "ActualCost", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Purchasing_ProductVendor", + "key": [ + "ProductID", + "BusinessEntityID" + ], + "use": [], + "properties": [ + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": true + }, + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Purchasing_Vendor", + "required": true, + "key": true + }, + { + "name": "AverageLeadTime", + "type": "int", + "required": true, + "key": false + }, + { + "name": "StandardPrice", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "LastReceiptCost", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "LastReceiptDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "MinOrderQty", + "type": "int", + "required": true, + "key": false + }, + { + "name": "MaxOrderQty", + "type": "int", + "required": true, + "key": false + }, + { + "name": "OnOrderQty", + "type": "int", + "required": false, + "key": false + }, + { + "name": "UnitMeasureCode", + "type": "reference", + "references": "Production_UnitMeasure", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Purchasing_PurchaseOrderDetail", + "key": [ + "PurchaseOrderID", + "PurchaseOrderDetailID" + ], + "use": [], + "properties": [ + { + "name": "PurchaseOrderID", + "type": "reference", + "references": "Purchasing_PurchaseOrderHeader", + "required": true, + "key": true + }, + { + "name": "PurchaseOrderDetailID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "DueDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "OrderQty", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": false + }, + { + "name": "UnitPrice", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "LineTotal", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "ReceivedQty", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "RejectedQty", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "StockedQty", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Purchasing_PurchaseOrderHeader", + "key": [ + "PurchaseOrderID" + ], + "use": [], + "properties": [ + { + "name": "PurchaseOrderID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "RevisionNumber", + "type": "int", + "required": true, + "key": false + }, + { + "name": "Status", + "type": "int", + "required": true, + "key": false + }, + { + "name": "EmployeeID", + "type": "reference", + "references": "HumanResources_Employee", + "required": true, + "key": false + }, + { + "name": "VendorID", + "type": "reference", + "references": "Purchasing_Vendor", + "required": true, + "key": false + }, + { + "name": "ShipMethodID", + "type": "reference", + "references": "Purchasing_ShipMethod", + "required": true, + "key": false + }, + { + "name": "OrderDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ShipDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "SubTotal", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "TaxAmt", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "Freight", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "TotalDue", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Purchasing_ShipMethod", + "key": [ + "ShipMethodID" + ], + "use": [], + "properties": [ + { + "name": "ShipMethodID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ShipBase", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "ShipRate", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Purchasing_Vendor", + "key": [ + "BusinessEntityID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Person_BusinessEntity", + "required": true, + "key": true + }, + { + "name": "AccountNumber", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "CreditRating", + "type": "int", + "required": true, + "key": false + }, + { + "name": "PreferredVendorStatus", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ActiveFlag", + "type": "int", + "required": true, + "key": false + }, + { + "name": "PurchasingWebServiceURL", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_CountryRegionCurrency", + "key": [ + "CountryRegionCode", + "CurrencyCode" + ], + "use": [], + "properties": [ + { + "name": "CountryRegionCode", + "type": "reference", + "references": "Person_CountryRegion", + "required": true, + "key": true + }, + { + "name": "CurrencyCode", + "type": "reference", + "references": "Sales_Currency", + "required": true, + "key": true + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_CreditCard", + "key": [ + "CreditCardID" + ], + "use": [], + "properties": [ + { + "name": "CreditCardID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "CardType", + "type": "string", + "required": true, + "key": false + }, + { + "name": "CardNumber", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ExpMonth", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ExpYear", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_Currency", + "key": [ + "CurrencyCode" + ], + "use": [], + "properties": [ + { + "name": "CurrencyCode", + "type": "string", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_CurrencyRate", + "key": [ + "CurrencyRateID" + ], + "use": [], + "properties": [ + { + "name": "CurrencyRateID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "CurrencyRateDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "FromCurrencyCode", + "type": "reference", + "references": "Sales_Currency", + "required": true, + "key": false + }, + { + "name": "ToCurrencyCode", + "type": "reference", + "references": "Sales_Currency", + "required": true, + "key": false + }, + { + "name": "AverageRate", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "EndOfDayRate", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_Customer", + "key": [ + "CustomerID" + ], + "use": [], + "properties": [ + { + "name": "CustomerID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "PersonID", + "type": "reference", + "references": "Person_Person", + "required": false, + "key": false + }, + { + "name": "StoreID", + "type": "reference", + "references": "Sales_Store", + "required": false, + "key": false + }, + { + "name": "TerritoryID", + "type": "reference", + "references": "Sales_SalesTerritory", + "required": false, + "key": false + }, + { + "name": "AccountNumber", + "type": "string", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_PersonCreditCard", + "key": [ + "BusinessEntityID", + "CreditCardID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Person_Person", + "required": true, + "key": true + }, + { + "name": "CreditCardID", + "type": "reference", + "references": "Sales_CreditCard", + "required": true, + "key": true + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_SalesOrderDetail", + "key": [ + "SalesOrderID", + "SalesOrderDetailID" + ], + "use": [], + "properties": [ + { + "name": "SalesOrderID", + "type": "reference", + "references": "Sales_SalesOrderHeader", + "required": true, + "key": true + }, + { + "name": "SalesOrderDetailID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "CarrierTrackingNumber", + "type": "string", + "required": false, + "key": false + }, + { + "name": "OrderQty", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ProductID", + "type": "reference", + "references": "Sales_SpecialOfferProduct", + "required": true, + "key": false + }, + { + "name": "SpecialOfferID", + "type": "reference", + "references": "Sales_SpecialOfferProduct", + "required": true, + "key": false + }, + { + "name": "UnitPrice", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "UnitPriceDiscount", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "LineTotal", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_SalesOrderHeader", + "key": [ + "SalesOrderID" + ], + "use": [], + "properties": [ + { + "name": "SalesOrderID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "RevisionNumber", + "type": "int", + "required": true, + "key": false + }, + { + "name": "OrderDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "DueDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ShipDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Status", + "type": "int", + "required": true, + "key": false + }, + { + "name": "OnlineOrderFlag", + "type": "int", + "required": true, + "key": false + }, + { + "name": "SalesOrderNumber", + "type": "string", + "required": false, + "key": false + }, + { + "name": "PurchaseOrderNumber", + "type": "string", + "required": false, + "key": false + }, + { + "name": "AccountNumber", + "type": "string", + "required": false, + "key": false + }, + { + "name": "CustomerID", + "type": "reference", + "references": "Sales_Customer", + "required": true, + "key": false + }, + { + "name": "SalesPersonID", + "type": "reference", + "references": "Sales_SalesPerson", + "required": false, + "key": false + }, + { + "name": "TerritoryID", + "type": "reference", + "references": "Sales_SalesTerritory", + "required": false, + "key": false + }, + { + "name": "BillToAddressID", + "type": "reference", + "references": "Person_Address", + "required": true, + "key": false + }, + { + "name": "ShipToAddressID", + "type": "reference", + "references": "Person_Address", + "required": true, + "key": false + }, + { + "name": "ShipMethodID", + "type": "reference", + "references": "Purchasing_ShipMethod", + "required": true, + "key": false + }, + { + "name": "CreditCardID", + "type": "reference", + "references": "Sales_CreditCard", + "required": false, + "key": false + }, + { + "name": "CreditCardApprovalCode", + "type": "string", + "required": false, + "key": false + }, + { + "name": "CurrencyRateID", + "type": "reference", + "references": "Sales_CurrencyRate", + "required": false, + "key": false + }, + { + "name": "SubTotal", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "TaxAmt", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "Freight", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "TotalDue", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "Comment", + "type": "string", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_SalesOrderHeaderSalesReason", + "key": [ + "SalesOrderID", + "SalesReasonID" + ], + "use": [], + "properties": [ + { + "name": "SalesOrderID", + "type": "reference", + "references": "Sales_SalesOrderHeader", + "required": true, + "key": true + }, + { + "name": "SalesReasonID", + "type": "reference", + "references": "Sales_SalesReason", + "required": true, + "key": true + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_SalesPerson", + "key": [ + "BusinessEntityID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "HumanResources_Employee", + "required": true, + "key": true + }, + { + "name": "TerritoryID", + "type": "reference", + "references": "Sales_SalesTerritory", + "required": false, + "key": false + }, + { + "name": "SalesQuota", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "Bonus", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "CommissionPct", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "SalesYTD", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "SalesLastYear", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_SalesPersonQuotaHistory", + "key": [ + "BusinessEntityID", + "QuotaDate" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Sales_SalesPerson", + "required": true, + "key": true + }, + { + "name": "QuotaDate", + "type": "string", + "required": true, + "key": true + }, + { + "name": "SalesQuota", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_SalesReason", + "key": [ + "SalesReasonID" + ], + "use": [], + "properties": [ + { + "name": "SalesReasonID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ReasonType", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_SalesTaxRate", + "key": [ + "SalesTaxRateID" + ], + "use": [], + "properties": [ + { + "name": "SalesTaxRateID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "StateProvinceID", + "type": "reference", + "references": "Person_StateProvince", + "required": true, + "key": false + }, + { + "name": "TaxType", + "type": "int", + "required": true, + "key": false + }, + { + "name": "TaxRate", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_SalesTerritory", + "key": [ + "TerritoryID" + ], + "use": [], + "properties": [ + { + "name": "TerritoryID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "CountryRegionCode", + "type": "reference", + "references": "Person_CountryRegion", + "required": true, + "key": false + }, + { + "name": "Group", + "type": "string", + "required": true, + "key": false + }, + { + "name": "SalesYTD", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "SalesLastYear", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "CostYTD", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "CostLastYear", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_SalesTerritoryHistory", + "key": [ + "BusinessEntityID", + "StartDate", + "TerritoryID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Sales_SalesPerson", + "required": true, + "key": true + }, + { + "name": "TerritoryID", + "type": "reference", + "references": "Sales_SalesTerritory", + "required": true, + "key": true + }, + { + "name": "StartDate", + "type": "string", + "required": true, + "key": true + }, + { + "name": "EndDate", + "type": "string", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_ShoppingCartItem", + "key": [ + "ShoppingCartItemID" + ], + "use": [], + "properties": [ + { + "name": "ShoppingCartItemID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "ShoppingCartID", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Quantity", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": false + }, + { + "name": "DateCreated", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_SpecialOffer", + "key": [ + "SpecialOfferID" + ], + "use": [], + "properties": [ + { + "name": "SpecialOfferID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Description", + "type": "string", + "required": true, + "key": false + }, + { + "name": "DiscountPct", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "Type", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Category", + "type": "string", + "required": true, + "key": false + }, + { + "name": "StartDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "EndDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "MinQty", + "type": "int", + "required": true, + "key": false + }, + { + "name": "MaxQty", + "type": "int", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_SpecialOfferProduct", + "key": [ + "SpecialOfferID", + "ProductID" + ], + "use": [], + "properties": [ + { + "name": "SpecialOfferID", + "type": "reference", + "references": "Sales_SpecialOffer", + "required": true, + "key": true + }, + { + "name": "ProductID", + "type": "reference", + "references": "Production_Product", + "required": true, + "key": true + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Sales_Store", + "key": [ + "BusinessEntityID" + ], + "use": [], + "properties": [ + { + "name": "BusinessEntityID", + "type": "reference", + "references": "Person_BusinessEntity", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "SalesPersonID", + "type": "reference", + "references": "Sales_SalesPerson", + "required": false, + "key": false + }, + { + "name": "Demographics", + "type": "string", + "required": false, + "key": false + }, + { + "name": "rowguid", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "dbo_AWBuildVersion", + "key": [ + "SystemInformationID" + ], + "use": [], + "properties": [ + { + "name": "SystemInformationID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Database_Version", + "type": "string", + "required": true, + "key": false + }, + { + "name": "VersionDate", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ModifiedDate", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "dbo_DatabaseLog", + "key": [ + "DatabaseLogID" + ], + "use": [], + "properties": [ + { + "name": "DatabaseLogID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "PostTime", + "type": "string", + "required": true, + "key": false + }, + { + "name": "DatabaseUser", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Event", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Schema", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Object", + "type": "string", + "required": false, + "key": false + }, + { + "name": "TSQL", + "type": "string", + "required": true, + "key": false + }, + { + "name": "XmlEvent", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "dbo_ErrorLog", + "key": [ + "ErrorLogID" + ], + "use": [], + "properties": [ + { + "name": "ErrorLogID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "ErrorTime", + "type": "string", + "required": true, + "key": false + }, + { + "name": "UserName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ErrorNumber", + "type": "int", + "required": true, + "key": false + }, + { + "name": "ErrorSeverity", + "type": "int", + "required": false, + "key": false + }, + { + "name": "ErrorState", + "type": "int", + "required": false, + "key": false + }, + { + "name": "ErrorProcedure", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ErrorLine", + "type": "int", + "required": false, + "key": false + }, + { + "name": "ErrorMessage", + "type": "string", + "required": true, + "key": false + } + ] + } + ], + "components": [] + }, + { + "id": "chinook", + "title": "Chinook music store", + "description": "The data model of the Chinook sample database: a digital music store with artists, albums, tracks, playlists, customers, employees and invoices, as 11 entities whose names are the upstream table and column names. Anything that stores Chinook, such as a SQLite file, a CSV export or an OpenVaultDB database, is a copy of this model.", + "status": "draft", + "homepage": "https://chinook.demodb.dev/model/", + "address": "modelspec://github.com/demo-db/chinook/chinook", + "repository": "https://github.com/demo-db/chinook", + "commit": "26e852cca00101f53a84ef8ee1f1ae389067f5cf", + "module": "chinook", + "module_id": "github.com/demo-db/chinook/model/chinook", + "module_version": "0.1.0", + "modelspec": "1.0-draft", + "licence": "MIT", + "files": { + "source": "model/chinook.modelspec.hcl", + "json": "model/chinook.modelspec.json" + }, + "maintainers": [ + "trakhimenok" + ], + "entities": [ + { + "name": "Artist", + "key": [ + "ArtistId" + ], + "use": [], + "properties": [ + { + "name": "ArtistId", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "Album", + "key": [ + "AlbumId" + ], + "use": [], + "properties": [ + { + "name": "AlbumId", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Title", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ArtistId", + "type": "reference", + "references": "Artist", + "required": true, + "key": false + } + ] + }, + { + "name": "Track", + "key": [ + "TrackId" + ], + "use": [], + "properties": [ + { + "name": "TrackId", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "AlbumId", + "type": "reference", + "references": "Album", + "required": false, + "key": false + }, + { + "name": "MediaTypeId", + "type": "reference", + "references": "MediaType", + "required": true, + "key": false + }, + { + "name": "GenreId", + "type": "reference", + "references": "Genre", + "required": false, + "key": false + }, + { + "name": "Composer", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Milliseconds", + "type": "int", + "required": true, + "key": false + }, + { + "name": "Bytes", + "type": "int", + "required": false, + "key": false + }, + { + "name": "UnitPrice", + "type": "decimal", + "required": true, + "key": false + } + ] + }, + { + "name": "Genre", + "key": [ + "GenreId" + ], + "use": [], + "properties": [ + { + "name": "GenreId", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "MediaType", + "key": [ + "MediaTypeId" + ], + "use": [], + "properties": [ + { + "name": "MediaTypeId", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "Playlist", + "key": [ + "PlaylistId" + ], + "use": [], + "properties": [ + { + "name": "PlaylistId", + "type": "int", + "required": true, + "key": true + }, + { + "name": "Name", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "PlaylistTrack", + "key": [ + "PlaylistId", + "TrackId" + ], + "use": [], + "properties": [ + { + "name": "PlaylistId", + "type": "reference", + "references": "Playlist", + "required": true, + "key": true + }, + { + "name": "TrackId", + "type": "reference", + "references": "Track", + "required": true, + "key": true + } + ] + }, + { + "name": "Customer", + "key": [ + "CustomerId" + ], + "use": [], + "properties": [ + { + "name": "CustomerId", + "type": "int", + "required": true, + "key": true + }, + { + "name": "FirstName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "LastName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Company", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Address", + "type": "string", + "required": false, + "key": false + }, + { + "name": "City", + "type": "string", + "required": false, + "key": false + }, + { + "name": "State", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Country", + "type": "string", + "required": false, + "key": false + }, + { + "name": "PostalCode", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Phone", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Fax", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Email", + "type": "string", + "required": true, + "key": false + }, + { + "name": "SupportRepId", + "type": "reference", + "references": "Employee", + "required": false, + "key": false + } + ] + }, + { + "name": "Employee", + "key": [ + "EmployeeId" + ], + "use": [], + "properties": [ + { + "name": "EmployeeId", + "type": "int", + "required": true, + "key": true + }, + { + "name": "LastName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "FirstName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Title", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ReportsTo", + "type": "reference", + "references": "Employee", + "required": false, + "key": false + }, + { + "name": "BirthDate", + "type": "datetime", + "required": false, + "key": false + }, + { + "name": "HireDate", + "type": "datetime", + "required": false, + "key": false + }, + { + "name": "Address", + "type": "string", + "required": false, + "key": false + }, + { + "name": "City", + "type": "string", + "required": false, + "key": false + }, + { + "name": "State", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Country", + "type": "string", + "required": false, + "key": false + }, + { + "name": "PostalCode", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Phone", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Fax", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Email", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "Invoice", + "key": [ + "InvoiceId" + ], + "use": [], + "properties": [ + { + "name": "InvoiceId", + "type": "int", + "required": true, + "key": true + }, + { + "name": "CustomerId", + "type": "reference", + "references": "Customer", + "required": true, + "key": false + }, + { + "name": "InvoiceDate", + "type": "datetime", + "required": true, + "key": false + }, + { + "name": "BillingAddress", + "type": "string", + "required": false, + "key": false + }, + { + "name": "BillingCity", + "type": "string", + "required": false, + "key": false + }, + { + "name": "BillingState", + "type": "string", + "required": false, + "key": false + }, + { + "name": "BillingCountry", + "type": "string", + "required": false, + "key": false + }, + { + "name": "BillingPostalCode", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Total", + "type": "decimal", + "required": true, + "key": false + } + ] + }, + { + "name": "InvoiceLine", + "key": [ + "InvoiceLineId" + ], + "use": [], + "properties": [ + { + "name": "InvoiceLineId", + "type": "int", + "required": true, + "key": true + }, + { + "name": "InvoiceId", + "type": "reference", + "references": "Invoice", + "required": true, + "key": false + }, + { + "name": "TrackId", + "type": "reference", + "references": "Track", + "required": true, + "key": false + }, + { + "name": "UnitPrice", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "Quantity", + "type": "int", + "required": true, + "key": false + } + ] + } + ], + "components": [] + }, + { + "id": "employees", + "title": "Employees (browser edition)", + "description": "The ModelSpec for a reproducible browser-sized edition of the MySQL Employees sample. It describes the six native tables and their declared primary and composite keys without inventing identities.", + "status": "draft", + "homepage": "https://demodb.dev/employees/", + "address": "modelspec://github.com/demo-db/employees/employees", + "repository": "https://github.com/demo-db/employees", + "commit": "2455bb327aa8444ec496d36e2fe6dcacfc58dc0a", + "module": "employees", + "module_id": "github.com/demo-db/employees/employees", + "module_version": "0.1.0", + "modelspec": "1.0-draft", + "licence": "CC-BY-SA-3.0", + "files": { + "source": "model/employees.modelspec.hcl", + "json": "model/employees.modelspec.json" + }, + "maintainers": [ + "trakhimenok" + ], + "entities": [ + { + "name": "departments", + "key": [ + "dept_no" + ], + "use": [], + "properties": [ + { + "name": "dept_no", + "type": "string", + "required": true, + "key": true + }, + { + "name": "dept_name", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "dept_emp", + "key": [ + "emp_no", + "dept_no" + ], + "use": [], + "properties": [ + { + "name": "emp_no", + "type": "reference", + "references": "employees", + "required": true, + "key": true + }, + { + "name": "dept_no", + "type": "reference", + "references": "departments", + "required": true, + "key": true + }, + { + "name": "from_date", + "type": "string", + "required": true, + "key": false + }, + { + "name": "to_date", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "dept_manager", + "key": [ + "emp_no", + "dept_no" + ], + "use": [], + "properties": [ + { + "name": "emp_no", + "type": "reference", + "references": "employees", + "required": true, + "key": true + }, + { + "name": "dept_no", + "type": "reference", + "references": "departments", + "required": true, + "key": true + }, + { + "name": "from_date", + "type": "string", + "required": true, + "key": false + }, + { + "name": "to_date", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "employees", + "key": [ + "emp_no" + ], + "use": [], + "properties": [ + { + "name": "emp_no", + "type": "int", + "required": true, + "key": true + }, + { + "name": "birth_date", + "type": "string", + "required": true, + "key": false + }, + { + "name": "first_name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "last_name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "gender", + "type": "string", + "required": true, + "key": false + }, + { + "name": "hire_date", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "salaries", + "key": [ + "emp_no", + "from_date" + ], + "use": [], + "properties": [ + { + "name": "emp_no", + "type": "reference", + "references": "employees", + "required": true, + "key": true + }, + { + "name": "salary", + "type": "int", + "required": true, + "key": false + }, + { + "name": "from_date", + "type": "string", + "required": true, + "key": true + }, + { + "name": "to_date", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "titles", + "key": [ + "emp_no", + "title", + "from_date" + ], + "use": [], + "properties": [ + { + "name": "emp_no", + "type": "reference", + "references": "employees", + "required": true, + "key": true + }, + { + "name": "title", + "type": "string", + "required": true, + "key": true + }, + { + "name": "from_date", + "type": "string", + "required": true, + "key": true + }, + { + "name": "to_date", + "type": "string", + "required": false, + "key": false + } + ] + } + ], + "components": [] + }, + { + "id": "northwind", + "title": "Northwind small-business database", + "description": "The data model of the classic Northwind sample database: customers, employees, orders, products, suppliers and related tables. Its entities and properties describe the pinned upstream SQLite schema; any copy of Northwind uses this model.", + "status": "draft", + "homepage": "https://northwind.demodb.dev/model/", + "address": "modelspec://github.com/demo-db/northwind/northwind", + "repository": "https://github.com/demo-db/northwind", + "commit": "e74726515c3833620b54b7a50d1d273276dd23c1", + "module": "northwind", + "module_id": "github.com/demo-db/northwind/northwind", + "module_version": "0.1.0", + "modelspec": "1.0-draft", + "licence": "MIT", + "files": { + "source": "model/northwind.modelspec.hcl", + "json": "model/northwind.modelspec.json" + }, + "maintainers": [ + "trakhimenok" + ], + "entities": [ + { + "name": "Categories", + "key": [ + "CategoryID" + ], + "use": [], + "properties": [ + { + "name": "CategoryID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "CategoryName", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Description", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Picture", + "type": "document", + "required": false, + "key": false + } + ] + }, + { + "name": "CustomerCustomerDemo", + "key": [ + "CustomerID", + "CustomerTypeID" + ], + "use": [], + "properties": [ + { + "name": "CustomerID", + "type": "reference", + "references": "Customers", + "required": true, + "key": true + }, + { + "name": "CustomerTypeID", + "type": "reference", + "references": "CustomerDemographics", + "required": true, + "key": true + } + ] + }, + { + "name": "CustomerDemographics", + "key": [ + "CustomerTypeID" + ], + "use": [], + "properties": [ + { + "name": "CustomerTypeID", + "type": "string", + "required": true, + "key": true + }, + { + "name": "CustomerDesc", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "Customers", + "key": [ + "CustomerID" + ], + "use": [], + "properties": [ + { + "name": "CustomerID", + "type": "string", + "required": true, + "key": true + }, + { + "name": "CompanyName", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ContactName", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ContactTitle", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Address", + "type": "string", + "required": false, + "key": false + }, + { + "name": "City", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Region", + "type": "string", + "required": false, + "key": false + }, + { + "name": "PostalCode", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Country", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Phone", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Fax", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "EmployeeTerritories", + "key": [ + "EmployeeID", + "TerritoryID" + ], + "use": [], + "properties": [ + { + "name": "EmployeeID", + "type": "reference", + "references": "Employees", + "required": true, + "key": true + }, + { + "name": "TerritoryID", + "type": "reference", + "references": "Territories", + "required": true, + "key": true + } + ] + }, + { + "name": "Employees", + "key": [ + "EmployeeID" + ], + "use": [], + "properties": [ + { + "name": "EmployeeID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "LastName", + "type": "string", + "required": false, + "key": false + }, + { + "name": "FirstName", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Title", + "type": "string", + "required": false, + "key": false + }, + { + "name": "TitleOfCourtesy", + "type": "string", + "required": false, + "key": false + }, + { + "name": "BirthDate", + "type": "date", + "required": false, + "key": false + }, + { + "name": "HireDate", + "type": "date", + "required": false, + "key": false + }, + { + "name": "Address", + "type": "string", + "required": false, + "key": false + }, + { + "name": "City", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Region", + "type": "string", + "required": false, + "key": false + }, + { + "name": "PostalCode", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Country", + "type": "string", + "required": false, + "key": false + }, + { + "name": "HomePhone", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Extension", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Photo", + "type": "document", + "required": false, + "key": false + }, + { + "name": "Notes", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ReportsTo", + "type": "reference", + "references": "Employees", + "required": false, + "key": false + }, + { + "name": "PhotoPath", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "OrderDetails", + "key": [ + "OrderID", + "ProductID" + ], + "use": [], + "properties": [ + { + "name": "OrderID", + "type": "reference", + "references": "Orders", + "required": true, + "key": true + }, + { + "name": "ProductID", + "type": "reference", + "references": "Products", + "required": true, + "key": true + }, + { + "name": "UnitPrice", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "Quantity", + "type": "int", + "required": true, + "key": false + }, + { + "name": "Discount", + "type": "decimal", + "required": true, + "key": false + } + ] + }, + { + "name": "Orders", + "key": [ + "OrderID" + ], + "use": [], + "properties": [ + { + "name": "OrderID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "CustomerID", + "type": "reference", + "references": "Customers", + "required": false, + "key": false + }, + { + "name": "EmployeeID", + "type": "reference", + "references": "Employees", + "required": false, + "key": false + }, + { + "name": "OrderDate", + "type": "datetime", + "required": false, + "key": false + }, + { + "name": "RequiredDate", + "type": "datetime", + "required": false, + "key": false + }, + { + "name": "ShippedDate", + "type": "datetime", + "required": false, + "key": false + }, + { + "name": "ShipVia", + "type": "reference", + "references": "Shippers", + "required": false, + "key": false + }, + { + "name": "Freight", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "ShipName", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ShipAddress", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ShipCity", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ShipRegion", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ShipPostalCode", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ShipCountry", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "Products", + "key": [ + "ProductID" + ], + "use": [], + "properties": [ + { + "name": "ProductID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "ProductName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "SupplierID", + "type": "reference", + "references": "Suppliers", + "required": false, + "key": false + }, + { + "name": "CategoryID", + "type": "reference", + "references": "Categories", + "required": false, + "key": false + }, + { + "name": "QuantityPerUnit", + "type": "string", + "required": false, + "key": false + }, + { + "name": "UnitPrice", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "UnitsInStock", + "type": "int", + "required": false, + "key": false + }, + { + "name": "UnitsOnOrder", + "type": "int", + "required": false, + "key": false + }, + { + "name": "ReorderLevel", + "type": "int", + "required": false, + "key": false + }, + { + "name": "Discontinued", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Regions", + "key": [ + "RegionID" + ], + "use": [], + "properties": [ + { + "name": "RegionID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "RegionDescription", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "Shippers", + "key": [ + "ShipperID" + ], + "use": [], + "properties": [ + { + "name": "ShipperID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "CompanyName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "Phone", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "Suppliers", + "key": [ + "SupplierID" + ], + "use": [], + "properties": [ + { + "name": "SupplierID", + "type": "int", + "required": true, + "key": true + }, + { + "name": "CompanyName", + "type": "string", + "required": true, + "key": false + }, + { + "name": "ContactName", + "type": "string", + "required": false, + "key": false + }, + { + "name": "ContactTitle", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Address", + "type": "string", + "required": false, + "key": false + }, + { + "name": "City", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Region", + "type": "string", + "required": false, + "key": false + }, + { + "name": "PostalCode", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Country", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Phone", + "type": "string", + "required": false, + "key": false + }, + { + "name": "Fax", + "type": "string", + "required": false, + "key": false + }, + { + "name": "HomePage", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "Territories", + "key": [ + "TerritoryID" + ], + "use": [], + "properties": [ + { + "name": "TerritoryID", + "type": "string", + "required": true, + "key": true + }, + { + "name": "TerritoryDescription", + "type": "string", + "required": true, + "key": false + }, + { + "name": "RegionID", + "type": "reference", + "references": "Regions", + "required": true, + "key": false + } + ] + } + ], + "components": [] + }, + { + "id": "pubs", + "title": "Pubs publishing database", + "description": "The classic Microsoft Pubs sample model covers authors, publications, publishers, stores, sales and employees. It preserves all 11 native tables, including keyless discounts and royalty schedules without fabricated identity.", + "status": "draft", + "homepage": "https://pubs.demodb.dev/model/", + "address": "modelspec://github.com/demo-db/pubs/pubs", + "repository": "https://github.com/demo-db/pubs", + "commit": "6c06c5c7395b03ff1a02c2b1a21485add3e1b65b", + "module": "pubs", + "module_id": "github.com/demo-db/pubs/pubs", + "module_version": "0.1.0", + "modelspec": "1.0-draft", + "licence": "MIT", + "files": { + "source": "model/pubs.modelspec.hcl", + "json": "model/pubs.modelspec.json" + }, + "maintainers": [ + "trakhimenok" + ], + "entities": [ + { + "name": "authors", + "key": [ + "au_id" + ], + "use": [], + "properties": [ + { + "name": "au_id", + "type": "string", + "required": true, + "key": true + }, + { + "name": "au_lname", + "type": "string", + "required": true, + "key": false + }, + { + "name": "au_fname", + "type": "string", + "required": true, + "key": false + }, + { + "name": "phone", + "type": "string", + "required": true, + "key": false + }, + { + "name": "address", + "type": "string", + "required": false, + "key": false + }, + { + "name": "city", + "type": "string", + "required": false, + "key": false + }, + { + "name": "state", + "type": "string", + "required": false, + "key": false + }, + { + "name": "zip", + "type": "string", + "required": false, + "key": false + }, + { + "name": "contract", + "type": "int", + "required": true, + "key": false + } + ] + }, + { + "name": "discounts", + "key": [], + "use": [], + "properties": [ + { + "name": "discounttype", + "type": "string", + "required": true, + "key": false + }, + { + "name": "stor_id", + "type": "reference", + "references": "stores", + "required": false, + "key": false + }, + { + "name": "lowqty", + "type": "int", + "required": false, + "key": false + }, + { + "name": "highqty", + "type": "int", + "required": false, + "key": false + }, + { + "name": "discount", + "type": "decimal", + "required": true, + "key": false + } + ] + }, + { + "name": "employee", + "key": [ + "emp_id" + ], + "use": [], + "properties": [ + { + "name": "emp_id", + "type": "string", + "required": true, + "key": true + }, + { + "name": "fname", + "type": "string", + "required": true, + "key": false + }, + { + "name": "minit", + "type": "string", + "required": false, + "key": false + }, + { + "name": "lname", + "type": "string", + "required": true, + "key": false + }, + { + "name": "job_id", + "type": "reference", + "references": "jobs", + "required": true, + "key": false + }, + { + "name": "job_lvl", + "type": "int", + "required": false, + "key": false + }, + { + "name": "pub_id", + "type": "reference", + "references": "publishers", + "required": true, + "key": false + }, + { + "name": "hire_date", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "jobs", + "key": [ + "job_id" + ], + "use": [], + "properties": [ + { + "name": "job_id", + "type": "int", + "required": true, + "key": true + }, + { + "name": "job_desc", + "type": "string", + "required": true, + "key": false + }, + { + "name": "min_lvl", + "type": "int", + "required": true, + "key": false + }, + { + "name": "max_lvl", + "type": "int", + "required": true, + "key": false + } + ] + }, + { + "name": "pub_info", + "key": [ + "pub_id" + ], + "use": [], + "properties": [ + { + "name": "pub_id", + "type": "reference", + "references": "publishers", + "required": true, + "key": true + }, + { + "name": "logo", + "type": "document", + "required": false, + "key": false + }, + { + "name": "pr_info", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "publishers", + "key": [ + "pub_id" + ], + "use": [], + "properties": [ + { + "name": "pub_id", + "type": "string", + "required": true, + "key": true + }, + { + "name": "pub_name", + "type": "string", + "required": false, + "key": false + }, + { + "name": "city", + "type": "string", + "required": false, + "key": false + }, + { + "name": "state", + "type": "string", + "required": false, + "key": false + }, + { + "name": "country", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "roysched", + "key": [], + "use": [], + "properties": [ + { + "name": "title_id", + "type": "reference", + "references": "titles", + "required": false, + "key": false + }, + { + "name": "lorange", + "type": "int", + "required": false, + "key": false + }, + { + "name": "hirange", + "type": "int", + "required": false, + "key": false + }, + { + "name": "royalty", + "type": "int", + "required": false, + "key": false + } + ] + }, + { + "name": "sales", + "key": [ + "stor_id", + "ord_num", + "title_id" + ], + "use": [], + "properties": [ + { + "name": "stor_id", + "type": "reference", + "references": "stores", + "required": true, + "key": true + }, + { + "name": "ord_num", + "type": "string", + "required": true, + "key": true + }, + { + "name": "ord_date", + "type": "string", + "required": true, + "key": false + }, + { + "name": "qty", + "type": "int", + "required": true, + "key": false + }, + { + "name": "payterms", + "type": "string", + "required": true, + "key": false + }, + { + "name": "title_id", + "type": "reference", + "references": "titles", + "required": true, + "key": true + } + ] + }, + { + "name": "stores", + "key": [ + "stor_id" + ], + "use": [], + "properties": [ + { + "name": "stor_id", + "type": "string", + "required": true, + "key": true + }, + { + "name": "stor_name", + "type": "string", + "required": false, + "key": false + }, + { + "name": "stor_address", + "type": "string", + "required": false, + "key": false + }, + { + "name": "city", + "type": "string", + "required": false, + "key": false + }, + { + "name": "state", + "type": "string", + "required": false, + "key": false + }, + { + "name": "zip", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "titleauthor", + "key": [ + "au_id", + "title_id" + ], + "use": [], + "properties": [ + { + "name": "au_id", + "type": "reference", + "references": "authors", + "required": true, + "key": true + }, + { + "name": "title_id", + "type": "reference", + "references": "titles", + "required": true, + "key": true + }, + { + "name": "au_ord", + "type": "int", + "required": false, + "key": false + }, + { + "name": "royaltyper", + "type": "int", + "required": false, + "key": false + } + ] + }, + { + "name": "titles", + "key": [ + "title_id" + ], + "use": [], + "properties": [ + { + "name": "title_id", + "type": "string", + "required": true, + "key": true + }, + { + "name": "title", + "type": "string", + "required": true, + "key": false + }, + { + "name": "type", + "type": "string", + "required": true, + "key": false + }, + { + "name": "pub_id", + "type": "reference", + "references": "publishers", + "required": false, + "key": false + }, + { + "name": "price", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "advance", + "type": "decimal", + "required": false, + "key": false + }, + { + "name": "royalty", + "type": "int", + "required": false, + "key": false + }, + { + "name": "ytd_sales", + "type": "int", + "required": false, + "key": false + }, + { + "name": "notes", + "type": "string", + "required": false, + "key": false + }, + { + "name": "pubdate", + "type": "string", + "required": true, + "key": false + } + ] + } + ], + "components": [] + }, + { + "id": "sakila", + "title": "Sakila video rental sample", + "description": "The MySQL Sakila sample model for films, actors, customers, rentals, payments, stores and staff, represented by its 16 native table entities.", + "status": "draft", + "homepage": "https://sakila.demodb.dev/model/", + "address": "modelspec://github.com/demo-db/sakila/sakila", + "repository": "https://github.com/demo-db/sakila", + "commit": "6567d30aec1592fe0917934a8bbe74ff70b04b01", + "module": "sakila", + "module_id": "github.com/demo-db/sakila/sakila", + "module_version": "0.1.0", + "modelspec": "1.0-draft", + "licence": "BSD-3-Clause", + "files": { + "source": "model/sakila.modelspec.hcl", + "json": "model/sakila.modelspec.json" + }, + "maintainers": [ + "trakhimenok" + ], + "entities": [ + { + "name": "actor", + "key": [ + "actor_id" + ], + "use": [], + "properties": [ + { + "name": "actor_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "first_name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "last_name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "address", + "key": [ + "address_id" + ], + "use": [], + "properties": [ + { + "name": "address_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "address", + "type": "string", + "required": true, + "key": false + }, + { + "name": "address2", + "type": "string", + "required": false, + "key": false + }, + { + "name": "district", + "type": "string", + "required": true, + "key": false + }, + { + "name": "city_id", + "type": "reference", + "references": "city", + "required": true, + "key": false + }, + { + "name": "postal_code", + "type": "string", + "required": false, + "key": false + }, + { + "name": "phone", + "type": "string", + "required": true, + "key": false + }, + { + "name": "location", + "type": "document", + "required": true, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "category", + "key": [ + "category_id" + ], + "use": [], + "properties": [ + { + "name": "category_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "city", + "key": [ + "city_id" + ], + "use": [], + "properties": [ + { + "name": "city_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "city", + "type": "string", + "required": true, + "key": false + }, + { + "name": "country_id", + "type": "reference", + "references": "country", + "required": true, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "country", + "key": [ + "country_id" + ], + "use": [], + "properties": [ + { + "name": "country_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "country", + "type": "string", + "required": true, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "customer", + "key": [ + "customer_id" + ], + "use": [], + "properties": [ + { + "name": "customer_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "store_id", + "type": "reference", + "references": "store", + "required": true, + "key": false + }, + { + "name": "first_name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "last_name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "email", + "type": "string", + "required": false, + "key": false + }, + { + "name": "address_id", + "type": "reference", + "references": "address", + "required": true, + "key": false + }, + { + "name": "active", + "type": "int", + "required": true, + "key": false + }, + { + "name": "create_date", + "type": "string", + "required": true, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "film", + "key": [ + "film_id" + ], + "use": [], + "properties": [ + { + "name": "film_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "title", + "type": "string", + "required": true, + "key": false + }, + { + "name": "description", + "type": "string", + "required": false, + "key": false + }, + { + "name": "release_year", + "type": "string", + "required": false, + "key": false + }, + { + "name": "language_id", + "type": "reference", + "references": "language", + "required": true, + "key": false + }, + { + "name": "original_language_id", + "type": "reference", + "references": "language", + "required": false, + "key": false + }, + { + "name": "rental_duration", + "type": "int", + "required": true, + "key": false + }, + { + "name": "rental_rate", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "length", + "type": "int", + "required": false, + "key": false + }, + { + "name": "replacement_cost", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "rating", + "type": "string", + "required": false, + "key": false + }, + { + "name": "special_features", + "type": "string", + "required": false, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "film_actor", + "key": [ + "actor_id", + "film_id" + ], + "use": [], + "properties": [ + { + "name": "actor_id", + "type": "reference", + "references": "actor", + "required": true, + "key": true + }, + { + "name": "film_id", + "type": "reference", + "references": "film", + "required": true, + "key": true + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "film_category", + "key": [ + "film_id", + "category_id" + ], + "use": [], + "properties": [ + { + "name": "film_id", + "type": "reference", + "references": "film", + "required": true, + "key": true + }, + { + "name": "category_id", + "type": "reference", + "references": "category", + "required": true, + "key": true + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "film_text", + "key": [ + "film_id" + ], + "use": [], + "properties": [ + { + "name": "film_id", + "type": "int", + "required": true, + "key": true + }, + { + "name": "title", + "type": "string", + "required": true, + "key": false + }, + { + "name": "description", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "inventory", + "key": [ + "inventory_id" + ], + "use": [], + "properties": [ + { + "name": "inventory_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "film_id", + "type": "reference", + "references": "film", + "required": true, + "key": false + }, + { + "name": "store_id", + "type": "reference", + "references": "store", + "required": true, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "language", + "key": [ + "language_id" + ], + "use": [], + "properties": [ + { + "name": "language_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "payment", + "key": [ + "payment_id" + ], + "use": [], + "properties": [ + { + "name": "payment_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "customer_id", + "type": "reference", + "references": "customer", + "required": true, + "key": false + }, + { + "name": "staff_id", + "type": "reference", + "references": "staff", + "required": true, + "key": false + }, + { + "name": "rental_id", + "type": "reference", + "references": "rental", + "required": false, + "key": false + }, + { + "name": "amount", + "type": "decimal", + "required": true, + "key": false + }, + { + "name": "payment_date", + "type": "string", + "required": true, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": false, + "key": false + } + ] + }, + { + "name": "rental", + "key": [ + "rental_id" + ], + "use": [], + "properties": [ + { + "name": "rental_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "rental_date", + "type": "string", + "required": true, + "key": false + }, + { + "name": "inventory_id", + "type": "reference", + "references": "inventory", + "required": true, + "key": false + }, + { + "name": "customer_id", + "type": "reference", + "references": "customer", + "required": true, + "key": false + }, + { + "name": "return_date", + "type": "string", + "required": false, + "key": false + }, + { + "name": "staff_id", + "type": "reference", + "references": "staff", + "required": true, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "staff", + "key": [ + "staff_id" + ], + "use": [], + "properties": [ + { + "name": "staff_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "first_name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "last_name", + "type": "string", + "required": true, + "key": false + }, + { + "name": "address_id", + "type": "reference", + "references": "address", + "required": true, + "key": false + }, + { + "name": "picture", + "type": "document", + "required": false, + "key": false + }, + { + "name": "email", + "type": "string", + "required": false, + "key": false + }, + { + "name": "store_id", + "type": "reference", + "references": "store", + "required": true, + "key": false + }, + { + "name": "active", + "type": "int", + "required": true, + "key": false + }, + { + "name": "username", + "type": "string", + "required": true, + "key": false + }, + { + "name": "password", + "type": "string", + "required": false, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + }, + { + "name": "store", + "key": [ + "store_id" + ], + "use": [], + "properties": [ + { + "name": "store_id", + "type": "int", + "required": false, + "key": true + }, + { + "name": "manager_staff_id", + "type": "reference", + "references": "staff", + "required": true, + "key": false + }, + { + "name": "address_id", + "type": "reference", + "references": "address", + "required": true, + "key": false + }, + { + "name": "last_update", + "type": "string", + "required": true, + "key": false + } + ] + } + ], + "components": [] + } + ] +} diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/snapshot.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/snapshot.json new file mode 100644 index 00000000..bc268d28 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/snapshot.json @@ -0,0 +1,17 @@ +{ + "source_release": "test-only", + "generator": { + "repository": "https://github.com/example/provider", + "revision": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + }, + "artifacts": [ + { + "path": "bridge.json", + "sha256": "fd225cf26ca051471e983c770e4af49edd8bdce8d429729e53b4722d677a5bc9" + }, + { + "path": "keys.json", + "sha256": "5aa000ef1e2c824aa4dd6315256e5e181ba0a334ed94b43aca252d17434f222e" + } + ] +} diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/source.modelspec.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/source.modelspec.json new file mode 100644 index 00000000..a0a6a02a --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/source.modelspec.json @@ -0,0 +1,18 @@ +{ + "modelspec": "1.0-draft", + "module": { + "name": "sample" + }, + "entities": { + "Customer": { + "properties": { + "Country": { + "type": "string" + }, + "BillingCountry": { + "type": "string" + } + } + } + } +} diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/target.meaning.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/target.meaning.json new file mode 100644 index 00000000..af1523e3 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/target.meaning.json @@ -0,0 +1,16 @@ +{ + "format": "meaning/draft-1", + "concepts": [ + { + "id": "native-country", + "extends": "meaning://github.com/example/source/country?ref=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "bindings": [ + { + "model": "modelspec:///geo.Countries", + "property": "iso", + "role": "identifier" + } + ] + } + ] +} diff --git a/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/target.modelspec.json b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/target.modelspec.json new file mode 100644 index 00000000..ab0ca047 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/target.modelspec.json @@ -0,0 +1,28 @@ +{ + "modelspec": "1.0-draft", + "module": { + "name": "geo" + }, + "entities": { + "Countries": { + "properties": { + "iso": { + "type": "string" + } + } + }, + "CustomerCountries": { + "properties": { + "raw_label": { + "type": "string" + }, + "target_key": { + "type": "string" + }, + "serving_id": { + "type": "string" + } + } + } + } +} diff --git a/libs/datatug/main/src/lib/queries/public-data/README.md b/libs/datatug/main/src/lib/queries/public-data/README.md new file mode 100644 index 00000000..bf48214f --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/README.md @@ -0,0 +1,19 @@ +# Public Data Fabric W1 preparation + +The project `public-data` route reads immutable canonical Directory, ModelSpec and MeaningGraph indexes, loads existing configured project sources and resolves an exact source field through explicit canonical provider and recordset-to-model mappings, inspects a scoped representation attachment and saves its pins through the existing query service. A metadata cache is disposable and keyed by repository, revision, path and checksum. Discovery admits at most 96 references, depth 8 and an aggregate 2MiB, rejects cycles, redirects, duplicate JSON keys and aliases of consumed structural fields, wrong hashes and missing scoped model/binding properties. + +The reviewed schema1 consumer checks provider-local references against the outer immutable Directory commit and external references against exact registry pins. It checks binding/property scope, generator provenance, bridge/key artifact hashes, native namespace and bridge collisions. These checks establish byte/shape compatibility, never semantic acceptance. Schema copied from `openvaultdb/ovdb` main `aec07d17be1e074210acefddb858363ce7777704`; native schema2 landed separately at OVDB commit `d166a1f8a38a644de9d11f381ad406dad3f89b36` and is not implemented by this consumer. + +Production discovery and all saved PublicDataScenario execution remain closed pending reviewed canonical Directory/provider companion publication and fresh admission. Serialized `eligible` flags cannot authorize execution. The ordinary explicit bounded executor can be tested independently; it does not certify a semantic source mapping. No fixture switch or scenario catalogue enables production discovery. + +The bounded transport first selects at most 1000 user rows, then queries only their exact keys through the installed DALgo `In` dialect. It caps distinct keys at 100 places or 50 ROR IDs, results at 5000, aggregate response bytes at 5MiB and the run at 10 seconds. It refuses undeclared origins, redirects, ignored filters, expired/changing snapshots and unsafe pagination, releases snapshots and supports cancellation. Worker termination enforces the wall deadline even during synchronous work. No full static dataset fallback is available in this mode. Results keep raw values, distinct NULL/empty/invalid/unmatched/ambiguous counts and affiliation grain; an explicit ROR:URL syntax validator prevents requests for malformed IDs while keeping raw invalid inputs; checksum-valid but absent IDs remain unmatched. This syntax support grants no semantic eligibility. Native ROR format/checksum validity is separate from membership, with inactive/withdrawn snapshot status retained. + +The primary route selects configured source/table/field metadata without requiring repository hashes or contract JSON. Explicit connection identity or exact upstream repository/revision must resolve to one canonical provider; API bases, host similarities and table names cannot infer provider identity. Recordset modelEntity, physical schema and string field coordinates must match the pinned registry and verified scoped attachment. Missing provenance stays visibly unavailable. Existing HTTPS catalog keys can list configured flat tables without fetching their rows. A deliberate preview button reuses the existing table view; advanced descriptor JSON remains secondary diagnostics. The route connects and explains without querying data. Run is explicit. Result pagination reads already bounded stored output. Save and reopen retain source, canonical index, model, meaning, snapshot, attachment, rights and decision pins. Reopening makes no lookup; changed canonical pins require visible acknowledgement, and publication admission remains required. + +Tests use synthetic copies of the reviewed helper fixture, a bounded runtime fixture and service doubles for the full template journey. Byte-pinned actual Directory and ModelSpec indexes separately test configured source service integration, wrong origin/revision/schema/recordset mappings and checksum preservation. They cover configured source/field selection without hash typing, connect, explanation, inspect, explicit run, exceptions, save and reopen; production-unavailable behavior is separate. Existing query-page tests verify reopened pins and reject forged eligibility. These tests prove fixture mechanics, not deployed OVDB filter support or a real published source journey. + +The independent [ROR fixture decision](https://github.com/datatug/datatug/blob/17263dbacabdfe95e53fc3c6980177416bdab941/spec/research/public-data-fabric/evidence/w1-ror-user-decision.json) accepts only the exact hypothetical affiliation field at apps commit `e7362033ec79c6663d7dbe0483b62fab01f7b9cd`. Its evidence SHA256 is `a4caa4b6461230a3fa2ae8548121344cb286cfb98a9e19761a207c9f74479595`. The original fixture bytes remain unchanged. It does not create a native ROR field in any of the six demos, prove affiliation truth or grant production eligibility. + +Remaining publication handoff: publish reviewed scoped Directory attachments; pin provider ModelSpec/MeaningGraph registrations; publish source-faithful native dataset/snapshot/provenance and discriminated native execution contract; expose bounded live keyed filtering with pagination, cancel and byte/deadline semantics; independently validate the actual browser journey. F1–F6/live acceptance and B1 remain open. + +Remaining app implementation is separate from publication: replace unconditional discovery/saved-execution closure with fresh verified canonical admission; implement schema2/native reader and exact accepted ROR decision scope; supply source-faithful native ID/location execution; complete source release/freshness/coverage/confidence/licence explanation and stale last-known-good handling; prove canonical mutation changes admitted suggestions after cache invalidation; and revalidate saved plans and acknowledged re-execution through the actual query store while retaining original results/provenance. No receipt-shape normalization is authorized for unsupported native snapshots. Real configured field, provider orphan/location/denominator, bounded runtime pagination/cancel/deadline/byte behavior and no-action browser network evidence remain acceptance gaps. The current fixtures and unavailable-state browser receipt close none of those gaps. diff --git a/libs/datatug/main/src/lib/queries/public-data/bounded-federation.spec.ts b/libs/datatug/main/src/lib/queries/public-data/bounded-federation.spec.ts new file mode 100644 index 00000000..a855daa2 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/bounded-federation.spec.ts @@ -0,0 +1,602 @@ +import 'fake-indexeddb/auto'; +import { describe, expect, it, vi } from 'vitest'; +import { parseDTQL } from '@dalgo/core'; +import { QueryType, type IQueryDef } from '../../models/definition/query-def'; +import { runFederatedQuery } from '../federated-query-executor'; +import { + createBoundedFederationFetch, + validateBounds, + boundedResponseText, + type BoundedFederation, + type BoundedRecord, +} from './bounded-federation'; +import { + publicDataExceptions, + validNativeRorUrl, +} from './public-data-scenario'; + +const base = 'https://demodb.dev/ovdb'; +export const fixtureBounds = (): BoundedFederation => ({ + userRows: 1000, + userOffset: 0, + identifierKind: 'place', + identifierLimit: 100, + resultRows: 5000, + bytes: 5 * 1024 * 1024, + timeoutMs: 10000, + sources: [ + { database: 'user', name: 'Customer', keyField: 'Country' }, + { + database: 'geo', + name: 'Countries', + keyField: 'raw', + parent: { database: 'user', name: 'Customer', field: 'Country' }, + }, + ], +}); +const row = (key: string, data: Record): BoundedRecord => ({ + key, + data, +}); +const validButHypotheticalRor = (value: number): string => { + const alphabet = '0123456789abcdefghjkmnpqrstvwxyz'; + let rest = value; + let token = ''; + for (let i = 0; i < 7; i++) { + token = alphabet[rest % 32] + token; + rest = Math.floor(rest / 32); + } + return `https://ror.org/${token}${String(98 - ((value * 100) % 97)).padStart(2, '0')}`; +}; +const response = (body: unknown): Response => + new Response(JSON.stringify(body), { + headers: { 'Content-Type': 'application/json' }, + }); +function serve( + source: readonly BoundedRecord[], + references: readonly BoundedRecord[], + ignoreFilter = false, +) { + const calls: { + name: string; + where?: { + left: { field: string }; + right: { values: string[] }; + op: string; + }; + limit: number; + offset?: number; + close: boolean; + }[] = []; + const http: typeof fetch = vi.fn(async (_input, init) => { + expect(init?.redirect).toBe('error'); + const query = JSON.parse(String(init?.body)) as { + from: { name: string }; + where?: { + left: { field: string }; + right: { values: string[] }; + op: string; + }; + limit: number; + offset?: number; + }; + const headers = new Headers(init?.headers); + const close = headers.get('OVDB-Page-Close') === 'true'; + calls.push({ + name: query.from.name, + where: query.where, + limit: query.limit, + offset: query.offset, + close, + }); + if (close) return new Response(null, { status: 204 }); + let rows = query.from.name === 'Customer' ? [...source] : [...references]; + if (query.where && !ignoreFilter) + rows = rows.filter((row) => + query.where?.right.values.includes( + String(row.data[query.where.left.field]), + ), + ); + rows = rows.slice(query.offset ?? 0, (query.offset ?? 0) + query.limit); + const start = Number(headers.get('OVDB-Page-Token') ?? 0); + const end = Math.min( + start + Number(headers.get('OVDB-Page-Size')), + rows.length, + ); + return response({ + records: rows.slice(start, end), + snapshotToken: `snapshot-${query.from.name}`, + ...(end < rows.length ? { nextPageToken: String(end) } : {}), + }); + }); + return { http, calls }; +} +function definition(bounds = fixtureBounds()): IQueryDef { + return { + id: 'bounded', + request: { + queryType: QueryType.DTQL, + text: JSON.stringify({ + from: { + database: 'user', + name: 'Customer', + alias: 'u', + joins: [ + { + type: 'left', + from: { database: 'geo', name: 'Countries', alias: 'p' }, + on: [ + { + left: { source: 'u', field: 'Country' }, + op: '==', + right: { source: 'p', field: 'raw' }, + }, + ], + }, + ], + }, + }), + }, + federation: { + ovdbBaseUrl: base, + bounds, + tables: [ + { database: 'user', name: 'Customer', fields: ['Country', 'id'] }, + { database: 'geo', name: 'Countries', fields: ['raw', 'iso'] }, + ], + }, + } as IQueryDef; +} + +describe('bounded public-data federation preparation (fixture requests, not deployed acceptance)', () => { + it('uses the installed DALgo In dialect and retains byte-exact literals', () => { + const query = parseDTQL( + { + from: { name: 'Countries' }, + where: { + left: { field: 'raw' }, + op: 'In', + right: { values: [' USA', 'USA', 'usa'] }, + }, + limit: 5000, + }, + { tables: [{ name: 'Countries', fields: ['raw'] }] }, + { maxLimit: 5000 }, + ); + expect(query.filters).toHaveLength(1); + expect(() => + parseDTQL( + { + from: { name: 'Countries' }, + where: { + left: { field: 'wrong' }, + op: 'In', + right: { values: ['USA'] }, + }, + }, + { tables: [{ name: 'Countries', fields: ['raw'] }] }, + ), + ).toThrow(); + }); + it('does no network work before action, loads the driver first even when DALgo asks for the reference first, and filters native keys', async () => { + const runtime = serve( + [ + row('1', { Country: 'USA' }), + row('2', { Country: ' USA' }), + row('3', { Country: null }), + row('4', { Country: '' }), + ], + [ + row('us', { raw: 'USA', iso: 'US' }), + row('ca', { raw: 'Canada', iso: 'CA' }), + ], + ); + const transport = createBoundedFederationFetch( + base, + fixtureBounds(), + runtime.http, + new AbortController().signal, + ); + expect(runtime.calls).toEqual([]); + const page = await transport.fetch(`${base}/v1/databases/geo/dtql`, { + method: 'POST', + body: JSON.stringify({ from: { name: 'Countries' } }), + headers: { 'OVDB-Page-Size': '100' }, + }); + expect((await page.json()).records).toHaveLength(1); + expect( + runtime.calls.filter((call) => !call.close).map((call) => call.name), + ).toEqual(['Customer', 'Countries']); + expect( + runtime.calls.find((call) => call.name === 'Countries')?.where?.right + .values, + ).toEqual(['USA', ' USA']); + expect(runtime.calls.filter((call) => call.close)).toHaveLength(2); + }); + it('executes actual installed browser DALgo against bounded fixture pages and keeps NULL/empty/unmatched denominators', async () => { + const runtime = serve( + [ + row('1', { id: '1', Country: 'USA' }), + row('2', { id: '2', Country: null }), + row('3', { id: '3', Country: '' }), + row('4', { id: '4', Country: 'usa' }), + ], + [row('us', { raw: 'USA', iso: 'US' })], + ); + const result = await runFederatedQuery( + definition(), + undefined, + '', + undefined, + undefined, + 'full', + undefined, + undefined, + { fetch: runtime.http }, + ); + expect(result.recordset.rows).toHaveLength(4); + expect(result.publicDataExceptions).toMatchObject({ + denominator: 4, + nonNull: 3, + null: 1, + empty: 1, + unmatched: 1, + matched: 1, + }); + expect(result.publicDataExceptions?.details[3]).toMatchObject({ + raw: 'usa', + status: 'unmatched', + }); + expect(result.publicDataBytes).toBeGreaterThan(0); + }); + it('rejects a runtime that ignores the native key filter and never falls back to a static scan', async () => { + const runtime = serve( + [row('1', { Country: 'USA' })], + [row('ca', { raw: 'Canada' })], + true, + ); + await expect( + runFederatedQuery( + definition(), + undefined, + '', + undefined, + undefined, + 'full', + undefined, + undefined, + { fetch: runtime.http }, + ), + ).rejects.toThrow(/did not enforce/); + expect(runtime.calls.filter((call) => !call.close)).toHaveLength(2); + }); + it('accepts the exact 1000-row/100-key boundary and rejects 101 distinct place or 51 ROR IDs', async () => { + const rows = Array.from({ length: 1000 }, (_, i) => + row(String(i), { Country: String(i % 100) }), + ); + const runtime = serve(rows, []); + await runFederatedQuery( + definition(), + undefined, + '', + undefined, + undefined, + 'full', + undefined, + undefined, + { fetch: runtime.http }, + ); + expect( + runtime.calls.filter((call) => call.name === 'Customer' && !call.close), + ).toHaveLength(10); + for (const [count, kind, cap] of [ + [101, 'place', 100], + [51, 'ror', 50], + ] as const) { + const tooMany = serve( + Array.from({ length: count }, (_, i) => + row(String(i), { + Country: kind === 'ror' ? validButHypotheticalRor(i) : String(i), + }), + ), + [], + ); + await expect( + runFederatedQuery( + definition({ + ...fixtureBounds(), + identifierKind: kind, + ...(kind === 'ror' ? { nativeNamespace: 'ROR:URL' as const } : {}), + identifierLimit: cap, + }), + undefined, + '', + undefined, + undefined, + 'full', + undefined, + undefined, + { fetch: tooMany.http }, + ), + ).rejects.toThrow(/identifier bound/); + expect(tooMany.calls.some((call) => call.name === 'Countries')).toBe( + false, + ); + } + expect(() => + validateBounds({ ...fixtureBounds(), userRows: 1001 }), + ).toThrow(/user rows/); + expect(() => + validateBounds({ ...fixtureBounds(), resultRows: 5001 }), + ).toThrow(/results/); + expect(() => + validateBounds({ ...fixtureBounds(), bytes: 5 * 1024 * 1024 + 1 }), + ).toThrow(/bytes/); + expect(() => + validateBounds({ ...fixtureBounds(), timeoutMs: 10001 }), + ).toThrow(/deadline/); + const branched = fixtureBounds(); + expect(() => + validateBounds({ + ...branched, + sources: [ + ...branched.sources, + { + database: 'geo', + name: 'Branches', + keyField: 'raw', + parent: { database: 'user', name: 'Customer', field: 'Country' }, + }, + ], + }), + ).toThrow(/immediately preceding/); + }); + it('enforces cumulative bytes, redirect refusal, cancellation and snapshot expiry', async () => { + const runtime = serve( + [row('1', { Country: 'USA' })], + [row('us', { raw: 'USA' })], + ); + await expect( + runFederatedQuery( + definition({ ...fixtureBounds(), bytes: 80 }), + undefined, + '', + undefined, + undefined, + 'full', + undefined, + undefined, + { fetch: runtime.http }, + ), + ).rejects.toThrow(/byte bound/); + const redirected = response({ records: [] }); + Object.defineProperty(redirected, 'redirected', { value: true }); + await expect(boundedResponseText(redirected, 1000)).rejects.toThrow( + /redirect/, + ); + const aborted = new AbortController(); + aborted.abort(new Error('cancelled by user')); + await expect( + runFederatedQuery( + definition(), + undefined, + '', + undefined, + aborted.signal, + 'full', + undefined, + undefined, + { fetch: runtime.http }, + ), + ).rejects.toThrow(/cancelled/); + await expect( + runFederatedQuery( + definition(), + undefined, + '', + undefined, + undefined, + 'full', + undefined, + undefined, + { fetch: async () => new Response('{}', { status: 410 }) }, + ), + ).rejects.toThrow(/expired/); + expect(() => + createBoundedFederationFetch( + 'https://unexpected.example', + fixtureBounds(), + runtime.http, + new AbortController().signal, + ), + ).toThrow(/allowlist/); + }); + it('distinguishes ambiguous and multiplied source matches from NULL/empty/missing, and counts repeated ROR affiliations once each', () => { + const bounds = fixtureBounds(); + const sources = new Map([ + [ + 'user.Customer', + [ + row('n', { Country: null }), + row('e', { Country: '' }), + row('u', { Country: 'Unknown' }), + row('a', { Country: 'USA' }), + ], + ], + ['geo.Countries', [row('1', { raw: 'USA' }), row('2', { raw: 'USA' })]], + ]); + expect(publicDataExceptions(bounds, sources)).toMatchObject({ + denominator: 4, + null: 1, + empty: 1, + unmatched: 1, + ambiguous: 1, + multiplied: 1, + }); + const native = 'https://ror.org/000025p04'; + expect( + publicDataExceptions( + { ...bounds, identifierKind: 'ror', identifierLimit: 50 }, + new Map([ + [ + 'user.Customer', + [row('a1', { Country: native }), row('a3', { Country: native })], + ], + [ + 'geo.Countries', + [ + row(native, { + raw: native, + locations: [{ ordinal: 0 }, { ordinal: 1 }], + }), + ], + ], + ]), + ), + ).toMatchObject({ denominator: 2, matched: 2, multiplied: 0 }); + }); + it('checks exact native ROR representation and checksum without confusing valid-but-absent membership or withdrawn status', () => { + for (const value of [ + 'https://ror.org/000025p04', + 'https://ror.org/000000098', + 'https://ror.org/0006jh821', + ]) + expect(validNativeRorUrl(value)).toBe(true); + for (const value of [ + 'https://ror.org/000025p05', + 'https://ror.org/100000096', + ' https://ror.org/000025p04', + 'https://ROR.org/000025p04', + '000025p04', + ]) + expect(validNativeRorUrl(value)).toBe(false); + const bounds = { + ...fixtureBounds(), + identifierKind: 'ror' as const, + identifierLimit: 50, + }; + const result = publicDataExceptions( + bounds, + new Map([ + [ + 'user.Customer', + [ + row('absent', { Country: 'https://ror.org/000000098' }), + row('withdrawn', { Country: 'https://ror.org/0006jh821' }), + ], + ], + [ + 'geo.Countries', + [row('w', { raw: 'https://ror.org/0006jh821', status: 'withdrawn' })], + ], + ]), + ); + expect(result).toMatchObject({ unmatched: 1, matched: 1, invalid: 0 }); + expect(result.details[1].targetStatus).toBe('withdrawn'); + }); + it('never requests malformed ROR values, retains them as invalid and queries checksum-valid absent references separately', async () => { + const runtime = serve( + [ + row('1', { Country: 'https://ror.org/000025p04' }), + row('2', { Country: ' https://ror.org/000025p04' }), + row('3', { Country: 'https://ror.org/000025p05' }), + row('4', { Country: 'https://ror.org/000000098' }), + ], + [row('active', { raw: 'https://ror.org/000025p04', status: 'active' })], + ); + const result = await runFederatedQuery( + definition({ + ...fixtureBounds(), + identifierKind: 'ror', + nativeNamespace: 'ROR:URL', + identifierLimit: 50, + }), + undefined, + '', + undefined, + undefined, + 'full', + undefined, + undefined, + { fetch: runtime.http }, + ); + expect( + runtime.calls.find((call) => call.name === 'Countries' && !call.close) + ?.where?.right.values, + ).toEqual(['https://ror.org/000025p04', 'https://ror.org/000000098']); + expect(result.publicDataExceptions).toMatchObject({ + denominator: 4, + matched: 1, + invalid: 2, + unmatched: 1, + }); + expect(result.publicDataExceptions?.details[1].raw).toBe( + ' https://ror.org/000025p04', + ); + expect(() => + validateBounds({ ...fixtureBounds(), identifierKind: 'ror' }), + ).toThrow(/explicit reviewed/); + }); + it('rejects a checksum-valid nonzero ROR prefix before any target request, while querying the valid absent control', async () => { + const malformed = 'https://ror.org/100000096'; + const absent = 'https://ror.org/000000098'; + for (const raw of [malformed, absent]) { + const runtime = serve([row('1', { Country: raw })], []); + const result = await runFederatedQuery( + definition({ + ...fixtureBounds(), + identifierKind: 'ror', + nativeNamespace: 'ROR:URL', + identifierLimit: 50, + }), + undefined, + '', + undefined, + undefined, + 'full', + undefined, + undefined, + { fetch: runtime.http }, + ); + const targetCalls = runtime.calls.filter( + (call) => call.name === 'Countries' && !call.close, + ); + expect(result.publicDataExceptions?.details[0].raw).toBe(raw); + if (raw === malformed) { + expect(targetCalls).toEqual([]); + expect(result.publicDataExceptions).toMatchObject({ + denominator: 1, + invalid: 1, + unmatched: 0, + }); + } else { + expect(targetCalls[0].where?.right.values).toEqual([absent]); + expect(result.publicDataExceptions).toMatchObject({ + denominator: 1, + invalid: 0, + unmatched: 1, + }); + } + } + }); + it('refuses duplicate raw response properties before they erase source identity', async () => { + await expect( + runFederatedQuery( + definition(), + undefined, + '', + undefined, + undefined, + 'full', + undefined, + undefined, + { + fetch: async () => + new Response( + '{"records":[{"key":"1","data":{"Country":"USA","Country":"Canada"}}]}', + ), + }, + ), + ).rejects.toThrow(/Duplicate/); + }); +}); diff --git a/libs/datatug/main/src/lib/queries/public-data/bounded-federation.ts b/libs/datatug/main/src/lib/queries/public-data/bounded-federation.ts new file mode 100644 index 00000000..a3cae045 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/bounded-federation.ts @@ -0,0 +1,399 @@ +import { strictJson } from './strict-json'; +import { parseDTQL } from '@dalgo/core'; +import { validNativeRorUrl } from './public-data-scenario'; + +export interface BoundedSource { + readonly database: string; + readonly name: string; + readonly keyField: string; + readonly parent?: { + readonly database: string; + readonly name: string; + readonly field: string; + }; +} + +/** Execution constraints, not a semantic acceptance mechanism. */ +export interface BoundedFederation { + readonly sources: readonly BoundedSource[]; + readonly userRows: number; + readonly userOffset: number; + readonly identifierKind: 'place' | 'ror'; + readonly nativeNamespace?: 'ROR:URL'; + readonly identifierLimit: number; + readonly resultRows: number; + readonly bytes: number; + readonly timeoutMs: number; +} + +export interface BoundedRecord { + readonly key: string; + readonly data: Record; +} +export interface BoundedReadReceipt { + readonly sources: ReadonlyMap; + readonly bytes: () => number; +} + +export const PUBLIC_DATA_OVDB_BASES = ['https://demodb.dev/ovdb'] as const; +const namePattern = /^[A-Za-z][A-Za-z0-9_]*$/; +const sourceId = (source: Pick): string => + `${source.database}.${source.name}`; + +export function validateBounds(bounds: BoundedFederation): void { + if ( + !['place', 'ror'].includes(bounds.identifierKind) || + (bounds.identifierKind === 'ror' && bounds.nativeNamespace !== 'ROR:URL') + ) + throw new Error( + 'A native ROR lookup requires the explicit reviewed ROR:URL syntax namespace.', + ); + const cap = bounds.identifierKind === 'ror' ? 50 : 100; + for (const [name, value, maximum, minimum] of [ + ['user rows', bounds.userRows, 1000, 1], + ['user offset', bounds.userOffset, 1_000_000, 0], + ['identifiers', bounds.identifierLimit, cap, 1], + ['results', bounds.resultRows, 5000, 1], + ['bytes', bounds.bytes, 5 * 1024 * 1024, 1], + ['deadline', bounds.timeoutMs, 10000, 1], + ] as const) { + if (!Number.isSafeInteger(value) || value < minimum || value > maximum) + throw new Error(`The ${name} bound must be ${minimum}–${maximum}.`); + } + if (bounds.sources.length < 2 || bounds.sources.length > 4) + throw new Error( + 'A bounded lookup needs two to four explicit source stages.', + ); + const known = new Set(); + for (const [index, source] of bounds.sources.entries()) { + if ( + ![source.database, source.name, source.keyField].every((name) => + namePattern.test(name), + ) + ) + throw new Error('Source fields must be explicit simple identifiers.'); + const id = sourceId(source); + if (known.has(id) || (index === 0) !== (source.parent === undefined)) + throw new Error( + 'The source stages must have one driver and distinct dependent references.', + ); + if ( + source.parent && + (sourceId(bounds.sources[index - 1]) !== sourceId(source.parent) || + !namePattern.test(source.parent.field)) + ) + throw new Error( + 'Each lookup must reference the immediately preceding source field.', + ); + known.add(id); + } +} + +/** One cumulative streaming byte guard, including failure bodies, with an abortable reader. */ +export async function boundedResponseText( + response: Response, + remaining: number, + signal?: AbortSignal, +): Promise<{ text: string; bytes: number }> { + if (response.redirected || response.type === 'opaqueredirect') + throw new Error('A public-data redirect is refused.'); + const advertised = response.headers.get('Content-Length'); + if ( + advertised !== null && + (!/^\d+$/.test(advertised) || Number(advertised) > remaining) + ) + throw new Error('The public-data response exceeds the byte bound.'); + const reader = response.body?.getReader(); + if (!reader) return { text: '', bytes: 0 }; + const chunks: Uint8Array[] = []; + let bytes = 0; + const abort = (): void => { + void reader.cancel(signal?.reason).catch(() => undefined); + }; + signal?.addEventListener('abort', abort, { once: true }); + try { + for (;;) { + signal?.throwIfAborted(); + const item = await reader.read(); + signal?.throwIfAborted(); + if (item.done) break; + bytes += item.value.byteLength; + if (bytes > remaining) + throw new Error('The public-data response exceeds the byte bound.'); + chunks.push(item.value); + } + const all = new Uint8Array(bytes); + let offset = 0; + for (const chunk of chunks) { + all.set(chunk, offset); + offset += chunk.byteLength; + } + return { + text: new TextDecoder('utf-8', { fatal: true }).decode(all), + bytes, + }; + } finally { + signal?.removeEventListener('abort', abort); + await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } +} + +/** + * Loads the driver first, then sends bounded native equality filters for each + * dependent source. DALgo sees only these verified pages. No data is fetched by + * construction; the first request occurs inside the user's run action. + */ +export function createBoundedFederationFetch( + base: string, + bounds: BoundedFederation, + httpFetch: typeof fetch, + signal: AbortSignal, +): { fetch: typeof fetch; receipt: BoundedReadReceipt } { + validateBounds(bounds); + if (!(PUBLIC_DATA_OVDB_BASES as readonly string[]).includes(base)) + throw new Error('This OVDB route is outside the public-data allowlist.'); + const loaded = new Map(); + const pending = new Map>(); + let bytes = 0; + const stages = new Map( + bounds.sources.map((source) => [sourceId(source), source]), + ); + const load = ( + source: BoundedSource, + headers: Headers, + ): Promise => { + const id = sourceId(source); + const existing = pending.get(id); + if (existing) return existing; + const request = (async () => { + let values: string[] | undefined; + if (source.parent) { + const parent = stages.get(sourceId(source.parent)); + if (!parent) throw new Error('The parent source is unavailable.'); + const rows = await load(parent, headers); + const unique = new Set(); + for (const row of rows) { + const value = row.data[source.parent.field]; + if (value === null || value === '') continue; + if ( + bounds.nativeNamespace === 'ROR:URL' && + (typeof value !== 'string' || !validNativeRorUrl(value)) + ) + continue; + if (typeof value !== 'string') + throw new Error( + 'A bounded native key must retain its declared string representation.', + ); + unique.add(value); + if (unique.size > bounds.identifierLimit) + throw new Error( + 'The selected rows exceed the distinct identifier bound. Narrow the user page.', + ); + } + values = [...unique]; + if (!values.length) { + loaded.set(id, []); + return []; + } + } + const maximum = source.parent ? bounds.resultRows : bounds.userRows; + const query = { + from: { name: source.name }, + limit: maximum, + ...(source.parent + ? { + where: { + left: { field: source.keyField }, + op: 'In', + right: { values }, + }, + } + : { offset: bounds.userOffset }), + }; + // Verify the exact installed DALgo dialect before trusting this request. + parseDTQL( + query, + { tables: [{ name: source.name, fields: [source.keyField] }] }, + { maxLimit: 5000 }, + ); + const body = JSON.stringify(query); + let pageToken: string | undefined; + let snapshot: string | undefined; + const seenTokens = new Set(); + const records: BoundedRecord[] = []; + try { + for (;;) { + signal.throwIfAborted(); + const pageSize = Math.min(100, maximum - records.length); + const requestHeaders = new Headers(headers); + requestHeaders.set('Content-Type', 'application/yaml'); + requestHeaders.set('Accept', 'application/json'); + requestHeaders.set('OVDB-Page-Size', String(pageSize)); + requestHeaders.delete('OVDB-Page-Close'); + requestHeaders.delete('OVDB-Page-Token'); + if (pageToken) requestHeaders.set('OVDB-Page-Token', pageToken); + const response = await httpFetch( + `${base}/v1/databases/${source.database}/dtql`, + { + method: 'POST', + headers: requestHeaders, + body, + signal, + redirect: 'error', + }, + ); + const read = await boundedResponseText( + response, + bounds.bytes - bytes, + signal, + ); + bytes += read.bytes; + if (response.status === 410) + throw new Error( + 'The selected public-data snapshot expired. Run explicitly again.', + ); + if (!response.ok) + throw new Error( + `The bounded source ${id} is unavailable or unsupported (${response.status}).`, + ); + const page = strictJson(read.text) as { + records?: BoundedRecord[]; + nextPageToken?: string; + snapshotToken?: string; + snapshotExpiresAt?: string; + }; + if (!Array.isArray(page.records) || page.records.length > pageSize) + throw new Error( + 'The bounded source returned too many rows or an invalid page.', + ); + if ( + page.snapshotToken !== undefined && + (typeof page.snapshotToken !== 'string' || + !page.snapshotToken || + (snapshot && page.snapshotToken !== snapshot)) + ) + throw new Error('The source changed snapshot during this lookup.'); + if ( + page.snapshotExpiresAt !== undefined && + (!Number.isFinite(Date.parse(page.snapshotExpiresAt)) || + Date.parse(page.snapshotExpiresAt) <= Date.now()) + ) + throw new Error('The selected source snapshot is stale.'); + snapshot = page.snapshotToken ?? snapshot; + for (const row of page.records) { + if ( + !row || + typeof row.key !== 'string' || + !row.data || + typeof row.data !== 'object' || + Array.isArray(row.data) + ) + throw new Error('The source returned a malformed raw row.'); + if ( + values && + (typeof row.data[source.keyField] !== 'string' || + !values.includes(row.data[source.keyField] as string)) + ) + throw new Error( + 'The runtime did not enforce the exact native-key filter.', + ); + records.push(row); + } + if (!page.nextPageToken) break; + if (records.length >= maximum) + throw new Error( + 'The source exceeds the selected row bound. Choose another explicit page.', + ); + if ( + typeof page.nextPageToken !== 'string' || + seenTokens.has(page.nextPageToken) || + page.records.length === 0 + ) + throw new Error('The source did not advance its bounded page.'); + seenTokens.add(page.nextPageToken); + pageToken = page.nextPageToken; + } + } finally { + if (snapshot) { + const close = new Headers(headers); + close.set('Content-Type', 'application/yaml'); + close.set('OVDB-Page-Token', snapshot); + close.set('OVDB-Page-Close', 'true'); + close.set('OVDB-Page-Size', '100'); + // Cancellation must also release the owned server snapshot. It does not fetch rows. + await httpFetch(`${base}/v1/databases/${source.database}/dtql`, { + method: 'POST', + headers: close, + body, + redirect: 'error', + signal: AbortSignal.timeout(1000), + }) + .then((response) => response.body?.cancel()) + .catch(() => undefined); + } + } + loaded.set(id, records); + return records; + })(); + pending.set(id, request); + return request; + }; + return { + receipt: { sources: loaded, bytes: () => bytes }, + fetch: async (input, init) => { + signal.throwIfAborted(); + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.href + : input.url; + if ( + init?.method !== 'POST' || + !url.startsWith(`${base}/v1/databases/`) || + !url.endsWith('/dtql') + ) + throw new Error( + 'Only the declared bounded DTQL sources may be requested.', + ); + const headers = new Headers(init.headers); + if (headers.get('OVDB-Page-Close') === 'true') + return new Response(null, { status: 204 }); + const document = strictJson(String(init.body)) as { + from: { name: string }; + }; + const database = url.slice( + `${base}/v1/databases/`.length, + -'/dtql'.length, + ); + const source = stages.get(`${database}.${document.from?.name}`); + if (!source) + throw new Error( + 'An undeclared source is ineligible for this bounded run.', + ); + const records = await load(source, headers); + const token = headers.get('OVDB-Page-Token'); + if (token !== null && !/^bounded:\d+$/.test(token)) + throw new Error('Invalid bounded page token.'); + const offset = token === null ? 0 : Number(token.slice(8)); + const size = Number(headers.get('OVDB-Page-Size') ?? 100); + if ( + !Number.isSafeInteger(size) || + size < 1 || + size > 500 || + !Number.isSafeInteger(offset) || + offset < 0 + ) + throw new Error('Invalid bounded page request.'); + const end = Math.min(offset + size, records.length); + return new Response( + JSON.stringify({ + records: records.slice(offset, end), + ...(end < records.length ? { nextPageToken: `bounded:${end}` } : {}), + }), + { headers: { 'Content-Type': 'application/json' } }, + ); + }, + }; +} diff --git a/libs/datatug/main/src/lib/queries/public-data/canonical-metadata.spec.ts b/libs/datatug/main/src/lib/queries/public-data/canonical-metadata.spec.ts new file mode 100644 index 00000000..6e0a81b1 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/canonical-metadata.spec.ts @@ -0,0 +1,134 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + CanonicalMetadataCache, + CanonicalMetadataReader, + INITIAL_CANONICAL_PINS, + immutableUrl, + sha256, + strictJson, + exactFields, + type ImmutableFile, +} from './canonical-metadata'; +import { sameSource, type SourceField } from './representation-discovery'; + +const file = async (text: string, path = 'a.json'): Promise => ({ + repository: 'https://github.com/example/fixture', + revision: 'a'.repeat(40), + path, + sha256: await sha256(text), +}); +const reader = (http: typeof fetch, cache = new CanonicalMetadataCache()) => + new CanonicalMetadataReader(http, cache, new AbortController().signal); + +describe('immutable canonical discovery admission', () => { + it('rejects duplicate/case-aliased keys, lone surrogates, trailing documents and mutable paths', () => { + for (const text of [ + '{"a":1,"a":2}', + '{"x":"\\ud800"}', + '{}{}', + '[1,]', + '{"a":1,}', + ]) + expect(() => strictJson(text)).toThrow(); + expect(strictJson('{"x":"\\ud83d\\ude00"}')).toEqual({ x: '😀' }); + expect(strictJson('{"Country":1,"COUNTRY":2}')).toEqual({ + Country: 1, + COUNTRY: 2, + }); + expect(() => + exactFields({ property: 1, Property: 2 }, ['property']), + ).toThrow(/Aliased/); + for (const encoded of [ + '\\uFFFD', + '�', + '\\ud83d\\ude00', + '\\uD83D\\uDE00', + '\\\\ud800', + '\\"label\\"', + ]) + expect(() => strictJson(`{"x":"${encoded}"}`)).not.toThrow(); + expect(() => + immutableUrl({ ...INITIAL_CANONICAL_PINS.directory, revision: 'main' }), + ).toThrow(); + expect(() => + immutableUrl({ + ...INITIAL_CANONICAL_PINS.directory, + path: '../index.json', + }), + ).toThrow(); + }); + it('caches exact verified bytes, invalidates on a revision mutation without code changes, and never aliases wrong hashes', async () => { + const text = '{"value":"fixture"}'; + const reference = await file(text); + const http = vi.fn(async () => new Response(text)); + const cache = new CanonicalMetadataCache(); + cache.begin(INITIAL_CANONICAL_PINS); + await reader(http, cache).json(reference); + await reader(http, cache).json(reference); + expect(http).toHaveBeenCalledTimes(1); + cache.begin({ + ...INITIAL_CANONICAL_PINS, + meanings: { + ...INITIAL_CANONICAL_PINS.meanings, + revision: 'b'.repeat(40), + }, + }); + await reader(http, cache).json(reference); + expect(http).toHaveBeenCalledTimes(2); + const operation = reader(http, cache); + await operation.json(reference); + await expect( + operation.json({ ...reference, sha256: '0'.repeat(64) }), + ).rejects.toThrow(/checksum/); + }); + it('charges aggregate cached and fetched metadata, rejects cycles/depth/count and refuses redirects/missing files', async () => { + const text = 'x'.repeat(1024 * 1024 + 1); + const a = await file(text); + const b = await file(text, 'b.json'); + const cache = new CanonicalMetadataCache(); + cache.set(a, text); + cache.set(b, text); + const operation = reader(vi.fn(), cache); + await operation.text(a); + await expect(operation.text(b)).rejects.toThrow(/aggregate/); + await expect(reader(vi.fn()).text(a, [immutableUrl(a)])).rejects.toThrow( + /cycle/, + ); + await expect( + reader(vi.fn()).text(a, Array(9).fill('ancestor')), + ).rejects.toThrow(/depth/); + const small = await file('{}'); + const many = reader(async () => new Response('{}')); + for (let i = 0; i < 96; i++) + await many.text({ ...small, path: `${i}.json` }); + await expect(many.text({ ...small, path: '97.json' })).rejects.toThrow( + /count/, + ); + const redirected = new Response('{}'); + Object.defineProperty(redirected, 'redirected', { value: true }); + await expect(reader(async () => redirected).text(small)).rejects.toThrow( + /redirect/, + ); + await expect( + reader(async () => new Response('{}', { status: 404 })).text(small), + ).rejects.toThrow(/unavailable/); + }); + it('uses full source coordinates and representation scope, with no case or space guesses', () => { + const source: SourceField = { + schema: INITIAL_CANONICAL_PINS.models, + module: 'sample', + entity: 'Customer', + property: 'Country', + datatype: 'string', + namespace: 'raw-country', + }; + expect(sameSource(source, source)).toBe(true); + for (const change of [ + { property: 'country' }, + { entity: 'Invoice' }, + { namespace: 'raw-country ' }, + { schema: { ...source.schema, revision: 'c'.repeat(40) } }, + ]) + expect(sameSource(source, { ...source, ...change })).toBe(false); + }); +}); diff --git a/libs/datatug/main/src/lib/queries/public-data/canonical-metadata.ts b/libs/datatug/main/src/lib/queries/public-data/canonical-metadata.ts new file mode 100644 index 00000000..6117ad3c --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/canonical-metadata.ts @@ -0,0 +1,215 @@ +import { strictJson } from './strict-json'; +export { strictJson } from './strict-json'; +import { boundedResponseText } from './bounded-federation'; + +export interface ImmutableFile { + readonly repository: string; + readonly revision: string; + readonly path: string; + readonly sha256: string; +} +export interface CanonicalPins { + readonly directory: ImmutableFile; + readonly models: ImmutableFile; + readonly meanings: ImmutableFile; +} +export const INITIAL_CANONICAL_PINS: CanonicalPins = { + directory: { + repository: 'https://github.com/openvaultdb/directory', + revision: '02db362144d7924c6081dd6768cc0d7187ac9bc3', + path: 'index.json', + sha256: '2795164e7051fb5b3c59400e261a0219382d68c9d74178c13f86f3ac3ef11995', + }, + models: { + repository: 'https://github.com/modelspec-org/registry', + revision: '48b30b250a61385d94d46a70968677870750ea35', + path: 'index.json', + sha256: '947e80bec116780f0ccb33c28caf4f2f5dbe53eb90e7b0e5c7f741116a69a320', + }, + meanings: { + repository: 'https://github.com/meaninggraph/registry', + revision: '2d92bbdd45fc2552f231f8f993a5b8b8d5b7fa29', + path: 'index.json', + sha256: '5d710692a320374c02912c932d755b80f2cbe67a463ef419daa6a1d7c8bfc0b9', + }, +}; + +export type MetadataObject = Record; +export function object(value: unknown, description: string): MetadataObject { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error(`Invalid ${description}.`); + return value as MetadataObject; +} +export function array(value: unknown, description: string): readonly unknown[] { + if (!Array.isArray(value)) throw new Error(`Invalid ${description}.`); + return value; +} +export function string(value: unknown, description: string): string { + if (typeof value !== 'string' || !value || value.length > 4096) + throw new Error(`Invalid ${description}.`); + return value; +} +export function exactFields( + record: MetadataObject, + consumed: readonly string[], +): void { + for (const key of Object.keys(record)) + if ( + consumed.some( + (name) => name !== key && name.toLowerCase() === key.toLowerCase(), + ) + ) + throw new Error('Aliased consumed metadata field.'); +} +export function immutableUrl(file: ImmutableFile): string { + if ( + !/^https:\/\/github\.com\/[a-z0-9_.-]+\/[a-z0-9_.-]+$/.test( + file.repository, + ) || + !/^[a-f0-9]{40}$/.test(file.revision) || + !/^[a-f0-9]{64}$/.test(file.sha256) || + !/^[A-Za-z0-9_./-]+$/.test(file.path) || + file.path.length > 1024 || + /(^\/|\/$|\/\/|(^|\/)(\.|\.\.|\.[gG][iI][tT])($|\/))/.test(file.path) + ) + throw new Error( + 'Canonical metadata must use immutable GitHub files and exact hashes.', + ); + return `https://raw.githubusercontent.com/${file.repository.slice('https://github.com/'.length)}/${file.revision}/${file.path}`; +} +export async function sha256(text: string): Promise { + const digest = await crypto.subtle.digest( + 'SHA-256', + new TextEncoder().encode(text), + ); + return Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, '0'), + ).join(''); +} + +/** Rebuildable bytes keyed by every immutable coordinate; never stores decisions. */ +export class CanonicalMetadataCache { + private readonly entries = new Map(); + private generation = ''; + begin(pins: CanonicalPins): void { + const generation = JSON.stringify(pins); + if (generation !== this.generation) { + this.entries.clear(); + this.generation = generation; + } + } + clear(): void { + this.entries.clear(); + this.generation = ''; + } + get(file: ImmutableFile): string | undefined { + return this.entries.get(JSON.stringify(file)); + } + set(file: ImmutableFile, text: string): void { + this.entries.set(JSON.stringify(file), text); + } +} + +/** One discovery operation: aggregate budget also charges reused cache bytes. */ +export class CanonicalMetadataReader { + private admittedBytes = 0; + private references = 0; + private readonly admitted = new Map>(); + private queue: Promise = Promise.resolve(); + constructor( + private readonly httpFetch: typeof fetch, + private readonly cache: CanonicalMetadataCache, + private readonly signal: AbortSignal, + ) {} + async text( + file: ImmutableFile, + ancestry: readonly string[] = [], + ): Promise { + const url = immutableUrl(file); + if (ancestry.length > 8 || ancestry.includes(url)) + throw new Error( + 'Canonical metadata has a cycle or exceeds reference depth.', + ); + const identity = JSON.stringify(file); + const previous = this.admitted.get(identity); + if (previous) return previous; + if (++this.references > 96) + throw new Error('Canonical metadata exceeds the reference count bound.'); + const request = this.queue.then(async () => { + this.signal.throwIfAborted(); + let text = this.cache.get(file); + if (text === undefined) { + const response = await this.httpFetch(url, { + redirect: 'error', + signal: this.signal, + headers: { Accept: 'application/json, application/yaml, text/plain' }, + }); + const read = await boundedResponseText( + response, + 2 * 1024 * 1024 - this.admittedBytes, + this.signal, + ); + if (!response.ok) + throw new Error( + `Canonical metadata unavailable (${response.status}).`, + ); + text = read.text; + } + this.admittedBytes += new TextEncoder().encode(text).byteLength; + if (this.admittedBytes > 2 * 1024 * 1024) + throw new Error('Discovery metadata exceeds the aggregate 2MiB bound.'); + if ((await sha256(text)) !== file.sha256) + throw new Error('Canonical metadata checksum mismatch.'); + this.cache.set(file, text); + return text; + }); + this.queue = request.then( + () => undefined, + () => undefined, + ); + this.admitted.set(identity, request); + return request; + } + async json( + file: ImmutableFile, + ancestry: readonly string[] = [], + ): Promise { + return object( + strictJson(await this.text(file, ancestry)), + 'canonical document', + ); + } + get bytes(): number { + return this.admittedBytes; + } +} + +export interface CanonicalIndexes { + readonly pins: CanonicalPins; + readonly directory: MetadataObject; + readonly models: MetadataObject; + readonly meanings: MetadataObject; + readonly bytes: number; +} +export async function readCanonicalIndexes( + pins: CanonicalPins, + reader: CanonicalMetadataReader, +): Promise { + // Sequential traversal keeps aggregate admission atomic and bounded. + const directory = await reader.json(pins.directory); + const models = await reader.json(pins.models); + const meanings = await reader.json(pins.meanings); + exactFields(directory, ['format', 'databases']); + exactFields(models, ['format', 'models']); + exactFields(meanings, ['format', 'graphs']); + if ( + directory['format'] !== 'ovdb-directory/draft-1' || + models['format'] !== 'modelspec-registry/draft-1' || + meanings['format'] !== 'meaning-registry/draft-1' + ) + throw new Error('Unsupported canonical index format.'); + array(directory['databases'], 'Directory databases'); + array(models['models'], 'ModelSpec records'); + array(meanings['graphs'], 'MeaningGraph records'); + return { pins, directory, models, meanings, bytes: reader.bytes }; +} diff --git a/libs/datatug/main/src/lib/queries/public-data/configured-public-data-sources.service.ts b/libs/datatug/main/src/lib/queries/public-data/configured-public-data-sources.service.ts new file mode 100644 index 00000000..6178138e --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/configured-public-data-sources.service.ts @@ -0,0 +1,171 @@ +import { Injectable, inject } from '@angular/core'; +import { firstValueFrom, fromEvent, takeUntil, type Observable } from 'rxjs'; +import type { IProjectContext } from '../../nav/nav-models'; +import { ProjectService } from '../../services/project/project.service'; +import { EnvironmentService } from '../../services/unsorted/environment.service'; +import { GithubProjectReaderService } from '../../services/repo/github/github-project-reader.service'; +import { + PublicDataService, + type PublicDataDiscovery, +} from './public-data.service'; +import { + configuredFieldChoices, + type ConfiguredFieldChoice, + type ConfiguredPublicSource, +} from './configured-source'; +import { object } from './canonical-metadata'; +import { validateHttpsJsonCatalog } from '../../project-files/https-json-catalog'; +import type { ITableFull } from '../../models/definition/apis/database'; + +async function read( + observable: Observable, + signal: AbortSignal, +): Promise { + signal.throwIfAborted(); + try { + return await firstValueFrom( + observable.pipe(takeUntil(fromEvent(signal, 'abort'))), + ); + } catch (error) { + signal.throwIfAborted(); + throw error; + } +} + +@Injectable({ providedIn: 'root' }) +export class ConfiguredPublicDataSourcesService { + private readonly projects = inject(ProjectService); + private readonly environments = inject(EnvironmentService); + private readonly github = inject(GithubProjectReaderService); + private readonly metadata = inject(PublicDataService); + + async list( + project: IProjectContext, + signal: AbortSignal = new AbortController().signal, + ): Promise { + const summary = + project.summary ?? + (await read(this.projects.getSummary(project.ref), signal)); + const github = ['github', 'github.com'].includes(project.ref.storeId); + const envs = + summary?.environments ?? + (github + ? await read( + this.environments.listEnvironments(project.ref.projectId), + signal, + ) + : []); + if (envs.length > 16) + throw new Error( + 'Select a project with at most 16 configured environments for bounded discovery.', + ); + const result: ConfiguredPublicSource[] = []; + for (const env of envs) { + const detail = await read( + this.environments.getEnvSummary(project.ref, env.id), + signal, + ); + for (const server of detail.dbServers ?? []) + for (const catalog of server.catalogs ?? []) { + if ( + ![env.id, catalog].every( + (value) => + /^[A-Za-z0-9_.-]{1,128}$/.test(value) && + value !== '.' && + value !== '..', + ) + ) + throw new Error('Unsafe configured source metadata path.'); + if (result.length >= 32) + throw new Error('Configured source discovery exceeds 32 catalogs.'); + let upstream: ConfiguredPublicSource['upstream']; + let sourceTables: readonly ITableFull[] | undefined; + if (github) { + const file = await read( + this.github.getRawJson( + project.ref.projectId, + `environments/${env.id}/catalogs/${catalog}/${catalog}.db.json`, + ), + signal, + ); + if ( + file && + typeof file === 'object' && + 'driver' in file && + file.driver === 'https-json' + ) { + const parsed = validateHttpsJsonCatalog(file, { + trust: 'untrusted', + }); + if (parsed.ok) + sourceTables = Object.keys(parsed.value.keys).map((name) => ({ + schema: '', + name, + dbType: 'BASE TABLE', + })); + } + if ( + file && + typeof file === 'object' && + 'upstream' in file && + file.upstream + ) { + const raw = object(file.upstream, 'configured upstream'); + if ( + typeof raw['repository'] === 'string' && + typeof raw['revision'] === 'string' + ) + upstream = { + repository: raw['repository'], + revision: raw['revision'], + }; + } + } + result.push({ + id: `${env.id}/${server.id}/${catalog}`, + title: `${detail.title} · ${catalog}`, + environment: env.id, + catalog, + driver: server.driver, + host: server.host, + ...(upstream ? { upstream } : {}), + ...(sourceTables ? { tables: sourceTables } : {}), + }); + } + } + return result; + } + + async inspect( + project: IProjectContext, + connection: ConfiguredPublicSource, + signal: AbortSignal, + ): Promise<{ + discovery: PublicDataDiscovery; + fields: readonly ConfiguredFieldChoice[]; + }> { + const tables = await read( + this.environments.getCatalogTables( + project.ref, + connection.environment, + connection.catalog, + ), + signal, + ); + signal.throwIfAborted(); + const listed = [...tables.tables, ...tables.views]; + const all = listed.length ? listed : (connection.tables ?? []); + if (all.length > 128) + throw new Error('Configured table discovery exceeds 128 tables/views.'); + const discovery = await this.metadata.discoverAll(signal); + return { + discovery, + fields: configuredFieldChoices( + connection, + all, + discovery.indexes, + discovery.suggestions, + ), + }; + } +} diff --git a/libs/datatug/main/src/lib/queries/public-data/configured-source.spec.ts b/libs/datatug/main/src/lib/queries/public-data/configured-source.spec.ts new file mode 100644 index 00000000..d1b9c4a0 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/configured-source.spec.ts @@ -0,0 +1,231 @@ +import { TestBed } from '@angular/core/testing'; +import { of } from 'rxjs'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; +import directory from '../fixtures/public-data-fabric/representation/directory.pinned.json'; +import models from '../fixtures/public-data-fabric/representation/models.pinned.json'; +import contractDocument from '../fixtures/public-data-fabric/representation/contract.json'; +import { + INITIAL_CANONICAL_PINS, + sha256, + type CanonicalIndexes, +} from './canonical-metadata'; +import { + configuredFieldChoices, + type ConfiguredPublicSource, +} from './configured-source'; +import { ConfiguredPublicDataSourcesService } from './configured-public-data-sources.service'; +import { PublicDataService } from './public-data.service'; +import { ProjectService } from '../../services/project/project.service'; +import { EnvironmentService } from '../../services/unsorted/environment.service'; +import { GithubProjectReaderService } from '../../services/repo/github/github-project-reader.service'; +import type { + RepresentationContract, + PublicDataSuggestion, +} from './representation-discovery'; + +function required(value: T | undefined): T { + if (value === undefined) throw new Error('Missing pinned fixture member.'); + return value; +} +const provider = required( + directory.databases.find((value) => value.localId === 'chinook'), +); +const model = required( + models.models.find( + (value) => + value.repository === provider.repository && + value.commit === provider.commit, + ), +); +const contract = structuredClone( + contractDocument.contracts[0], +) as RepresentationContract; +Object.assign(contract.source, { + module: model.module, + entity: 'Customer', + property: 'Country', + namespace: 'fixture-only-exact-labels', + schema: { + repository: provider.repository, + revision: provider.commit, + path: model.files.json, + sha256: '5e81f79ae02760bbdc372985022ab94209b97aaa76bf456fb59e040a99476870', + }, +}); +const suggestion: PublicDataSuggestion = { + provider: {}, + attachment: INITIAL_CANONICAL_PINS.directory, + contract, + matchesSource: false, + eligible: false, + reason: 'Synthetic bounded consumer fixture, pending publication', + rights: { source: 'MIT', model: 'MIT', meaning: 'fixture' }, + snapshot: {}, +}; +const indexes: CanonicalIndexes = { + pins: INITIAL_CANONICAL_PINS, + bytes: 0, + directory, + models, + meanings: {}, +}; +const table = { schema: '', name: 'Customer', dbType: 'BASE TABLE' }; +const connection: ConfiguredPublicSource = { + id: 'web/ovdb/chinook', + title: 'Configured Chinook', + environment: 'web', + catalog: 'chinook', + driver: 'ovdb', + host: provider.apiUrl, +}; + +describe('configured source adapter using pinned actual Directory and registry fixtures', () => { + it('retains exact immutable fixture bytes', async () => { + for (const [name, pin] of [ + ['directory', INITIAL_CANONICAL_PINS.directory], + ['models', INITIAL_CANONICAL_PINS.models], + ] as const) + expect( + await sha256( + readFileSync( + resolve( + `libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation/${name}.pinned.json`, + ), + 'utf8', + ), + ), + ).toBe(pin.sha256); + }); + it('resolves provider→explicit recordset modelEntity→property scope without user-authored hashes', () => { + const result = configuredFieldChoices(connection, [table], indexes, [ + suggestion, + ]); + const field = result.find((value) => value.property === 'Country'); + expect(field?.source).toEqual(contract.source); + expect(field?.source?.schema.revision).toBe(provider.commit); + expect( + result.find((value) => value.property === 'FirstName')?.source, + ).toBeUndefined(); + }); + it('rejects wrong origin/revision, API-base identity guessing, wrong entity mapping and ambiguous schema selection', () => { + for (const invalid of [ + { ...connection, host: 'https://unexpected.example/api' }, + { ...connection, host: provider.serverId }, + { + ...connection, + upstream: { repository: provider.repository, revision: 'b'.repeat(40) }, + }, + ]) + expect( + configuredFieldChoices(invalid, [table], indexes, [suggestion]).every( + (value) => !value.source, + ), + ).toBe(true); + expect( + configuredFieldChoices( + connection, + [{ ...table, schema: 'guessed-main' }], + indexes, + [suggestion], + )[0].source, + ).toBeUndefined(); + const changed = structuredClone(directory); + const record = required( + changed.databases.find((value) => value.localId === 'chinook'), + ); + required( + record.recordsets.find((value) => value.name === 'Customer'), + ).modelEntity = 'Invoice'; + expect( + configuredFieldChoices( + connection, + [table], + { ...indexes, directory: changed }, + [suggestion], + ).every((value) => !value.source), + ).toBe(true); + expect( + configuredFieldChoices( + { + ...connection, + driver: 'https-json', + host: 'chinook.demodb.dev', + upstream: { + repository: 'https://github.com/lerocha/chinook-database', + revision: '7f67772503d71ba90f19283c38e93923addb43fa', + }, + }, + [table], + indexes, + [suggestion], + )[0].reason, + ).toMatch(/original upstream/); + }); + it('uses existing project/environment/catalog services to resolve selected source metadata without querying rows', async () => { + const environments = { + getEnvSummary: vi.fn(() => + of({ + id: 'web', + title: 'Web', + dbServers: [ + { + id: 'ovdb', + driver: 'ovdb', + host: provider.apiUrl, + catalogs: ['chinook'], + }, + ], + }), + ), + getCatalogTables: vi.fn(() => of({ tables: [table], views: [] })), + }; + const github = { getRawJson: vi.fn(() => of({})) }; + const metadata = { + discoverAll: vi.fn(async () => ({ + indexes, + suggestions: [suggestion], + metadataBytes: 257101, + })), + }; + TestBed.configureTestingModule({ + providers: [ + ConfiguredPublicDataSourcesService, + { provide: ProjectService, useValue: { getSummary: vi.fn() } }, + { provide: EnvironmentService, useValue: environments }, + { provide: GithubProjectReaderService, useValue: github }, + { provide: PublicDataService, useValue: metadata }, + ], + }); + const service = TestBed.inject(ConfiguredPublicDataSourcesService); + const project = { + ref: { storeId: 'github.com', projectId: 'fixture@example@' }, + summary: { + id: 'fixture', + title: 'Fixture', + access: 'public' as const, + environments: [{ id: 'web' }], + }, + }; + const [selected] = await service.list(project); + const result = await service.inspect( + project, + selected, + new AbortController().signal, + ); + expect( + result.fields.find((value) => value.property === 'Country')?.source, + ).toEqual(contract.source); + expect(environments.getCatalogTables).toHaveBeenCalledWith( + project.ref, + 'web', + 'chinook', + ); + expect(github.getRawJson).toHaveBeenCalledWith( + project.ref.projectId, + 'environments/web/catalogs/chinook/chinook.db.json', + ); + expect(metadata.discoverAll).toHaveBeenCalledOnce(); + }); +}); diff --git a/libs/datatug/main/src/lib/queries/public-data/configured-source.ts b/libs/datatug/main/src/lib/queries/public-data/configured-source.ts new file mode 100644 index 00000000..c18eb9ce --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/configured-source.ts @@ -0,0 +1,185 @@ +import type { ITableFull } from '../../models/definition/apis/database'; +import { + array, + exactFields, + object, + type CanonicalIndexes, +} from './canonical-metadata'; +import { + sameSource, + type PublicDataSuggestion, + type SourceField, +} from './representation-discovery'; + +export interface ConfiguredPublicSource { + readonly id: string; + readonly title: string; + readonly environment: string; + readonly catalog: string; + readonly driver: string; + readonly host: string; + readonly tables?: readonly ITableFull[]; + readonly upstream?: { + readonly repository: string; + readonly revision: string; + }; +} +export interface ConfiguredFieldChoice { + readonly id: string; + readonly table: ITableFull; + readonly property: string; + readonly source?: SourceField; + readonly reason: string; +} + +/** Connection identity and explicit modelEntity mapping, never a table-name guess. */ +export function configuredFieldChoices( + connection: ConfiguredPublicSource, + tables: readonly ITableFull[], + indexes: CanonicalIndexes, + suggestions: readonly PublicDataSuggestion[], +): readonly ConfiguredFieldChoice[] { + const unavailable = (reason: string): readonly ConfiguredFieldChoice[] => + tables.map((table) => ({ + id: `${table.schema}.${table.name}`, + table, + property: '', + reason, + })); + const providers = array(indexes.directory['databases'], 'providers') + .map((value) => object(value, 'provider')) + .filter((provider) => { + exactFields(provider, [ + 'repository', + 'commit', + 'apiUrl', + 'id', + 'recordsets', + ]); + const explicitHost = + connection.driver === 'ovdb' && + (connection.host === provider['apiUrl'] || + connection.host === provider['id']); + const explicitUpstream = + connection.upstream?.repository === provider['repository'] && + connection.upstream?.revision === provider['commit']; + if (explicitHost && connection.upstream && !explicitUpstream) + return false; + return explicitHost || explicitUpstream; + }); + if (providers.length !== 1) + return unavailable( + 'This configured source has no exact canonical provider identity/revision. Its original upstream or table names cannot supply that provenance.', + ); + const provider = providers[0]; + const models = array(indexes.models['models'], 'models') + .map((value) => object(value, 'model')) + .filter( + (model) => + model['repository'] === provider['repository'] && + model['commit'] === provider['commit'], + ); + if (models.length !== 1) + return unavailable( + 'The exact provider revision has no unambiguous canonical ModelSpec registration.', + ); + const model = models[0]; + const recordsets = array(provider['recordsets'], 'recordsets').map((value) => + object(value, 'recordset'), + ); + const choices: ConfiguredFieldChoice[] = []; + for (const table of tables) { + const mappings = recordsets.filter( + (set) => + set['name'] === table.name && (set['schema'] ?? '') === table.schema, + ); + if ( + mappings.length !== 1 || + typeof mappings[0]['modelEntity'] !== 'string' + ) { + choices.push({ + id: `${table.schema}.${table.name}`, + table, + property: '', + reason: + 'No explicit canonical recordset-to-model entity mapping exists for this configured table.', + }); + continue; + } + const entity = mappings[0]['modelEntity']; + const entities = array(model['entities'], 'registered entities') + .map((value) => object(value, 'registered entity')) + .filter((value) => value['name'] === entity); + if (entities.length !== 1) { + choices.push({ + id: `${table.schema}.${table.name}`, + table, + property: '', + reason: + 'The explicit recordset modelEntity is absent or ambiguous in this pinned model.', + }); + continue; + } + for (const value of array( + entities[0]['properties'], + 'registered properties', + )) { + const property = object(value, 'registered property'); + if (property['type'] !== 'string' || typeof property['name'] !== 'string') + continue; + const physical = array(mappings[0]['fields'], 'canonical fields') + .map((value) => object(value, 'canonical field')) + .filter( + (value) => + value['name'] === property['name'] && value['type'] === 'string', + ); + if ( + physical.length !== 1 || + (table.columns && + !table.columns.some((column) => column.name === property['name'])) + ) { + choices.push({ + id: `${table.schema}.${table.name}.${property['name']}`, + table, + property: property['name'], + reason: + 'The exact model property has no unambiguous canonical raw string field in this configured recordset.', + }); + continue; + } + const scopes = suggestions.flatMap((suggestion) => { + const source = suggestion.contract?.source as SourceField | undefined; + if ( + !source || + !suggestion.rights || + !suggestion.snapshot || + source.schema.repository !== provider['repository'] || + source.schema.revision !== provider['commit'] || + source.schema.path !== + object(model['files'], 'model files')['json'] || + source.module !== model['module'] || + source.entity !== entity || + source.property !== property['name'] || + source.datatype !== 'string' + ) + return []; + return [source]; + }); + const unique = scopes.filter( + (scope, i) => + !scopes.slice(0, i).some((prior) => sameSource(prior, scope)), + ); + choices.push({ + id: `${table.schema}.${table.name}.${property['name']}`, + table, + property: property['name'], + ...(unique.length === 1 ? { source: unique[0] } : {}), + reason: + unique.length === 1 + ? 'Source scope resolved from the configured provider identity, pinned model and verified representation metadata. Inspect its explanation before any lookup.' + : 'This exact field has no unambiguous verified raw representation/namespace attachment at the pinned revisions. Lookup is unavailable.', + }); + } + } + return choices; +} diff --git a/libs/datatug/main/src/lib/queries/public-data/public-data-page.component.html b/libs/datatug/main/src/lib/queries/public-data/public-data-page.component.html new file mode 100644 index 00000000..c828a5aa --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/public-data-page.component.html @@ -0,0 +1,370 @@ +Public data lookup + + Connect a configured source +

+ Choose a project source, table and field. Canonical metadata supplies + its provenance and explains supported lookups. Values keep their + original case and spaces. +

+ Load configured sources + @if (connectionLoading() || loading()) { + Cancel source inspection + } + @if (connectionsLoaded() && !connections().length) { +

+ No configured project source is available. Configure a connection in + the existing environment view before choosing a public lookup. +

+ } + @if (connections().length) { + + } + @if (fields().length) { + + } + @if (connection() && !loading() && !fields().length) { +

+ No configured table metadata is available for this source. Lookup is + unavailable. +

+ } + @if (field(); as selectedField) { +

{{ selectedField.reason }}

+ Preview in existing table view + } + @if (source(); as scope) { +

+ Selected field: {{ scope.entity }}.{{ scope.property }} · + {{ scope.namespace }} +

+
+ Selected field provenance +

+ {{ scope.schema.repository }} at {{ scope.schema.revision }} · + {{ scope.schema.path }} · SHA256 {{ scope.schema.sha256 }} +

+
+ } + +
+ Advanced source descriptor diagnostics + Inspect a declared source field +

+ Use the immutable schema repository, revision, path and SHA256, plus + its exact module, entity, property, string datatype and raw namespace. + Values retain their original case and spaces. +

+ + Connect and explain + @if (loading()) { + Cancel + } + @if (source(); as field) { +

+ {{ field.schema.repository }} at {{ field.schema.revision }} · + {{ field.module }}.{{ field.entity }}.{{ field.property }} · raw + {{ field.datatype }} · {{ field.namespace }} +

+ } + +
+ @if (error()) { + {{ error() }} + } + @if (discovery(); as data) { + Canonical suggestions +

+ Source: {{ connection()?.title ?? 'Declared source field' }}. Public + release date and coverage: unknown. +

+
+ Canonical provenance and metadata limits +

+ Metadata read: {{ data.metadataBytes }} of 2097152 bytes. Metadata + inspection does not query source data. +

+

+ Directory {{ data.indexes.pins.directory.revision }} · ModelSpec + {{ data.indexes.pins.models.revision }} · MeaningGraph + {{ data.indexes.pins.meanings.revision }} +

+ @for ( + pin of [ + data.indexes.pins.directory, + data.indexes.pins.models, + data.indexes.pins.meanings, + ]; + track pin.repository + ) { +

+ {{ pin.repository }} · {{ pin.revision }} · {{ pin.path }} · + SHA256 {{ pin.sha256 }} +

+ } +
+ @if (!data.suggestions.length) { +

+ Lookup is unavailable: no supported scoped attachment is published + for this source. Labels alone do not authorize a lookup. +

+ } + @for (suggestion of data.suggestions; track $index) { + +

{{ suggestion.provider['title'] }}

+

{{ suggestion.reason }}

+ @if (suggestion.contract; as contract) { +

+ {{ contract.source.entity }}.{{ contract.source.property }} → + {{ contract.target.entity }}.{{ contract.target.property }} · + {{ contract.target.namespace }} +

+

+ Identity projection; UTF8 byte-exact equality; zero or one + native target. Missing keys remain exceptions. +

+
+ Lookup provenance +

Decision scope: {{ contract.decision.scope }}.

+

+ Raw label table {{ contract.bridge.table }}.{{ + contract.bridge.raw_label_column + }} + → {{ contract.bridge.target_key_column }}. The original + source field remains in the result. +

+
+ } + @if (suggestion.rights; as rights) { +

+ Source: {{ rights.source }} · Model: {{ rights.model }} · + Meaning: {{ rights.meaning }} +

+ } + @if (suggestion.matchesSource) { + Save pending scenario + } + @if (suggestion.matchesSource) { + Inspect this lookup + } + + } + + +
+ Execution limits and saved revisions +

+ Execution bounds: 1000 user rows; 100 place IDs or 50 ROR IDs; 5000 + results; 5MiB; 10 seconds. Saved scenarios reopen with their exact + revisions and require an explicit run. +

+
+ Run bounded lookup + @if (running()) { + Cancel lookup + } + Open saved queries + + } + @if (result(); as output) { + Results and exceptions + @if (output.publicDataExceptions; as counts) { + @if (hasRorStatuses()) { +

+ ROR matches are references to the pinned source snapshot, not + proof of affiliation or current validity. Repeated input + affiliations remain distinct; child locations do not increase the + affiliation or unique organization count. +

+ } +

+ Source rows {{ counts.denominator }} · non-NULL + {{ counts.nonNull }} · NULL {{ counts.null }} · empty + {{ counts.empty }} · invalid {{ counts.invalid }} · unmatched + {{ counts.unmatched }} · ambiguous {{ counts.ambiguous }} · matched + {{ counts.matched }} · multiplied {{ counts.multiplied }} +

+ + + + + + + + + + + + + @for (row of counts.details; track row.key) { + + + + + + + + } + +
+ Raw keys and projected keys +
Source rowRaw valueProjected keysExceptionTarget snapshot status
{{ row.key }}{{ row.raw === null ? 'NULL' : row.raw }}{{ row.projected.join(', ') }}{{ row.status }} + {{ row.targetStatus }} + @if (row.targetStatus === 'withdrawn') { + · erroneous, duplicate or out-of-scope record reference + } + @if (row.targetStatus === 'inactive') { + · historical reference context + } +
+ } +

+ Observed {{ output.provenance.observedAt }}. + {{ output.provenance.source }} +

+
+ Result provenance and transfer size +

Read {{ output.publicDataBytes }} bytes.

+
+ @for (limitation of output.limitations; track $index) { +

{{ limitation.policy }}

+ } + + + + + @for (column of output.recordset.columns; track $index) { + + } + + + + @for (row of resultRows(); track $index) { + + @for (cell of row; track $index) { + + } + + } + +
+ Selected result page +
{{ column.name }}
{{ cell.value === null ? 'NULL' : cell.value }}
+ Previous result page + Next result page + + } + diff --git a/libs/datatug/main/src/lib/queries/public-data/public-data-page.component.spec.ts b/libs/datatug/main/src/lib/queries/public-data/public-data-page.component.spec.ts new file mode 100644 index 00000000..db7b7aba --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/public-data-page.component.spec.ts @@ -0,0 +1,384 @@ +import { CUSTOM_ELEMENTS_SCHEMA } from '@angular/core'; +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { provideRouter } from '@angular/router'; +import { Subject, of } from 'rxjs'; +import { describe, expect, it, vi } from 'vitest'; +import { PublicDataPageComponent } from './public-data-page.component'; +import { + PublicDataService, + type PublicDataDiscovery, +} from './public-data.service'; +import { DatatugNavContextService } from '../../services/nav/datatug-nav-context.service'; +import { DatatugNavService } from '../../services/nav/datatug-nav.service'; +import { QueriesService } from '../queries.service'; +import { + FederatedQueryService, + type FederatedQueryResult, +} from '../federated-query.service'; +import { QueryType, type IQueryDef } from '../../models/definition/query-def'; +import { INITIAL_CANONICAL_PINS } from './canonical-metadata'; +import { + type PublicDataSuggestion, + type RepresentationContract, + type SourceField, + REPRESENTATION_PUBLICATION_BLOCKER, +} from './representation-discovery'; +import { type PublicDataScenario } from './public-data-scenario'; +import contractDocument from '../fixtures/public-data-fabric/representation/contract.json'; +import type { IProjectContext } from '../../nav/nav-models'; +import { ConfiguredPublicDataSourcesService } from './configured-public-data-sources.service'; +import { publicDataExceptions } from './public-data-scenario'; + +const contract = contractDocument.contracts[0] as RepresentationContract; +const source = contract.source as SourceField; +function scenario(eligible: boolean): PublicDataScenario { + const own = { ...INITIAL_CANONICAL_PINS.directory, path: 'fixture.json' }; + return { + source, + canonical: INITIAL_CANONICAL_PINS, + attachment: own, + snapshot: own, + model: own, + meaning: INITIAL_CANONICAL_PINS.meanings, + decision: own, + decisionScope: 'fixture-only', + namespace: contract.target.namespace, + projection: 'identity', + equality: 'utf8-byte-exact', + rights: { + source: 'fixture licence', + model: 'fixture model licence', + meaning: 'fixture meaning licence', + attribution: 'fixture only', + }, + observedAt: '2026-10-05T00:00:00Z', + eligible, + unavailableReason: REPRESENTATION_PUBLICATION_BLOCKER, + }; +} + +describe('Public data route fixture UI journey (no deployed runtime claims)', () => { + async function setup(eligible: boolean) { + const projectChanges = new Subject(); + const suggestion: PublicDataSuggestion = { + provider: { title: 'Synthetic fixture provider' }, + attachment: scenario(eligible).attachment, + contract, + matchesSource: true, + eligible, + reason: eligible + ? 'Explicit hypothetical UI fixture admission; no production acceptance.' + : REPRESENTATION_PUBLICATION_BLOCKER, + rights: scenario(eligible).rights, + snapshot: {}, + }; + const discovery: PublicDataDiscovery = { + indexes: { + pins: INITIAL_CANONICAL_PINS, + bytes: 10, + directory: {}, + models: {}, + meanings: {}, + }, + metadataBytes: 10, + suggestions: [suggestion], + }; + const definition = { + id: 'fixture-saved-query', + request: { queryType: QueryType.DTQL, text: '{}' }, + federation: { ovdbBaseUrl: 'https://demodb.dev/ovdb', tables: [] }, + publicData: scenario(eligible), + } as IQueryDef; + const metadata = { + discover: vi.fn(async () => discovery), + scenario: vi.fn(() => definition), + }; + const output = { + recordset: { + columns: [{ name: 'Country' }], + rows: [[{ value: 'USA' }], [{ value: null }]], + }, + provenance: { + source: 'bounded runtime fixture', + observedAt: '2026-10-05', + }, + limitations: [], + publicDataBytes: 80, + publicDataExceptions: { + denominator: 2, + nonNull: 1, + null: 1, + empty: 0, + invalid: 0, + unmatched: 0, + ambiguous: 0, + matched: 1, + multiplied: 0, + details: [ + { + key: '1', + raw: 'USA', + projected: ['US'], + status: 'matched', + matches: 1, + }, + { key: '2', raw: null, projected: [], status: 'NULL', matches: 0 }, + ], + }, + } as unknown as FederatedQueryResult; + const federation = { + run: vi.fn(async () => output), + dispose: vi.fn(async () => undefined), + getPage: vi.fn(), + }; + let reopened: IQueryDef | undefined; + const queries = { + createQuery: vi.fn((_ref, query: IQueryDef) => + of(structuredClone(query)), + ), + }; + const navigation = { + goTable: vi.fn(), + goQuery: vi.fn((_project, query: IQueryDef) => { + reopened = structuredClone(query); + }), + }; + const connection = { + id: 'fixture-source', + title: 'Configured fixture source', + environment: 'fixture', + catalog: 'user', + driver: 'ovdb', + host: 'https://fixture.invalid/api', + }; + const configured = { + list: vi.fn(async () => [connection]), + inspect: vi.fn(async () => ({ + discovery, + fields: [ + { + id: 'Customer.Country', + table: { schema: '', name: 'Customer', dbType: 'BASE TABLE' }, + property: 'Country', + source, + reason: 'Scope resolved from immutable canonical fixture metadata.', + }, + ], + })), + }; + await TestBed.configureTestingModule({ + imports: [PublicDataPageComponent], + schemas: [CUSTOM_ELEMENTS_SCHEMA], + providers: [ + provideRouter([]), + { provide: PublicDataService, useValue: metadata }, + { provide: ConfiguredPublicDataSourcesService, useValue: configured }, + { provide: FederatedQueryService, useValue: federation }, + { provide: QueriesService, useValue: queries }, + { provide: DatatugNavService, useValue: navigation }, + { + provide: DatatugNavContextService, + useValue: { currentProject: projectChanges }, + }, + ], + }).compileComponents(); + const fixture: ComponentFixture = + TestBed.createComponent(PublicDataPageComponent); + fixture.detectChanges(); + // Emit after initial render; zoneless DOM assertions wait for stability. + projectChanges.next({ + ref: { storeId: 'github.com', projectId: 'fixture@example@' }, + }); + fixture.componentInstance.sourceText.set(JSON.stringify(source)); + await fixture.whenStable(); + const click = async (label: string) => { + const buttons = Array.from( + fixture.nativeElement.querySelectorAll('ion-button'), + ) as HTMLElement[]; + const button = buttons.find( + (value) => value.textContent?.trim() === label, + ); + expect(button).toBeDefined(); + button?.click(); + await fixture.whenStable(); + }; + return { + fixture, + federation, + metadata, + queries, + navigation, + click, + configured, + reopened: () => reopened, + }; + } + it('connects, explains exact provenance, inspects, explicitly runs, shows exceptions, saves and reopens the exact query pins', async () => { + const state = await setup(true); + expect(state.federation.run).not.toHaveBeenCalled(); + state.fixture.componentInstance.sourceText.set(''); + await state.click('Load configured sources'); + const connection = state.fixture.nativeElement.querySelector( + '#fabric-connection', + ) as HTMLSelectElement; + connection.value = 'fixture-source'; + connection.dispatchEvent(new Event('change')); + await state.fixture.whenStable(); + const field = state.fixture.nativeElement.querySelector( + '#fabric-field', + ) as HTMLSelectElement; + field.value = 'Customer.Country'; + field.dispatchEvent(new Event('change')); + await state.fixture.whenStable(); + expect(state.fixture.componentInstance.sourceText()).toBe(''); + expect(state.fixture.componentInstance.source()).toEqual(source); + expect(state.configured.inspect).toHaveBeenCalledOnce(); + expect(state.fixture.nativeElement.textContent).toContain( + 'Customer.Country', + ); + expect(state.fixture.nativeElement.textContent).toContain( + 'UTF8 byte-exact equality', + ); + expect(state.fixture.nativeElement.textContent).toContain( + 'fixture licence', + ); + expect(state.federation.run).not.toHaveBeenCalled(); + await state.click('Inspect this lookup'); + expect(state.federation.run).not.toHaveBeenCalled(); + await state.click('Run bounded lookup'); + expect(state.federation.run).toHaveBeenCalledTimes(1); + expect( + state.fixture.nativeElement.querySelector( + '[data-testid="exception-counts"]', + ).textContent, + ).toContain('NULL 1'); + expect(state.fixture.nativeElement.textContent).toContain('USA'); + await state.click('Save pending scenario'); + expect(state.navigation.goQuery).toHaveBeenCalledTimes(1); + expect(state.reopened()?.publicData).toEqual(scenario(true)); + expect(state.reopened()?.federation).toEqual( + state.metadata.scenario.mock.results[0].value.federation, + ); + expect(state.federation.run).toHaveBeenCalledTimes(1); // save/reopen performs no implicit lookup + }); + it('renders production publication unavailable and refuses programmatic execution', async () => { + const state = await setup(false); + await state.click('Connect and explain'); + await state.click('Inspect this lookup'); + expect(state.fixture.nativeElement.textContent).toContain( + REPRESENTATION_PUBLICATION_BLOCKER, + ); + const run = Array.from( + state.fixture.nativeElement.querySelectorAll('ion-button'), + ).find( + (value) => + (value as HTMLElement).textContent?.trim() === 'Run bounded lookup', + ) as HTMLButtonElement; + expect(run.disabled).toBe(true); + await state.fixture.componentInstance.run(); + await state.fixture.whenStable(); + expect(state.federation.run).not.toHaveBeenCalled(); + }); + it('leads with readable availability while preserving immutable coordinates in closed expandable provenance', async () => { + const state = await setup(false); + await state.click('Connect and explain'); + const root = state.fixture.nativeElement as HTMLElement; + expect( + root.querySelector('[data-testid="source-availability"]')?.textContent, + ).toContain('Public release date and coverage: unknown'); + for (const selector of ['canonical-provenance', 'execution-limits']) { + const details = root.querySelector( + `[data-testid="${selector}"]`, + ) as HTMLDetailsElement; + expect(details.open).toBe(false); + expect(details.querySelector('summary')).toBeTruthy(); + } + const provenance = root.querySelector( + '[data-testid="canonical-provenance"]', + ) as HTMLDetailsElement; + expect(provenance.textContent).toContain( + INITIAL_CANONICAL_PINS.directory.revision, + ); + expect(provenance.textContent).toContain( + INITIAL_CANONICAL_PINS.directory.sha256, + ); + expect(provenance.textContent).toContain('2097152 bytes'); + expect( + root.querySelector('[data-testid="execution-limits"]')?.textContent, + ).toContain('5MiB'); + provenance.querySelector('summary')?.click(); + expect(provenance.open).toBe(true); + expect(state.federation.run).not.toHaveBeenCalled(); + }); + it('renders snapshot status warnings and keeps repeated affiliations separate from child locations', async () => { + const state = await setup(false); + const raw = 'https://ror.org/0042xzm63'; + const counts = publicDataExceptions( + { + userRows: 1000, + userOffset: 0, + identifierKind: 'ror', + nativeNamespace: 'ROR:URL', + identifierLimit: 50, + resultRows: 5000, + bytes: 5242880, + timeoutMs: 10000, + sources: [ + { database: 'user', name: 'Affiliation', keyField: 'ror_id' }, + { + database: 'ror', + name: 'organizations', + keyField: 'id', + parent: { database: 'user', name: 'Affiliation', field: 'ror_id' }, + }, + ], + }, + new Map([ + [ + 'user.Affiliation', + [ + { key: 'a1', data: { ror_id: raw } }, + { key: 'a2', data: { ror_id: raw } }, + { key: 'w', data: { ror_id: 'https://ror.org/0006jh821' } }, + { key: 'i', data: { ror_id: 'https://ror.org/00067tc54' } }, + ], + ], + [ + 'ror.organizations', + [ + { + key: raw, + data: { + id: raw, + status: 'active', + locations: [{ ordinal: 0 }, { ordinal: 1 }], + }, + }, + { + key: 'w', + data: { id: 'https://ror.org/0006jh821', status: 'withdrawn' }, + }, + { + key: 'i', + data: { id: 'https://ror.org/00067tc54', status: 'inactive' }, + }, + ], + ], + ]), + ); + state.fixture.componentInstance.result.set({ + recordset: { columns: [], rows: [] }, + provenance: { source: 'ROR rendering fixture', observedAt: '2026-09-22' }, + limitations: [], + publicDataExceptions: counts, + } as unknown as FederatedQueryResult); + await state.fixture.whenStable(); + const text = state.fixture.nativeElement.textContent; + expect(text).toContain('matched 4'); + expect(text).toContain('multiplied 0'); + expect(text).toContain('child locations do not increase'); + expect(text).toContain('erroneous, duplicate or out-of-scope'); + expect(text).toContain('historical reference context'); + expect(state.federation.run).not.toHaveBeenCalled(); + }); +}); diff --git a/libs/datatug/main/src/lib/queries/public-data/public-data-page.component.ts b/libs/datatug/main/src/lib/queries/public-data/public-data-page.component.ts new file mode 100644 index 00000000..feba8ab8 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/public-data-page.component.ts @@ -0,0 +1,423 @@ +import { + ChangeDetectionStrategy, + Component, + computed, + inject, + OnDestroy, + signal, +} from '@angular/core'; +import { FormsModule } from '@angular/forms'; +import { RouterLink } from '@angular/router'; +import { + IonBackButton, + IonButton, + IonButtons, + IonCard, + IonCardContent, + IonCardHeader, + IonCardTitle, + IonContent, + IonHeader, + IonInput, + IonItem, + IonMenuButton, + IonSpinner, + IonTextarea, + IonTitle, + IonToolbar, +} from '@ionic/angular'; +import { firstValueFrom, Subscription } from 'rxjs'; +import { DatatugNavContextService } from '../../services/nav/datatug-nav-context.service'; +import { DatatugNavService } from '../../services/nav/datatug-nav.service'; +import { projectPageHref } from '../../nav/project-page-href'; +import type { IProjectContext } from '../../nav/nav-models'; +import { QueriesService } from '../queries.service'; +import { + FederatedQueryService, + type FederatedQueryResult, +} from '../federated-query.service'; +import { + PublicDataService, + type PublicDataDiscovery, +} from './public-data.service'; +import { immutableUrl, object, strictJson } from './canonical-metadata'; +import type { + PublicDataSuggestion, + SourceField, +} from './representation-discovery'; +import { + sameSource, + REPRESENTATION_PUBLICATION_BLOCKER, +} from './representation-discovery'; +import { ConfiguredPublicDataSourcesService } from './configured-public-data-sources.service'; +import type { + ConfiguredPublicSource, + ConfiguredFieldChoice, +} from './configured-source'; + +@Component({ + selector: 'sneat-datatug-public-data-page', + templateUrl: './public-data-page.component.html', + changeDetection: ChangeDetectionStrategy.OnPush, + imports: [ + FormsModule, + RouterLink, + IonBackButton, + IonButton, + IonButtons, + IonCard, + IonCardContent, + IonCardHeader, + IonCardTitle, + IonContent, + IonHeader, + IonInput, + IonItem, + IonMenuButton, + IonSpinner, + IonTextarea, + IonTitle, + IonToolbar, + ], +}) +export class PublicDataPageComponent implements OnDestroy { + private readonly metadata = inject(PublicDataService); + private readonly configured = inject(ConfiguredPublicDataSourcesService); + private readonly queries = inject(QueriesService); + private readonly federation = inject(FederatedQueryService); + private readonly navigation = inject(DatatugNavService); + private readonly navContext = inject(DatatugNavContextService); + readonly project = signal(undefined); + readonly sourceText = signal(''); + readonly connections = signal([]); + readonly connection = signal(undefined); + readonly fields = signal([]); + readonly field = signal(undefined); + readonly connectionLoading = signal(false); + readonly connectionsLoaded = signal(false); + readonly source = signal(undefined); + readonly discovery = signal(undefined); + readonly loading = signal(false); + readonly saving = signal(false); + readonly running = signal(false); + readonly result = signal(undefined); + readonly selected = signal(undefined); + readonly page = signal(0); + readonly resultRows = signal([]); + readonly hasRorStatuses = computed( + () => + this.result()?.publicDataExceptions?.details.some( + (row) => !!row.targetStatus, + ) ?? false, + ); + readonly error = signal(''); + readonly userRows = signal(100); + readonly userOffset = signal(0); + readonly backHref = computed(() => projectPageHref(this.project()?.ref)); + readonly queriesHref = computed(() => + projectPageHref(this.project()?.ref, 'queries'), + ); + readonly compatible = computed( + () => + this.discovery()?.suggestions.filter((value) => value.matchesSource) ?? + [], + ); + private controller?: AbortController; + private generation = 0; + private readonly subscription: Subscription = + this.navContext.currentProject.subscribe((value) => { + this.controller?.abort(); + if (this.running()) void this.federation.dispose().catch(() => undefined); + this.generation++; + this.loading.set(false); + this.connectionLoading.set(false); + this.connectionsLoaded.set(false); + this.project.set(value); + this.connections.set([]); + this.connection.set(undefined); + this.fields.set([]); + this.field.set(undefined); + this.source.set(undefined); + this.discovery.set(undefined); + this.result.set(undefined); + this.selected.set(undefined); + }); + + async loadConnections(): Promise { + const project = this.project(); + if (!project || this.connectionLoading()) return; + const generation = this.generation; + const controller = new AbortController(); + this.controller?.abort(); + this.controller = controller; + const deadline = setTimeout( + () => + controller.abort(new Error('Configured source discovery timed out.')), + 10000, + ); + this.connectionLoading.set(true); + this.error.set(''); + try { + const connections = await this.configured.list( + project, + controller.signal, + ); + if (generation === this.generation) { + this.connections.set(connections); + this.connectionsLoaded.set(true); + } + } catch (error) { + if (generation === this.generation) + this.error.set( + error instanceof Error + ? error.message + : 'Configured source metadata is unavailable.', + ); + } finally { + clearTimeout(deadline); + if (generation === this.generation) this.connectionLoading.set(false); + } + } + async selectConnection(id: string): Promise { + const connection = this.connections().find((value) => value.id === id); + const project = this.project(); + if (!connection || !project) return; + this.controller?.abort(); + const controller = new AbortController(); + this.controller = controller; + const generation = ++this.generation; + this.connection.set(connection); + this.fields.set([]); + this.field.set(undefined); + this.source.set(undefined); + this.selected.set(undefined); + this.discovery.set(undefined); + this.error.set(''); + this.loading.set(true); + const deadline = setTimeout( + () => + controller.abort( + new Error('Configured source metadata inspection timed out.'), + ), + 10000, + ); + try { + const inspected = await this.configured.inspect( + project, + connection, + controller.signal, + ); + if (generation === this.generation) { + this.fields.set(inspected.fields); + this.discovery.set(inspected.discovery); + } + } catch (error) { + if (generation === this.generation) + this.error.set( + error instanceof Error + ? error.message + : 'Configured source inspection failed.', + ); + } finally { + clearTimeout(deadline); + if (generation === this.generation) this.loading.set(false); + } + } + selectField(id: string): void { + const field = this.fields().find((value) => value.id === id); + this.field.set(field); + this.source.set(field?.source); + this.selected.set(undefined); + const discovery = this.discovery(); + if (discovery) + this.discovery.set({ + ...discovery, + suggestions: discovery.suggestions.map((value) => { + const matchesSource = + !!field?.source && + !!value.contract && + sameSource(field.source, value.contract.source); + return { + ...value, + matchesSource, + reason: + matchesSource && !value.eligible + ? REPRESENTATION_PUBLICATION_BLOCKER + : value.reason, + }; + }), + }); + } + previewTable(): void { + const project = this.project(); + const connection = this.connection(); + const field = this.field(); + if (project && connection && field) + this.navigation.goTable({ + project, + env: connection.environment, + db: connection.catalog, + schema: field.table.schema, + name: field.table.name, + }); + } + + async connect(): Promise { + if (this.loading()) return; + const generation = ++this.generation; + this.error.set(''); + this.discovery.set(undefined); + this.source.set(undefined); + this.result.set(undefined); + this.selected.set(undefined); + try { + const record = object( + strictJson(this.sourceText()), + 'declared source field', + ); + if ( + Object.keys(record).sort().join(',') !== + 'datatype,entity,module,namespace,property,schema' || + record['datatype'] !== 'string' || + !['entity', 'module', 'property', 'namespace'].every( + (key) => typeof record[key] === 'string' && !!record[key], + ) + ) + throw new Error( + 'Declare the exact schema, module, entity, property, string datatype and raw namespace.', + ); + const source = record as unknown as SourceField; + immutableUrl(source.schema); + this.source.set(source); + const controller = new AbortController(); + this.controller = controller; + const deadline = setTimeout( + () => controller.abort(new Error('Canonical metadata read timed out.')), + 10000, + ); + this.loading.set(true); + try { + const discovery = await this.metadata.discover( + source, + controller.signal, + ); + if (generation === this.generation) this.discovery.set(discovery); + } finally { + clearTimeout(deadline); + } + } catch (error) { + if (generation === this.generation) + this.error.set( + error instanceof Error + ? error.message + : 'Canonical discovery failed.', + ); + } finally { + if (generation === this.generation) this.loading.set(false); + } + } + cancel(): void { + this.controller?.abort(new Error('Discovery cancelled.')); + if (this.running()) + void this.federation + .dispose() + .catch((error: unknown) => + this.error.set( + error instanceof Error + ? error.message + : 'Cannot cancel the lookup.', + ), + ); + } + async run(): Promise { + const generation = this.generation; + const source = this.source(); + const discovery = this.discovery(); + const selected = this.selected(); + if (!source || !discovery || !selected || this.running()) return; + if (!selected.eligible) { + this.error.set(selected.reason); + return; + } + this.error.set(''); + this.running.set(true); + this.result.set(undefined); + this.page.set(0); + this.resultRows.set([]); + try { + const query = this.metadata.scenario(source, discovery, selected, { + userRows: this.userRows(), + userOffset: this.userOffset(), + }); + const result = await this.federation.run(query, undefined, '', 'full'); + if (generation !== this.generation) return; + this.result.set(result); + this.resultRows.set(result.recordset.rows); + } catch (error) { + this.error.set( + error instanceof Error ? error.message : 'The bounded lookup failed.', + ); + } finally { + this.running.set(false); + } + } + async changePage(delta: number): Promise { + const next = this.page() + delta; + if ( + next < 0 || + next * 100 >= + (this.result()?.totalRows ?? this.result()?.recordset.rows.length ?? 0) + ) + return; + try { + const rows = await this.federation.getPage(next); + this.resultRows.set(rows); + this.page.set(next); + } catch (error) { + this.error.set( + error instanceof Error ? error.message : 'Cannot read result page.', + ); + } + } + async save(suggestion: PublicDataSuggestion): Promise { + const generation = this.generation; + const source = this.source(); + const discovery = this.discovery(); + const project = this.project(); + if (!source || !discovery || !project || this.saving()) return; + this.error.set(''); + this.saving.set(true); + try { + const query = this.metadata.scenario(source, discovery, suggestion, { + userRows: this.userRows(), + userOffset: this.userOffset(), + }); + const saved = await firstValueFrom( + this.queries.createQuery(project.ref, query), + ); + if (generation !== this.generation) return; + if ( + !saved.publicData || + saved.publicData.attachment.sha256 !== + query.publicData?.attachment.sha256 || + JSON.stringify(saved.federation) !== JSON.stringify(query.federation) + ) + throw new Error( + 'The query store did not preserve the exact scenario pins. This store cannot save the scenario safely yet.', + ); + this.navigation.goQuery(project, saved); + } catch (error) { + this.error.set( + error instanceof Error ? error.message : 'Scenario save failed.', + ); + } finally { + this.saving.set(false); + } + } + ngOnDestroy(): void { + this.generation++; + this.controller?.abort(); + this.subscription.unsubscribe(); + void this.federation.dispose().catch(() => undefined); + } +} diff --git a/libs/datatug/main/src/lib/queries/public-data/public-data-scenario.ts b/libs/datatug/main/src/lib/queries/public-data/public-data-scenario.ts new file mode 100644 index 00000000..087c0a23 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/public-data-scenario.ts @@ -0,0 +1,155 @@ +import type { BoundedFederation, BoundedRecord } from './bounded-federation'; +import type { CanonicalPins, ImmutableFile } from './canonical-metadata'; +import type { SourceField } from './representation-discovery'; + +export const SAVED_SCENARIO_PUBLICATION_BLOCKER = + 'Saved public-data execution awaits canonical companion publication and fresh metadata admission. A saved eligibility value does not authorize execution.'; + +/** Saved with the existing query; never refreshes or executes on reopening. */ +export interface PublicDataScenario { + readonly source: SourceField; + readonly canonical: CanonicalPins; + readonly attachment: ImmutableFile; + readonly snapshot: ImmutableFile; + readonly model: ImmutableFile; + readonly meaning: ImmutableFile; + readonly decision: ImmutableFile; + readonly decisionScope: string; + readonly namespace: string; + readonly projection: 'identity'; + readonly equality: 'utf8-byte-exact'; + readonly rights: { + readonly source: string; + readonly model: string; + readonly meaning: string; + readonly attribution: string; + }; + readonly observedAt: string; + readonly eligible: boolean; + readonly unavailableReason: string; +} +export interface PublicDataExceptions { + readonly denominator: number; + readonly nonNull: number; + readonly null: number; + readonly empty: number; + readonly invalid: number; + readonly unmatched: number; + readonly ambiguous: number; + readonly matched: number; + readonly multiplied: number; + readonly details: readonly { + readonly key: string; + readonly raw: unknown; + readonly projected: readonly string[]; + readonly status: string; + readonly matches: number; + readonly targetStatus?: string; + }[]; +} + +/** Representation validity only; exact snapshot membership is a separate lookup. */ +export function validNativeRorUrl(raw: string): boolean { + if (!/^https:\/\/ror\.org\/0[0-9a-hj-km-np-tv-z]{6}[0-9]{2}$/.test(raw)) + return false; + const token = raw.slice('https://ror.org/'.length); + const alphabet = '0123456789abcdefghjkmnpqrstvwxyz'; + let value = 0; + for (const character of token.slice(0, 7)) + value = value * 32 + alphabet.indexOf(character); + return Number(token.slice(7)) === 98 - ((value * 100) % 97); +} + +/** Counts source rows/affiliations, never one affiliation once per location. */ +export function publicDataExceptions( + bounds: BoundedFederation, + sources: ReadonlyMap, +): PublicDataExceptions { + const driver = bounds.sources[0]; + const reference = bounds.sources[1]; + const rows = sources.get(`${driver.database}.${driver.name}`) ?? []; + const targets = sources.get(`${reference.database}.${reference.name}`) ?? []; + const counts = { + denominator: rows.length, + nonNull: 0, + null: 0, + empty: 0, + invalid: 0, + unmatched: 0, + ambiguous: 0, + matched: 0, + multiplied: 0, + }; + const details = rows.map((row) => { + const raw = row.data[reference.parent?.field ?? driver.keyField]; + let status: string; + const bridgeMatches = targets.filter( + (target) => target.data[reference.keyField] === raw, + ); + let matches = bridgeMatches; + for (const stage of bounds.sources.slice(2)) { + const candidates = sources.get(`${stage.database}.${stage.name}`) ?? []; + matches = matches.flatMap((parent) => + candidates.filter( + (target) => + target.data[stage.keyField] === + parent.data[stage.parent?.field ?? ''], + ), + ); + } + if (raw === null || raw === undefined) { + counts.null++; + status = 'NULL'; + } else { + counts.nonNull++; + if (raw === '') { + counts.empty++; + status = 'empty'; + } else if ( + typeof raw !== 'string' || + (bounds.identifierKind === 'ror' && !validNativeRorUrl(raw)) + ) { + counts.invalid++; + status = 'invalid'; + } else if (!matches.length) { + counts.unmatched++; + status = 'unmatched'; + } else if (matches.length > 1) { + counts.ambiguous++; + counts.multiplied += matches.length - 1; + status = 'ambiguous'; + } else { + counts.matched++; + status = 'matched'; + } + } + const nextField = bounds.sources[2]?.parent?.field ?? reference.keyField; + const projected = bridgeMatches + .map((match) => match.data[nextField]) + .filter((value): value is string => typeof value === 'string'); + const targetStatus = + bounds.identifierKind === 'ror' && + matches.length === 1 && + typeof matches[0].data['status'] === 'string' + ? matches[0].data['status'] + : undefined; + return { + key: row.key, + raw, + projected, + status, + matches: matches.length, + ...(targetStatus ? { targetStatus } : {}), + }; + }); + return { ...counts, details }; +} +export function scenarioRevisionChanges( + saved: PublicDataScenario, + current: CanonicalPins, +): readonly string[] { + return (['directory', 'models', 'meanings'] as const).filter( + (kind) => + JSON.stringify(saved.canonical[kind]) !== JSON.stringify(current[kind]), + ); +} diff --git a/libs/datatug/main/src/lib/queries/public-data/public-data.service.spec.ts b/libs/datatug/main/src/lib/queries/public-data/public-data.service.spec.ts new file mode 100644 index 00000000..5c6d9279 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/public-data.service.spec.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from 'vitest'; +import document from '../fixtures/public-data-fabric/representation/contract.json'; +import { INITIAL_CANONICAL_PINS } from './canonical-metadata'; +import { + PublicDataService, + type PublicDataDiscovery, +} from './public-data.service'; +import type { + PublicDataSuggestion, + RepresentationContract, + SourceField, +} from './representation-discovery'; + +const contract = document.contracts[0] as RepresentationContract; +const source = contract.source as SourceField; +const driver = { + name: 'Customer', + modelEntity: 'Customer', + fields: [{ name: 'Country' }], +}; +const sourceProvider = { + repository: source.schema.repository, + commit: source.schema.revision, + localId: 'source', + apiUrl: 'https://demodb.dev/ovdb/v1/databases/source', + recordsets: [driver], +}; +const suggestion: PublicDataSuggestion = { + contract, + matchesSource: true, + eligible: false, + reason: 'Staged fixture', + attachment: INITIAL_CANONICAL_PINS.directory, + rights: { source: 'fixture', model: 'fixture', meaning: 'fixture' }, + snapshot: {}, + provider: { + repository: 'https://github.com/example/public', + commit: 'a'.repeat(40), + localId: 'public', + apiUrl: 'https://demodb.dev/ovdb/v1/databases/public', + recordsets: [ + { + name: 'CustomerCountries', + fields: [{ name: 'raw_label' }, { name: 'target_key' }], + }, + { + name: 'countries_physical', + modelEntity: 'Countries', + fields: [{ name: 'iso' }], + }, + ], + }, +}; +const discovery: PublicDataDiscovery = { + metadataBytes: 0, + suggestions: [suggestion], + indexes: { + pins: INITIAL_CANONICAL_PINS, + bytes: 0, + directory: { databases: [sourceProvider] }, + models: {}, + meanings: {}, + }, +}; + +describe('pending bounded scenario factory', () => { + it('uses explicit modelEntity mappings and a separately declared physical bridge table', () => { + const query = new PublicDataService().scenario( + source, + discovery, + suggestion, + { userRows: 1000, userOffset: 0 }, + ); + expect(query.federation?.tables.map((value) => value.name)).toEqual([ + 'Customer', + 'CustomerCountries', + 'countries_physical', + ]); + expect(query.publicData?.eligible).toBe(false); + }); + it('rejects name-only model guesses, mismatched source claims and out-of-bound saved scenarios', () => { + const changed = structuredClone(discovery); + changed.indexes.directory['databases'] = [ + { + ...sourceProvider, + recordsets: [{ name: 'Customer', fields: driver.fields }], + }, + ]; + const service = new PublicDataService(); + expect(() => + service.scenario(source, changed, suggestion, { + userRows: 1000, + userOffset: 0, + }), + ).toThrow(/exact model entity/); + expect(() => + service.scenario( + { ...source, property: 'InvoiceCountry' }, + discovery, + suggestion, + { userRows: 1000, userOffset: 0 }, + ), + ).toThrow(/exact verified source/); + expect(() => + service.scenario(source, discovery, suggestion, { + userRows: 1001, + userOffset: 0, + }), + ).toThrow(/user rows/); + }); +}); diff --git a/libs/datatug/main/src/lib/queries/public-data/public-data.service.ts b/libs/datatug/main/src/lib/queries/public-data/public-data.service.ts new file mode 100644 index 00000000..16819d85 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/public-data.service.ts @@ -0,0 +1,310 @@ +import { Injectable } from '@angular/core'; +import type { IQueryDef } from '../../models/definition/query-def'; +import { QueryType } from '../../models/definition/query-def'; +import { + PUBLIC_DATA_OVDB_BASES, + validateBounds, + type BoundedFederation, +} from './bounded-federation'; +import { + array, + object, + string, + CanonicalMetadataCache, + CanonicalMetadataReader, + readCanonicalIndexes, + INITIAL_CANONICAL_PINS, + type CanonicalIndexes, + type CanonicalPins, + type ImmutableFile, + type MetadataObject, +} from './canonical-metadata'; +import { + discoverRepresentations, + sameSource, + type PublicDataSuggestion, + type SourceField, +} from './representation-discovery'; + +export interface PublicDataDiscovery { + readonly indexes: CanonicalIndexes; + readonly suggestions: readonly PublicDataSuggestion[]; + readonly metadataBytes: number; +} + +function recordset( + provider: MetadataObject, + identity: string, + field: 'modelEntity' | 'name' = 'modelEntity', +): MetadataObject { + const matches = array(provider['recordsets'], 'recordsets') + .map((value) => object(value, 'recordset')) + .filter((value) => value[field] === identity); + if (matches.length !== 1) + throw new Error( + 'The exact model entity has no unambiguous Directory recordset.', + ); + return matches[0]; +} +function ownFile( + reference: { path: string; sha256: string }, + provider: MetadataObject, +): ImmutableFile { + return { + ...reference, + repository: string(provider['repository'], 'provider repository'), + revision: string(provider['commit'], 'provider commit'), + }; +} + +@Injectable({ providedIn: 'root' }) +export class PublicDataService { + private readonly cache = new CanonicalMetadataCache(); + readonly currentPins = INITIAL_CANONICAL_PINS; + async discoverAll( + signal: AbortSignal, + pins: CanonicalPins = this.currentPins, + ): Promise { + this.cache.begin(pins); + const reader = new CanonicalMetadataReader( + (input, init) => fetch(input, init), + this.cache, + signal, + ); + const indexes = await readCanonicalIndexes(pins, reader); + const suggestions = await discoverRepresentations( + undefined, + indexes, + reader, + ); + return { indexes, suggestions, metadataBytes: reader.bytes }; + } + async discover( + source: SourceField, + signal: AbortSignal, + pins: CanonicalPins = this.currentPins, + ): Promise { + this.cache.begin(pins); + const reader = new CanonicalMetadataReader( + (input, init) => fetch(input, init), + this.cache, + signal, + ); + const indexes = await readCanonicalIndexes(pins, reader); + const suggestions = await discoverRepresentations(source, indexes, reader); + return { indexes, suggestions, metadataBytes: reader.bytes }; + } + clearCache(): void { + this.cache.clear(); + } + + /** Saves the exact pending plan in the existing query shape, without running it. */ + scenario( + source: SourceField, + discovery: PublicDataDiscovery, + suggestion: PublicDataSuggestion, + bounds: Pick, + ): IQueryDef { + const contract = suggestion.contract; + if ( + !contract || + !suggestion.matchesSource || + !sameSource(source, contract.source) || + !suggestion.rights || + !suggestion.snapshot + ) + throw new Error( + 'An exact verified source contract is needed before saving its pending scenario.', + ); + const sourceProvider = array( + discovery.indexes.directory['databases'], + 'source databases', + ) + .map((value) => object(value, 'database')) + .find( + (value) => + value['repository'] === source.schema.repository && + value['commit'] === source.schema.revision, + ); + if (!sourceProvider) + throw new Error( + 'This declared user source has no pinned Directory runtime mapping.', + ); + const driver = recordset(sourceProvider, source.entity); + const bridge = recordset( + suggestion.provider, + contract.bridge.table, + 'name', + ); + const target = recordset(suggestion.provider, contract.target.entity); + const sourceDatabase = string(sourceProvider['localId'], 'user database'); + const targetDatabase = string( + suggestion.provider['localId'], + 'public database', + ); + const base = PUBLIC_DATA_OVDB_BASES.find( + (value) => + sourceProvider['apiUrl'] === + `${value}/v1/databases/${sourceDatabase}` && + suggestion.provider['apiUrl'] === + `${value}/v1/databases/${targetDatabase}`, + ); + if (!base) + throw new Error( + 'The sources have no jointly approved bounded OVDB route.', + ); + const userName = string(driver['name'], 'user recordset'); + const bridgeName = string(bridge['name'], 'bridge recordset'); + const targetName = string(target['name'], 'target recordset'); + const physicalFields = (set: MetadataObject): string[] => + array(set['fields'], 'physical fields').map((value) => + string(object(value, 'physical field')['name'], 'physical field name'), + ); + const boundsConfig: BoundedFederation = { + ...bounds, + identifierKind: 'place', + identifierLimit: 100, + resultRows: 5000, + bytes: 5 * 1024 * 1024, + timeoutMs: 10000, + sources: [ + { database: sourceDatabase, name: userName, keyField: source.property }, + { + database: targetDatabase, + name: bridgeName, + keyField: contract.bridge.raw_label_column, + parent: { + database: sourceDatabase, + name: userName, + field: source.property, + }, + }, + { + database: targetDatabase, + name: targetName, + keyField: contract.target.property, + parent: { + database: targetDatabase, + name: bridgeName, + field: contract.bridge.target_key_column, + }, + }, + ], + }; + validateBounds(boundsConfig); + const query = { + from: { + database: sourceDatabase, + name: userName, + alias: 'u', + joins: [ + { + type: 'left', + from: { + database: targetDatabase, + name: bridgeName, + alias: 'b', + joins: [ + { + type: 'left', + from: { + database: targetDatabase, + name: targetName, + alias: 'p', + }, + on: [ + { + left: { + source: 'b', + field: contract.bridge.target_key_column, + }, + op: '==', + right: { source: 'p', field: contract.target.property }, + }, + ], + }, + ], + }, + on: [ + { + left: { source: 'u', field: source.property }, + op: '==', + right: { source: 'b', field: contract.bridge.raw_label_column }, + }, + ], + }, + ], + }, + }; + const decision = contract.decision.document; + const meaning = contract.target.binding.meaning.document; + if ( + !decision.repository || + !decision.revision || + !meaning.repository || + !meaning.revision + ) + throw new Error('External decision/meaning revisions are missing.'); + return { + id: `public-data-${crypto.randomUUID()}`, + title: `${source.entity}.${source.property} public lookup`, + request: { + queryType: QueryType.DTQL, + text: JSON.stringify(query, null, 2), + } as IQueryDef['request'], + federation: { + ovdbBaseUrl: base, + bounds: boundsConfig, + tables: [ + { + database: sourceDatabase, + name: userName, + fields: physicalFields(driver), + }, + { + database: targetDatabase, + name: bridgeName, + fields: physicalFields(bridge), + }, + { + database: targetDatabase, + name: targetName, + fields: physicalFields(target), + }, + ], + }, + publicData: { + source, + canonical: discovery.indexes.pins, + attachment: suggestion.attachment, + snapshot: ownFile(contract.target.snapshot, suggestion.provider), + model: ownFile(contract.target.model, suggestion.provider), + meaning: { + ...meaning, + repository: meaning.repository, + revision: meaning.revision, + }, + decision: { + ...decision, + repository: decision.repository, + revision: decision.revision, + }, + decisionScope: contract.decision.scope, + namespace: contract.target.namespace, + projection: 'identity', + equality: 'utf8-byte-exact', + rights: { + ...suggestion.rights, + attribution: JSON.stringify( + suggestion.snapshot['licences'] ?? + suggestion.snapshot['licenses'] ?? + 'See the pinned source snapshot and source licence.', + ), + }, + observedAt: new Date().toISOString(), + eligible: suggestion.eligible, + unavailableReason: suggestion.reason, + }, + }; + } +} diff --git a/libs/datatug/main/src/lib/queries/public-data/representation-contract.schema.json b/libs/datatug/main/src/lib/queries/public-data/representation-contract.schema.json new file mode 100644 index 00000000..27b7bc2a --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/representation-contract.schema.json @@ -0,0 +1,294 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "format": { + "const": "ovdb-representation-contract/1" + }, + "contracts": { + "type": "array", + "minItems": 1, + "maxItems": 32, + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "source": { + "type": "object", + "additionalProperties": false, + "properties": { + "schema": { + "$ref": "#/$defs/ref" + }, + "module": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_]*$", + "maxLength": 128 + }, + "entity": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_]*$", + "maxLength": 128 + }, + "property": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_]*$", + "maxLength": 128 + }, + "datatype": { + "const": "string" + }, + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + } + }, + "required": [ + "schema", + "module", + "entity", + "property", + "datatype", + "namespace" + ] + }, + "target": { + "type": "object", + "additionalProperties": false, + "properties": { + "snapshot": { + "$ref": "#/$defs/ref" + }, + "model": { + "$ref": "#/$defs/ref" + }, + "module": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_]*$", + "maxLength": 128 + }, + "entity": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_]*$", + "maxLength": 128 + }, + "property": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_]*$", + "maxLength": 128 + }, + "datatype": { + "const": "string" + }, + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "binding": { + "type": "object", + "additionalProperties": false, + "properties": { + "document": { + "$ref": "#/$defs/ref" + }, + "concept": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + }, + "role": { + "const": "identifier" + }, + "meaning": { + "type": "object", + "additionalProperties": false, + "properties": { + "document": { + "$ref": "#/$defs/ref" + }, + "concept": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + } + }, + "required": [ + "document", + "concept" + ] + } + }, + "required": [ + "document", + "concept", + "role", + "meaning" + ] + }, + "keys": { + "$ref": "#/$defs/ref" + } + }, + "required": [ + "snapshot", + "model", + "module", + "entity", + "property", + "datatype", + "namespace", + "binding", + "keys" + ] + }, + "bridge": { + "type": "object", + "additionalProperties": false, + "properties": { + "artifact": { + "$ref": "#/$defs/ref" + }, + "table": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_]*$", + "maxLength": 128 + }, + "raw_label_column": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_]*$", + "maxLength": 128 + }, + "target_key_column": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_]*$", + "maxLength": 128 + }, + "serving_identity_column": { + "type": "string", + "pattern": "^[A-Za-z][A-Za-z0-9_]*$", + "maxLength": 128 + } + }, + "required": [ + "artifact", + "table", + "raw_label_column", + "target_key_column" + ] + }, + "policy": { + "type": "object", + "additionalProperties": false, + "properties": { + "transform": { + "const": "identity" + }, + "equality": { + "const": "utf8-byte-exact" + }, + "cardinality": { + "const": "zero-or-one" + }, + "unmatched": { + "const": "exception" + }, + "collision": { + "const": "ineligible" + } + }, + "required": [ + "transform", + "equality", + "cardinality", + "unmatched", + "collision" + ] + }, + "decision": { + "type": "object", + "additionalProperties": false, + "properties": { + "document": { + "$ref": "#/$defs/ref" + }, + "scope": { + "type": "string", + "minLength": 1, + "maxLength": 1024 + } + }, + "required": [ + "document", + "scope" + ] + } + }, + "required": [ + "source", + "target", + "bridge", + "policy", + "decision" + ] + } + } + }, + "required": [ + "format", + "contracts" + ], + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://openvaultdb.com/schemas/representation-contract-1.json", + "$defs": { + "ref": { + "type": "object", + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1, + "maxLength": 1024, + "pattern": "^[A-Za-z0-9_./-]+$", + "allOf": [ + { + "not": { + "pattern": "(^|/)(\\.|\\.\\.|\\.[gG][iI][tT])(/|$)" + } + }, + { + "not": { + "pattern": "(^/|/$|//)" + } + } + ] + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "repository": { + "type": "string", + "pattern": "^https://github\\.com/[a-z0-9_.-]+/[a-z0-9_.-]+$" + }, + "revision": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + } + }, + "required": [ + "path", + "sha256" + ], + "dependentRequired": { + "repository": [ + "revision" + ], + "revision": [ + "repository" + ] + } + } + } +} diff --git a/libs/datatug/main/src/lib/queries/public-data/representation-discovery.spec.ts b/libs/datatug/main/src/lib/queries/public-data/representation-discovery.spec.ts new file mode 100644 index 00000000..dd2ba11c --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/representation-discovery.spec.ts @@ -0,0 +1,192 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { describe, expect, it, vi } from 'vitest'; +import { + CanonicalMetadataCache, + CanonicalMetadataReader, + INITIAL_CANONICAL_PINS, + sha256, + type CanonicalIndexes, +} from './canonical-metadata'; +import { + discoverRepresentations, + parseMeaningDocument, + type RepresentationContract, + type SourceField, +} from './representation-discovery'; + +function fixtureFiles(): Record { + return Object.fromEntries( + [ + 'contract.json', + 'source.modelspec.json', + 'target.modelspec.json', + 'target.meaning.json', + 'core.meaning.json', + 'snapshot.json', + 'bridge.json', + 'keys.json', + 'decision.md', + ].map((name) => [ + name, + readFileSync( + resolve( + 'libs/datatug/main/src/lib/queries/fixtures/public-data-fabric/representation', + name, + ), + 'utf8', + ), + ]), + ); +} +async function discover( + files = fixtureFiles(), + change?: (contract: RepresentationContract) => void, +) { + const document = JSON.parse(files['contract.json']) as { + contracts: RepresentationContract[]; + }; + if (change) change(document.contracts[0]); + files['contract.json'] = JSON.stringify(document); + const contract = document.contracts[0]; + const source = contract.source as SourceField; + const provider = { + repository: 'https://github.com/example/provider', + commit: 'a'.repeat(40), + title: 'Synthetic reviewed-helper fixture', + licence: 'fixture-only', + representation_contract: { + path: 'contract.json', + sha256: await sha256(files['contract.json']), + }, + }; + const indexes: CanonicalIndexes = { + pins: INITIAL_CANONICAL_PINS, + bytes: 0, + directory: { databases: [provider] }, + models: { + models: [ + { + repository: source.schema.repository, + commit: source.schema.revision, + files: { json: 'source.modelspec.json' }, + }, + { + repository: provider.repository, + commit: provider.commit, + files: { json: 'target.modelspec.json' }, + licence: 'fixture-only', + }, + ], + }, + meanings: { + graphs: [ + { + repository: provider.repository, + commit: provider.commit, + meaning_files: ['target.meaning.json'], + meaning_licence: 'fixture-only', + depends: [{ id: 'fixture-core', commit: source.schema.revision }], + }, + { + id: 'fixture-core', + repository: source.schema.repository, + commit: source.schema.revision, + meaning_files: ['core.meaning.json'], + }, + ], + }, + }; + const http: typeof fetch = vi.fn(async (input, options) => { + expect(options?.redirect).toBe('error'); + const path = String(input).split('/').at(-1) ?? ''; + return new Response(files[path] ?? '', { + status: files[path] === undefined ? 404 : 200, + }); + }); + const reader = new CanonicalMetadataReader( + http, + new CanonicalMetadataCache(), + new AbortController().signal, + ); + return discoverRepresentations(source, indexes, reader); +} + +describe('closed reviewed-helper consumer fixture, never production admission', () => { + it('matches reviewed strict Unicode and single-document YAML controls', () => { + const text = 'format: meaning/draft-1\nconcepts: []\n'; + for (const extra of ['---\nformat: broken\n', '---\n', '---\n[']) + expect(() => parseMeaningDocument(text + extra)).toThrow(); + expect( + parseMeaningDocument(text + '...\n# trailing comment\n'), + ).toMatchObject({ format: 'meaning/draft-1' }); + expect(() => + parseMeaningDocument( + 'format: meaning/draft-1\nFORMAT: broken\nconcepts: []', + ), + ).toThrow(/Aliased/); + expect(() => + parseMeaningDocument( + 'format: meaning/draft-1\nconcepts: []\nx: "\\ud800"', + ), + ).toThrow(); + }); + it('verifies the pinned companion graph but remains unavailable without canonical publication', async () => { + const suggestions = await discover(); + expect(suggestions).toHaveLength(1); + expect(suggestions[0]).toMatchObject({ + matchesSource: true, + eligible: false, + }); + expect(suggestions[0].reason).toMatch(/publication is pending/); + }); + it('rejects exact wrong property, revision, checksum, namespace and reference scope', async () => { + const changes: ((contract: RepresentationContract) => void)[] = [ + (c) => Object.assign(c.source, { property: 'country' }), + (c) => Object.assign(c.source.schema, { revision: 'c'.repeat(40) }), + (c) => Object.assign(c.target.model, { sha256: '0'.repeat(64) }), + (c) => Object.assign(c.target, { namespace: 'iso-3166-1-alpha-2 ' }), + (c) => + Object.assign(c.target.model, { + repository: 'https://github.com/example/provider', + revision: 'a'.repeat(40), + }), + ]; + for (const change of changes) { + const [suggestion] = await discover(fixtureFiles(), change); + expect(suggestion.eligible).toBe(false); + expect(suggestion.contract).toBeUndefined(); + expect(suggestion.reason).not.toMatch(/publication is pending/); + } + }); + it('rejects a hashed bridge collision and binding to the wrong exact property', async () => { + for (const name of ['bridge.json', 'target.meaning.json']) { + const files = fixtureFiles(); + const value = JSON.parse(files[name]); + if (name === 'bridge.json') value.rows.push({ ...value.rows[0] }); + else value.concepts[0].bindings[0].property = 'not_iso'; + files[name] = JSON.stringify(value); + const hash = await sha256(files[name]); + const snapshot = JSON.parse(files['snapshot.json']); + const artifact = snapshot.artifacts.find( + (a: { path: string }) => a.path === name, + ); + if (artifact) artifact.sha256 = hash; + files['snapshot.json'] = JSON.stringify(snapshot); + const snapshotHash = await sha256(files['snapshot.json']); + const [suggestion] = await discover(files, (c) => { + Object.assign(c.target.snapshot, { sha256: snapshotHash }); + Object.assign( + name === 'bridge.json' + ? c.bridge.artifact + : c.target.binding.document, + { sha256: hash }, + ); + }); + expect(suggestion.reason).toMatch( + name === 'bridge.json' ? /collision/ : /exact identifier/, + ); + expect(suggestion.eligible).toBe(false); + } + }); +}); diff --git a/libs/datatug/main/src/lib/queries/public-data/representation-discovery.ts b/libs/datatug/main/src/lib/queries/public-data/representation-discovery.ts new file mode 100644 index 00000000..6e818016 --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/representation-discovery.ts @@ -0,0 +1,510 @@ +import Ajv2020 from 'ajv/dist/2020'; +import { parseAllDocuments } from 'yaml'; +import schema from './representation-contract.schema.json'; +import { + array, + object, + string, + strictJson, + immutableUrl, + exactFields, + type CanonicalIndexes, + type CanonicalMetadataReader, + type ImmutableFile, + type MetadataObject, +} from './canonical-metadata'; + +interface Reference { + readonly path: string; + readonly sha256: string; + readonly repository?: string; + readonly revision?: string; +} +export interface SourceField { + readonly schema: ImmutableFile; + readonly module: string; + readonly entity: string; + readonly property: string; + readonly datatype: 'string'; + readonly namespace: string; +} +export interface RepresentationContract { + readonly source: Omit & { readonly schema: Reference }; + readonly target: { + readonly snapshot: Reference; + readonly keys: Reference; + readonly model: Reference; + readonly module: string; + readonly entity: string; + readonly property: string; + readonly datatype: 'string'; + readonly namespace: string; + readonly binding: { + readonly document: Reference; + readonly concept: string; + readonly role: 'identifier'; + readonly meaning: { + readonly document: Reference; + readonly concept: string; + }; + }; + }; + readonly bridge: { + readonly artifact: Reference; + readonly table: string; + readonly raw_label_column: string; + readonly target_key_column: string; + readonly serving_identity_column?: string; + }; + readonly policy: { + readonly transform: 'identity'; + readonly equality: 'utf8-byte-exact'; + readonly cardinality: 'zero-or-one'; + readonly unmatched: 'exception'; + readonly collision: 'ineligible'; + }; + readonly decision: { readonly document: Reference; readonly scope: string }; +} +export interface PublicDataSuggestion { + readonly provider: MetadataObject; + readonly attachment: ImmutableFile; + readonly contract?: RepresentationContract; + readonly matchesSource: boolean; + readonly eligible: boolean; + readonly reason: string; + readonly snapshot?: MetadataObject; + readonly rights?: { + readonly source: string; + readonly model: string; + readonly meaning: string; + }; +} + +const validate = new Ajv2020({ allErrors: true, strict: true }).compile(schema); +/** This consumer proposal does not make unfrozen companion metadata executable. */ +export const REPRESENTATION_PUBLICATION_BLOCKER = + 'Scoped representation publication is pending independent contract review and canonical Directory/publisher support. Lookup is unavailable.'; + +export function sameSource( + scope: SourceField, + source: RepresentationContract['source'], +): boolean { + return ( + scope.schema.repository === source.schema.repository && + scope.schema.revision === source.schema.revision && + scope.schema.path === source.schema.path && + scope.schema.sha256 === source.schema.sha256 && + scope.module === source.module && + scope.entity === source.entity && + scope.property === source.property && + scope.datatype === source.datatype && + scope.namespace === source.namespace + ); +} + +function referenceFile( + reference: Reference, + provider: MetadataObject, + external: boolean, +): ImmutableFile { + const owner = string(provider['repository'], 'provider repository'); + if ( + (external && (!reference.repository || !reference.revision)) || + (!external && + (reference.repository !== undefined || + reference.revision !== undefined)) || + reference.repository === owner + ) + throw new Error( + 'Incorrect own-provider/external immutable reference scope.', + ); + const file = { + ...reference, + repository: reference.repository ?? owner, + revision: + reference.revision ?? string(provider['commit'], 'provider commit'), + }; + immutableUrl(file); + return file; +} +function findRecord( + records: readonly unknown[], + repository: string, + revision: string, + description: string, +): MetadataObject { + const matches = records + .map((value) => { + const record = object(value, description); + exactFields(record, [ + 'repository', + 'commit', + 'files', + 'id', + 'licence', + 'meaning_files', + 'meaning_licence', + 'depends', + ]); + return record; + }) + .filter( + (record) => + record['repository'] === repository && record['commit'] === revision, + ); + if (matches.length !== 1) + throw new Error( + `Canonical ${description} is missing or ambiguous at this immutable revision.`, + ); + return matches[0]; +} +function modelProperty( + model: MetadataObject, + module: string, + entity: string, + property: string, +): MetadataObject { + exactFields(model, ['modelspec', 'module', 'entities']); + if (model['modelspec'] !== '1.0-draft') + throw new Error('Unsupported canonical model format.'); + exactFields(object(model['module'], 'model module'), ['name']); + if (object(model['module'], 'model module')['name'] !== module) + throw new Error('Wrong canonical model module.'); + const entityRecord = object( + object(model['entities'], 'entities')[entity], + 'entity', + ); + exactFields(entityRecord, ['properties']); + const value = object(entityRecord['properties'], 'properties')[property]; + const result = object(value, 'source-qualified property'); + exactFields(result, ['type', 'required']); + if (result['type'] !== 'string') + throw new Error( + 'The exact model property does not declare the raw string representation.', + ); + return result; +} +export function parseMeaningDocument(text: string): MetadataObject { + const documents = parseAllDocuments(text, { uniqueKeys: true, strict: true }); + if ( + documents.length !== 1 || + documents[0].errors.length || + documents[0].warnings.length + ) + throw new Error( + 'The canonical meaning document has invalid or multiple YAML documents.', + ); + const value = object( + strictJson(JSON.stringify(documents[0].toJS({ maxAliasCount: 0 }))), + 'meaning document', + ); + exactFields(value, ['format', 'concepts']); + for (const item of array(value['concepts'], 'meaning concepts')) { + const concept = object(item, 'meaning concept'); + exactFields(concept, ['id', 'extends', 'bindings']); + if (concept['bindings']) + for (const binding of array(concept['bindings'], 'meaning bindings')) + exactFields(object(binding, 'meaning binding'), [ + 'model', + 'property', + 'role', + ]); + } + return value; +} + +async function verifyContract( + contract: RepresentationContract, + provider: MetadataObject, + indexes: CanonicalIndexes, + reader: CanonicalMetadataReader, + attachment: ImmutableFile, +): Promise<{ + snapshot: MetadataObject; + rights: PublicDataSuggestion['rights']; +}> { + const target = contract.target; + const modelRecords = array(indexes.models['models'], 'model registry'); + const graphRecords = array(indexes.meanings['graphs'], 'meaning registry'); + const sourceFile = referenceFile(contract.source.schema, provider, true); + const sourceRecord = findRecord( + modelRecords, + sourceFile.repository, + sourceFile.revision, + 'source model', + ); + if ( + object(sourceRecord['files'], 'source model files')['json'] !== + sourceFile.path + ) + throw new Error( + 'The source schema is not the registered canonical JSON model.', + ); + const providerRepository = string( + provider['repository'], + 'provider repository', + ); + const providerRevision = string(provider['commit'], 'provider commit'); + const targetRecord = findRecord( + modelRecords, + providerRepository, + providerRevision, + 'target model', + ); + const graphRecord = findRecord( + graphRecords, + providerRepository, + providerRevision, + 'target meaning graph', + ); + const modelFile = referenceFile(target.model, provider, false); + if ( + object(targetRecord['files'], 'target model files')['json'] !== + modelFile.path + ) + throw new Error('Wrong registered target model path.'); + const bindingFile = referenceFile(target.binding.document, provider, false); + if ( + !array(graphRecord['meaning_files'], 'meaning files').includes( + bindingFile.path, + ) + ) + throw new Error('Wrong registered meaning binding document.'); + const coreFile = referenceFile( + target.binding.meaning.document, + provider, + true, + ); + const coreRecord = findRecord( + graphRecords, + coreFile.repository, + coreFile.revision, + 'canonical meaning', + ); + const corePaths = array(coreRecord['meaning_files'], 'core meaning files'); + if ( + !corePaths.some( + (path) => + path === coreFile.path || + (typeof path === 'string' && + path.startsWith('*') && + !coreFile.path.includes('/') && + coreFile.path.endsWith(path.slice(1))), + ) + ) + throw new Error('Wrong canonical meaning file.'); + const dependencies = array(graphRecord['depends'], 'meaning dependencies'); + if ( + !dependencies.some((value) => { + const record = object(value, 'meaning dependency'); + return ( + record['id'] === coreRecord['id'] && + record['commit'] === coreFile.revision + ); + }) + ) + throw new Error('Unregistered pinned canonical meaning dependency.'); + const ancestry = [immutableUrl(attachment)]; + const source = await reader.json(sourceFile, ancestry); + const model = await reader.json(modelFile, ancestry); + modelProperty( + source, + contract.source.module, + contract.source.entity, + contract.source.property, + ); + modelProperty(model, target.module, target.entity, target.property); + for (const field of [ + contract.bridge.raw_label_column, + contract.bridge.target_key_column, + ...(contract.bridge.serving_identity_column + ? [contract.bridge.serving_identity_column] + : []), + ]) + modelProperty(model, target.module, contract.bridge.table, field); + const binding = parseMeaningDocument( + await reader.text(bindingFile, ancestry), + ); + const core = parseMeaningDocument(await reader.text(coreFile, ancestry)); + if ( + binding['format'] !== 'meaning/draft-1' || + core['format'] !== 'meaning/draft-1' + ) + throw new Error('Unsupported canonical meaning format.'); + if ( + !array(core['concepts'], 'canonical concepts').some( + (value) => + object(value, 'canonical concept')['id'] === + target.binding.meaning.concept, + ) + ) + throw new Error('Missing canonical target concept.'); + const concept = array(binding['concepts'], 'target concepts') + .map((value) => object(value, 'concept')) + .find((value) => value['id'] === target.binding.concept); + const expectedMeaning = `meaning://${coreFile.repository.slice('https://'.length)}/${target.binding.meaning.concept}?ref=${coreFile.revision}`; + if ( + !concept || + concept['extends'] !== expectedMeaning || + !array(concept['bindings'], 'bindings').some((value) => { + const record = object(value, 'binding'); + return ( + record['model'] === `modelspec:///${target.module}.${target.entity}` && + record['property'] === target.property && + record['role'] === 'identifier' + ); + }) + ) + throw new Error( + 'The binding does not implement this exact identifier property and canonical meaning.', + ); + const snapshotFile = referenceFile(target.snapshot, provider, false); + const keysFile = referenceFile(target.keys, provider, false); + const bridgeFile = referenceFile(contract.bridge.artifact, provider, false); + const snapshot = await reader.json(snapshotFile, ancestry); + exactFields(snapshot, ['generator', 'artifacts']); + const generator = object(snapshot['generator'], 'snapshot generator'); + exactFields(generator, ['repository', 'revision']); + if ( + !/^https:\/\/github\.com\/[a-z0-9_.-]+\/[a-z0-9_.-]+$/.test( + string(generator['repository'], 'generator repository'), + ) || + !/^[a-f0-9]{40}$/.test(string(generator['revision'], 'generator revision')) + ) + throw new Error('Mutable snapshot generator provenance.'); + const artifacts = array(snapshot['artifacts'], 'snapshot artifacts'); + for (const file of [keysFile, bridgeFile]) + if ( + !artifacts.some((value) => { + const artifact = object(value, 'artifact'); + exactFields(artifact, ['path', 'sha256']); + return ( + artifact['path'] === file.path && artifact['sha256'] === file.sha256 + ); + }) + ) + throw new Error('Snapshot does not pin the exact bridge/key artifact.'); + const keys = await reader.json(keysFile, ancestry); + exactFields(keys, ['namespace', 'keys']); + if (keys['namespace'] !== target.namespace) + throw new Error('Wrong native-key namespace.'); + const nativeKeys = array(keys['keys'], 'native keys').map((value) => + string(value, 'native key'), + ); + if (new Set(nativeKeys).size !== nativeKeys.length) + throw new Error('Duplicate native identifier keys.'); + const bridge = await reader.json(bridgeFile, ancestry); + exactFields(bridge, ['table', 'rows']); + if (bridge['table'] !== contract.bridge.table) + throw new Error('Wrong physical bridge table.'); + const labels = new Set(); + for (const value of array(bridge['rows'], 'bridge rows')) { + const row = object(value, 'bridge row'); + if (Object.keys(row).sort().join(',') !== 'raw_label,target_key') + throw new Error('Unknown physical bridge export fields.'); + const raw = string(row['raw_label'], 'raw label'); + const key = string(row['target_key'], 'target key'); + if (labels.has(raw) || !nativeKeys.includes(key)) + throw new Error('Raw-label collision or missing native target.'); + labels.add(raw); + } + // Provenance only: immutable bytes do not parse or self-accept a semantic decision. + await reader.text( + referenceFile(contract.decision.document, provider, true), + ancestry, + ); + return { + snapshot, + rights: { + source: string(provider['licence'], 'source licence'), + model: string(targetRecord['licence'], 'model licence'), + meaning: string(graphRecord['meaning_licence'], 'meaning licence'), + }, + }; +} + +/** Diagnostic discovery remains unavailable until reviewed canonical companion publication. */ +export async function discoverRepresentations( + scope: SourceField | undefined, + indexes: CanonicalIndexes, + reader: CanonicalMetadataReader, +): Promise { + if (scope) immutableUrl(scope.schema); + const suggestions: PublicDataSuggestion[] = []; + for (const value of array( + indexes.directory['databases'], + 'Directory databases', + )) { + const provider = object(value, 'Directory provider'); + exactFields(provider, [ + 'repository', + 'commit', + 'representation_contract', + 'licence', + ]); + if (provider['representation_contract'] === undefined) continue; + let attachment: ImmutableFile | undefined; + try { + const ref = object( + provider['representation_contract'], + 'representation attachment', + ); + if (Object.keys(ref).sort().join(',') !== 'path,sha256') + throw new Error( + 'Unsupported Directory representation attachment envelope.', + ); + attachment = referenceFile( + { + path: string(ref['path'], 'attachment path'), + sha256: string(ref['sha256'], 'attachment hash'), + }, + provider, + false, + ); + const document = await reader.json(attachment); + if (!validate(document)) + throw new Error( + 'Unsupported or malformed scoped representation schema.', + ); + for (const contract of document[ + 'contracts' + ] as readonly RepresentationContract[]) { + const matchesSource = scope + ? sameSource(scope, contract.source) + : false; + const verified = await verifyContract( + contract, + provider, + indexes, + reader, + attachment, + ); + suggestions.push({ + provider, + attachment, + contract, + matchesSource, + eligible: false, + reason: matchesSource + ? REPRESENTATION_PUBLICATION_BLOCKER + : 'This contract belongs to a different exact source/schema/property/representation scope.', + ...verified, + }); + } + } catch (error) { + if (attachment) + suggestions.push({ + provider, + attachment, + matchesSource: false, + eligible: false, + reason: + error instanceof Error + ? error.message + : 'Canonical representation unavailable.', + }); + else throw error; + } + } + return suggestions; +} diff --git a/libs/datatug/main/src/lib/queries/public-data/strict-json.ts b/libs/datatug/main/src/lib/queries/public-data/strict-json.ts new file mode 100644 index 00000000..6aa854ae --- /dev/null +++ b/libs/datatug/main/src/lib/queries/public-data/strict-json.ts @@ -0,0 +1,81 @@ +/** Strict JSON preserves raw Unicode and refuses duplicate/consumed case aliases. */ +export function strictJson(text: string): unknown { + let cursor = 0; + const whitespace = (): void => { + while (/\s/.test(text[cursor] ?? '') && cursor < text.length) cursor++; + }; + const token = (): string => { + const start = cursor++; + while (cursor < text.length) { + const char = text[cursor++]; + if (char === '\\') { + cursor++; + continue; + } + if (char === '"') { + const value: unknown = JSON.parse(text.slice(start, cursor)); + if (typeof value !== 'string') throw new Error('Invalid JSON string.'); + for (let i = 0; i < value.length; i++) { + const code = value.charCodeAt(i); + if (code >= 0xd800 && code <= 0xdbff) { + const next = value.charCodeAt(++i); + if (!(next >= 0xdc00 && next <= 0xdfff)) + throw new Error('Unpaired Unicode surrogate.'); + } else if (code >= 0xdc00 && code <= 0xdfff) + throw new Error('Unpaired Unicode surrogate.'); + } + return value; + } + } + throw new Error('Unterminated JSON string.'); + }; + const visit = (depth: number): void => { + if (depth > 32) throw new Error('Metadata nesting exceeds the bound.'); + whitespace(); + if (text[cursor] === '"') { + token(); + return; + } + if (text[cursor] === '{') { + cursor++; + whitespace(); + const names = new Set(); + while (text[cursor] !== '}') { + if (text[cursor] !== '"') throw new Error('Invalid JSON object.'); + const name = token(); + if (names.has(name)) throw new Error('Duplicate JSON field.'); + names.add(name); + whitespace(); + if (text[cursor++] !== ':') throw new Error('Invalid JSON object.'); + visit(depth + 1); + whitespace(); + if (text[cursor] !== ',') break; + cursor++; + whitespace(); + } + if (text[cursor++] !== '}') throw new Error('Invalid JSON object.'); + return; + } + if (text[cursor] === '[') { + cursor++; + whitespace(); + while (text[cursor] !== ']') { + visit(depth + 1); + whitespace(); + if (text[cursor] !== ',') break; + cursor++; + whitespace(); + } + if (text[cursor++] !== ']') throw new Error('Invalid JSON array.'); + return; + } + const start = cursor; + while (cursor < text.length && !/[\s,}\]]/.test(text[cursor])) cursor++; + if (start === cursor) throw new Error('Invalid JSON value.'); + JSON.parse(text.slice(start, cursor)); + }; + visit(0); + whitespace(); + if (cursor !== text.length) throw new Error('Trailing JSON data.'); + return JSON.parse(text) as unknown; +} diff --git a/libs/datatug/main/src/lib/queries/query/page/query-page.component.html b/libs/datatug/main/src/lib/queries/query/page/query-page.component.html index 6caf12c7..18863089 100644 --- a/libs/datatug/main/src/lib/queries/query/page/query-page.component.html +++ b/libs/datatug/main/src/lib/queries/query/page/query-page.component.html @@ -29,6 +29,56 @@ + @if (queryDef()?.publicData; as saved) { + Saved public-data lookup +

+ {{ saved.source.entity }}.{{ saved.source.property }} → + {{ saved.namespace }}. Source release date and coverage are not + recorded for this saved plan. +

+

{{ saved.unavailableReason }}

+
+ Saved provenance and immutable revisions +

+ {{ saved.source.schema.repository }} at + {{ saved.source.schema.revision }} · {{ saved.source.module }}.{{ + saved.source.entity + }}.{{ saved.source.property }} · raw {{ saved.source.namespace }} → + {{ saved.namespace }} +

+

+ Directory {{ saved.canonical.directory.revision }} · ModelSpec + {{ saved.model.revision }} · Meaning {{ saved.meaning.revision }} · + Snapshot {{ saved.snapshot.sha256 }} +

+

+ Identity projection · UTF8 byte-exact equality · decision + {{ saved.decisionScope }} at {{ saved.decision.revision }} +

+

+ Source licence {{ saved.rights.source }} · model + {{ saved.rights.model }} · meaning {{ saved.rights.meaning }} · + {{ saved.rights.attribution }} +

+
+ @if (publicDataRevisionChanges().length) { +

+ Changed canonical revisions: + {{ publicDataRevisionChanges().join(', ') }}. Saved results retain + their original snapshot; reopening runs no lookup. +

+ Acknowledge changed revisions + } + + } { sessionStorage.clear(); }); + it('reopens saved immutable provenance without lookup, exposes changed revisions and rejects a stored eligibility flag', async () => { + const file = INITIAL_CANONICAL_PINS.directory; + const saved: PublicDataScenario = { source: { schema: file, module: 'fixture', entity: 'Customer', property: 'Country', datatype: 'string', namespace: 'fixture-labels' }, canonical: { ...INITIAL_CANONICAL_PINS, directory: { ...file, revision: 'b'.repeat(40) } }, attachment: file, snapshot: file, model: file, meaning: file, decision: file, decisionScope: 'fixture-only', namespace: 'fixture-native', projection: 'identity', equality: 'utf8-byte-exact', rights: { source: 'fixture', model: 'fixture', meaning: 'fixture', attribution: 'fixture' }, observedAt: '2026-10-05', eligible: true, unavailableReason: 'pending' }; + const definition: IQueryDef = { ...queryDef, id: 'saved-fixture', federation: { ovdbBaseUrl: 'https://demodb.dev/ovdb', tables: [] }, publicData: saved }; + component = await createComponent({}, definition); + expect(component.queryDef()?.publicData).toEqual(saved); + expect(component.publicDataRevisionChanges()).toEqual(['directory']); + expect(federatedRunMock).not.toHaveBeenCalled(); + component.runQuery(); + expect(component.runError()).toMatch(/revisions changed/); + component.publicDataRevisionAcknowledged.set(true); component.runQuery(); + expect(component.runError()).toBe(SAVED_SCENARIO_PUBLICATION_BLOCKER); + expect(federatedRunMock).not.toHaveBeenCalled(); + }); + it('runs a federated query in the browser and exposes its download and lookup progress', async () => { const definition: IQueryDef = { ...queryDef, diff --git a/libs/datatug/main/src/lib/queries/query/page/query-page.component.ts b/libs/datatug/main/src/lib/queries/query/page/query-page.component.ts index 3a00f391..3ba0bad4 100644 --- a/libs/datatug/main/src/lib/queries/query/page/query-page.component.ts +++ b/libs/datatug/main/src/lib/queries/query/page/query-page.component.ts @@ -1,4 +1,6 @@ import { HttpErrorResponse } from '@angular/common/http'; +import { INITIAL_CANONICAL_PINS } from '../../public-data/canonical-metadata'; +import { scenarioRevisionChanges, SAVED_SCENARIO_PUBLICATION_BLOCKER } from '../../public-data/public-data-scenario'; import { ChangeDetectorRef, Component, @@ -476,6 +478,11 @@ export class QueryPageComponent implements OnDestroy, ViewDidEnter { public readonly accessBlockers = signal([]); public readonly runError = signal(undefined); public readonly runResult = signal(undefined); + public readonly publicDataRevisionAcknowledged = signal(false); + public readonly publicDataRevisionChanges = computed(() => { + const saved = this.queryDef()?.publicData; + return saved ? scenarioRevisionChanges(saved, INITIAL_CANONICAL_PINS) : []; + }); public readonly resultTotalRows = computed(() => this.runResult()?.totalRows ?? this.runResult()?.recordset.rows.length ?? 0); public readonly resultHasMore = computed(() => this.runResult()?.hasMore === true); public readonly visibleResultRows = computed(() => { @@ -636,7 +643,7 @@ export class QueryPageComponent implements OnDestroy, ViewDidEnter { this.queryState = queryState; // Signal write (zoneless-safe, unlike the plain-field write just // above) — see `queryDef`'s own doc comment. - if (this.queryDef()?.id !== queryState.def?.id) this.ovdbToken.set(''); + if (this.queryDef()?.id !== queryState.def?.id) { this.ovdbToken.set(''); this.publicDataRevisionAcknowledged.set(false); } if (queryState.def && this.queryDef()?.id !== queryState.def.id) this.federatedMode.set(federatedVisibleMode(queryState.def).defaultMode); this.queryDef.set(queryState.def); if (this.queryState.environments && !this.queryState.activeEnv) { @@ -1495,6 +1502,14 @@ export class QueryPageComponent implements OnDestroy, ViewDidEnter { this.resultPageIndex.set(0); this.resultPageRows.set([]); const definition = this.queryDef(); + if (definition?.publicData && this.publicDataRevisionChanges().length && !this.publicDataRevisionAcknowledged()) { + this.runError.set('Canonical metadata revisions changed. Inspect the saved pins and explicitly acknowledge before re-execution.'); + return; + } + if (definition?.publicData) { + this.runError.set(SAVED_SCENARIO_PUBLICATION_BLOCKER); + return; + } if (definition?.federation) { if (this.running()) return; this.running.set(true); diff --git a/libs/datatug/main/src/lib/routes/datatug-routing-proj.ts b/libs/datatug/main/src/lib/routes/datatug-routing-proj.ts index fd98bd61..33746139 100644 --- a/libs/datatug/main/src/lib/routes/datatug-routing-proj.ts +++ b/libs/datatug/main/src/lib/routes/datatug-routing-proj.ts @@ -68,6 +68,10 @@ const emptyShellProjectRoutes: Routes = [ ]; export const datatugProjectRoutes: Routes = [ + { + path: 'public-data', + loadComponent: () => import('../queries/public-data/public-data-page.component').then((m) => m.PublicDataPageComponent), + }, { path: 'chat', loadComponent: () => diff --git a/libs/datatug/main/src/lib/services/nav/datatug-nav.service.ts b/libs/datatug/main/src/lib/services/nav/datatug-nav.service.ts index 02619b87..310114c9 100644 --- a/libs/datatug/main/src/lib/services/nav/datatug-nav.service.ts +++ b/libs/datatug/main/src/lib/services/nav/datatug-nav.service.ts @@ -31,6 +31,7 @@ export type ProjectTopLevelPage = | 'environments' | 'servers' | 'queries' + | 'public-data' | 'query' | 'tags' | 'variables' diff --git a/package.json b/package.json index ed83012b..797a442c 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,8 @@ "posthog-js": "1.357.0", "rxjs": "7.8.2", "tabulator-tables": "6.3.1", - "tslib": "2.8.1" + "tslib": "2.8.1", + "yaml": "2.9.0" }, "devDependencies": { "@analogjs/vite-plugin-angular": "2.7.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 75ae03d0..b5320a98 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -208,6 +208,9 @@ importers: tslib: specifier: 2.8.1 version: 2.8.1 + yaml: + specifier: 2.9.0 + version: 2.9.0 devDependencies: '@analogjs/vite-plugin-angular': specifier: 2.7.0