diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index 05ec121213d..ea2e4b5ec85 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -12,22 +12,23 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "RawViewPrimaryKeyDefV10.g.h" #include "RawTableDefV10.g.h" -#include "RawViewDefV10.g.h" -#include "RawScheduleDefV10.g.h" -#include "CaseConversionPolicy.g.h" -#include "RawRowLevelSecurityDefV9.g.h" -#include "RawHttpRouteDefV10.g.h" -#include "RawReducerDefV10.g.h" #include "Typespace.g.h" #include "RawLifeCycleReducerDefV10.g.h" -#include "ExplicitNames.g.h" -#include "RawHttpHandlerDefV10.g.h" +#include "RawReducerDefV10.g.h" #include "RawTypeDefV10.g.h" +#include "RawRowLevelSecurityDefV9.g.h" +#include "RawHttpRouteDefV10.g.h" +#include "RawSubmoduleV10.g.h" +#include "RawViewDefV10.g.h" +#include "ExplicitNames.g.h" #include "RawProcedureDefV10.g.h" +#include "CaseConversionPolicy.g.h" +#include "RawScheduleDefV10.g.h" +#include "RawViewPrimaryKeyDefV10.g.h" +#include "RawHttpHandlerDefV10.g.h" namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector, std::vector) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawSubmoduleV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawSubmoduleV10.g.h new file mode 100644 index 00000000000..c089a4c7820 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawSubmoduleV10.g.h @@ -0,0 +1,32 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" + +namespace SpacetimeDB::Internal { +struct RawModuleDefV10; +} // namespace SpacetimeDB::Internal + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawSubmoduleV10) { + std::string namespace_; + std::shared_ptr module; + + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, namespace_); + ::SpacetimeDB::bsatn::serialize(writer, *module); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(namespace_, module) +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs index 124cf639b64..61212c98e89 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs @@ -22,6 +22,7 @@ public partial record RawModuleDefV10Section : SpacetimeDB.TaggedEnum<( ExplicitNames ExplicitNames, System.Collections.Generic.List HttpHandlers, System.Collections.Generic.List HttpRoutes, - System.Collections.Generic.List ViewPrimaryKeys + System.Collections.Generic.List ViewPrimaryKeys, + System.Collections.Generic.List Submodules )>; } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawSubmoduleV10.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawSubmoduleV10.g.cs new file mode 100644 index 00000000000..c5e6172da9b --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawSubmoduleV10.g.cs @@ -0,0 +1,36 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + [DataContract] + public sealed partial class RawSubmoduleV10 + { + [DataMember(Name = "namespace")] + public string Namespace; + [DataMember(Name = "module")] + public RawModuleDefV10 Module; + + public RawSubmoduleV10( + string Namespace, + RawModuleDefV10 Module + ) + { + this.Namespace = Namespace; + this.Module = Module; + } + + public RawSubmoduleV10() + { + this.Namespace = ""; + this.Module = new(); + } + } +} diff --git a/crates/bindings-typescript/src/lib/autogen/types.ts b/crates/bindings-typescript/src/lib/autogen/types.ts index 2c2b00143ff..3cee51f03d5 100644 --- a/crates/bindings-typescript/src/lib/autogen/types.ts +++ b/crates/bindings-typescript/src/lib/autogen/types.ts @@ -390,6 +390,9 @@ export const RawModuleDefV10Section = __t.enum('RawModuleDefV10Section', { get ViewPrimaryKeys() { return __t.array(RawViewPrimaryKeyDefV10); }, + get Submodules() { + return __t.array(RawSubmoduleV10); + }, }); export type RawModuleDefV10Section = __Infer; @@ -547,6 +550,14 @@ export const RawSequenceDefV9 = __t.object('RawSequenceDefV9', { }); export type RawSequenceDefV9 = __Infer; +export const RawSubmoduleV10 = __t.object('RawSubmoduleV10', { + namespace: __t.string(), + get module(): any { + return RawModuleDefV10; + }, +}); +export type RawSubmoduleV10 = __Infer; + export const RawTableDefV10 = __t.object('RawTableDefV10', { sourceName: __t.string(), productTypeRef: __t.u32(), diff --git a/crates/bindings-typescript/src/lib/query.ts b/crates/bindings-typescript/src/lib/query.ts index fda31f68f45..ab5c93a7baf 100644 --- a/crates/bindings-typescript/src/lib/query.ts +++ b/crates/bindings-typescript/src/lib/query.ts @@ -2,6 +2,7 @@ import { ConnectionId } from './connection_id'; import { Identity } from './identity'; import type { ColumnIndex, IndexColumns, IndexOpts } from './indexes'; import type { UntypedSchemaDef } from './schema'; +import type { UntypedTableDef } from './table'; import type { UntypedTableSchema } from './table_schema'; import { Timestamp } from './timestamp'; import type { @@ -223,6 +224,25 @@ export type QueryBuilder = { > as Tbl['accessorName']]: TableRef & From; } & {}; +/** + * The type of `q.from` in an `addQuery` callback. + * + * Root-level tables appear as direct properties (same as `QueryBuilder`). + * Declared namespaces appear as sub-objects — each is itself a `QueryBuilder` for that + * namespace's schema, so `q.from..` is fully typed. + * + * When `SchemaDef['namespaces']` is absent or `{}`, no namespace properties appear — + * accessing an undeclared namespace is a compile error. + */ +export type SubscriptionFromBuilder = + QueryBuilder & { + readonly [NS in keyof NonNullable]: NonNullable< + SchemaDef['namespaces'] + >[NS] extends UntypedSchemaDef + ? QueryBuilder[NS]> + : never; + }; + /** * A runtime reference to a table. This materializes the RowExpr for us. * TODO: Maybe add the full SchemaDef to the type signature depending on how joins will work. @@ -335,6 +355,42 @@ export function makeQueryBuilder( return Object.freeze(qb) as QueryBuilder; } +/** + * Builds the `q.from` object for use in `addQuery` callbacks. + * + * Tables whose `sourceName` contains no `.` are placed at the root. + * Tables with a dotted `sourceName` (e.g. `"namespace.table"`) are grouped under a + * sub-object keyed by the namespace alias, with the part after the dot as the + * property key within that namespace. + */ +export function makeFromBuilder( + tables: SchemaDef['tables'] +): SubscriptionFromBuilder { + const result: Record = Object.create(null); + const namespaces: Record> = Object.create( + null + ); + + for (const table of Object.values(tables) as UntypedTableDef[]) { + const dotIdx = table.sourceName.indexOf('.'); + if (dotIdx === -1) { + result[table.accessorName] = createTableRefFromDef(table as any); + } else { + const ns = table.sourceName.slice(0, dotIdx); + const key = table.sourceName.slice(dotIdx + 1); + (namespaces[ns] ??= Object.create(null))[key] = createTableRefFromDef( + table as any + ); + } + } + + for (const [ns, nsTables] of Object.entries(namespaces)) { + result[ns] = Object.freeze(nsTables); + } + + return Object.freeze(result) as unknown as SubscriptionFromBuilder; +} + function createRowExpr( tableDef: TableDef ): RowExpr { @@ -879,7 +935,10 @@ function literalValueToSql(value: unknown): string { } function quoteIdentifier(name: string): string { - return `"${name.replace(/"/g, '""')}"`; + return name + .split('.') + .map(part => `"${part.replace(/"/g, '""')}"`) + .join('.'); } function isLiteralExpr( diff --git a/crates/bindings-typescript/src/lib/reducers.ts b/crates/bindings-typescript/src/lib/reducers.ts index 0eae2adc2a9..27c57f0721b 100644 --- a/crates/bindings-typescript/src/lib/reducers.ts +++ b/crates/bindings-typescript/src/lib/reducers.ts @@ -98,6 +98,12 @@ export interface JwtClaims { readonly fullPayload: JsonObject; } +export type AliasViews = SchemaDef extends { + namespaces: infer NS extends Record; +} + ? { readonly [K in keyof NS]: ReducerCtx } + : {}; + /** * Reducer context parametrized by the inferred Schema */ @@ -113,4 +119,5 @@ export type ReducerCtx = Readonly<{ newUuidV4(): Uuid; newUuidV7(): Uuid; random: Random; + as: AliasViews; }>; diff --git a/crates/bindings-typescript/src/lib/schema.ts b/crates/bindings-typescript/src/lib/schema.ts index 1229aed82b8..eb3821f6606 100644 --- a/crates/bindings-typescript/src/lib/schema.ts +++ b/crates/bindings-typescript/src/lib/schema.ts @@ -7,6 +7,7 @@ import { } from './algebraic_type'; import type { CaseConversionPolicy, + RawSubmoduleV10, RawModuleDefV10, RawModuleDefV10Section, RawScopedTypeNameV10, @@ -42,6 +43,7 @@ export type TableNamesOf = Values< */ export type UntypedSchemaDef = { tables: Record; + namespaces?: Record; }; /** @@ -202,6 +204,7 @@ export class ModuleContext { explicitNames: { entries: [], }, + submodules: [], }; get moduleDef(): ModuleDef { @@ -266,9 +269,19 @@ export class ModuleContext { value: module.caseConversionPolicy, } ); + push( + module.submodules && { + tag: 'Submodules', + value: module.submodules, + } + ); return { sections }; } + addSubmodule(submodule: RawSubmoduleV10) { + this.#moduleDef.submodules.push(submodule); + } + /** * Set the case conversion policy for this module. * Called by the settings mechanism. diff --git a/crates/bindings-typescript/src/lib/util.ts b/crates/bindings-typescript/src/lib/util.ts index e4b989b811b..b79de68c622 100644 --- a/crates/bindings-typescript/src/lib/util.ts +++ b/crates/bindings-typescript/src/lib/util.ts @@ -113,7 +113,7 @@ export function toPascalCase(s: string): string { */ export function toCamelCase(s: T): CamelCase { const str = s - .replace(/[-_]+/g, '_') // collapse runs to a single separator (no backtracking issue) + .replace(/[-_]+/g, '_') .replace(/_([a-zA-Z0-9])/g, (_, c) => c.toUpperCase()); return (str.charAt(0).toLowerCase() + str.slice(1)) as CamelCase; } diff --git a/crates/bindings-typescript/src/sdk/db_connection_impl.ts b/crates/bindings-typescript/src/sdk/db_connection_impl.ts index f462c88a56e..f2a6c6e6a0d 100644 --- a/crates/bindings-typescript/src/sdk/db_connection_impl.ts +++ b/crates/bindings-typescript/src/sdk/db_connection_impl.ts @@ -35,7 +35,11 @@ import type { } from './message_types.ts'; import type { ReducerEvent } from './reducer_event.ts'; import { type UntypedRemoteModule } from './spacetime_module.ts'; -import { makeQueryBuilder } from '../lib/query'; +import { + makeFromBuilder, + makeQueryBuilder, + type SubscriptionFromBuilder, +} from '../lib/query'; import { type TableCache, type Operation, @@ -57,6 +61,7 @@ import type { } from './reducers.ts'; import type { ClientDbView } from './db_view.ts'; import type { RowType, UntypedTableDef } from '../lib/table.ts'; +import type { UntypedSchemaDef } from '../lib/schema'; import type { ProceduresView } from './procedures.ts'; import type { Values } from '../lib/type_util.ts'; import type { TransactionUpdate } from './client_api/types.ts'; @@ -488,6 +493,14 @@ export class DbConnectionImpl return makeQueryBuilder({ tables: this.#remoteModule.tables } as any); } + getFromBuilder< + SchemaDef extends UntypedSchemaDef, + >(): SubscriptionFromBuilder { + return makeFromBuilder( + this.#remoteModule.tables as SchemaDef['tables'] + ); + } + registerSubscription( handle: SubscriptionHandleImpl, handleEmitter: EventEmitter< @@ -534,8 +547,10 @@ export class DbConnectionImpl const rows: Operation[] = []; const deserializeRow = this.#rowDeserializers[tableName]; - const { primaryKeyColName, primaryKeyColType } = - this.#rowIdMetadata[tableName]; + if (!deserializeRow) return []; + const rowIdInfo = this.#rowIdMetadata[tableName]; + if (!rowIdInfo) return []; + const { primaryKeyColName, primaryKeyColType } = rowIdInfo; let previousOffset = 0; while (reader.remaining > 0) { const row = deserializeRow(reader); @@ -824,6 +839,7 @@ export class DbConnectionImpl // Get table information for the table being updated const tableName = tableUpdate.tableName; const tableDef = this.#sourceNameToTableDef[tableName]; + if (!tableDef) continue; const table = this.clientCache.getOrCreateTable(tableDef); const newCallbacks = table.applyOperations( tableUpdate.operations as Operation< diff --git a/crates/bindings-typescript/src/sdk/subscription_builder_impl.ts b/crates/bindings-typescript/src/sdk/subscription_builder_impl.ts index 1ac436c8c9c..e057dc1a0ba 100644 --- a/crates/bindings-typescript/src/sdk/subscription_builder_impl.ts +++ b/crates/bindings-typescript/src/sdk/subscription_builder_impl.ts @@ -6,13 +6,20 @@ import type { } from './event_context'; import { EventEmitter } from './event_emitter'; import type { UntypedRemoteModule } from './spacetime_module'; -import { isRowTypedQuery, toSql, type RowTypedQuery } from '../lib/query'; +import { + isRowTypedQuery, + toSql, + type SubscriptionFromBuilder, + type RowTypedQuery, +} from '../lib/query'; +import type { UntypedSchemaDef } from '../lib/schema'; import type { Values } from '../lib/type_util'; export class SubscriptionBuilderImpl { #onApplied?: (ctx: SubscriptionEventContextInterface) => void = undefined; #onError?: (ctx: ErrorContextInterface) => void = undefined; + #pendingQueries: Array> = []; constructor(private db: DbConnectionImpl) {} /** @@ -64,6 +71,32 @@ export class SubscriptionBuilderImpl { return this; } + /** + * Accumulates a query for a later `subscribe()` call. + * Queries added via `addQuery` and queries passed directly to `subscribe` are mutually exclusive — + * call `subscribe()` with no arguments to send all accumulated queries. + * + * @param queryFn - Receives `{ from }`, where `from` exposes all tables (root and namespaced). + * @returns The current `SubscriptionBuilder` instance for chaining. + * + * @example + * ```ts + * conn.subscriptionBuilder() + * .addQuery(q => q.from.players.build()) + * .addQuery(q => q.from.inventory.items.build()) + * .subscribe(); + * ``` + */ + addQuery( + queryFn: (q: { + from: SubscriptionFromBuilder; + }) => RowTypedQuery + ): this { + const from = this.db.getFromBuilder(); + this.#pendingQueries.push(queryFn({ from })); + return this; + } + /** * Subscribe to a single query. The results of the query will be merged into the client * cache and deduplicated on the client. @@ -80,6 +113,7 @@ export class SubscriptionBuilderImpl { * subscription.unsubscribe(); * ``` */ + subscribe(): SubscriptionHandleImpl; subscribe( query_sql: string | RowTypedQuery ): SubscriptionHandleImpl; @@ -92,14 +126,22 @@ export class SubscriptionBuilderImpl { ) => RowTypedQuery | RowTypedQuery[] ): SubscriptionHandleImpl; subscribe( - query_sql: + query_sql?: | string | RowTypedQuery | Array> | ((tables: any) => RowTypedQuery | RowTypedQuery[]) ): SubscriptionHandleImpl { let queries: Array>; - if (typeof query_sql === 'function') { + if (query_sql === undefined) { + if (this.#pendingQueries.length === 0) { + throw new Error( + 'subscriptionBuilder().subscribe() called with no queries; use addQuery() first or pass a query argument' + ); + } + queries = this.#pendingQueries; + this.#pendingQueries = []; + } else if (typeof query_sql === 'function') { const tablesMap = this.db.getTablesMap?.(); const result = query_sql(tablesMap); queries = Array.isArray(result) ? result : [result]; diff --git a/crates/bindings-typescript/src/server/db_view.ts b/crates/bindings-typescript/src/server/db_view.ts index 9e0350ff863..847a74f957b 100644 --- a/crates/bindings-typescript/src/server/db_view.ts +++ b/crates/bindings-typescript/src/server/db_view.ts @@ -9,7 +9,11 @@ export type ReadonlyDbView = { readonly [Tbl in Values< SchemaDef['tables'] > as Tbl['accessorName']]: ReadonlyTable; -}; +} & (SchemaDef extends { + namespaces: infer NS extends Record; +} + ? { readonly [K in keyof NS]: ReadonlyDbView } + : {}); /** * A type representing the database view, mapping table names to their corresponding Table handles. @@ -18,4 +22,8 @@ export type DbView = { readonly [Tbl in Values< SchemaDef['tables'] > as Tbl['accessorName']]: Table; -}; +} & (SchemaDef extends { + namespaces: infer NS extends Record; +} + ? { readonly [K in keyof NS]: DbView } + : {}); diff --git a/crates/bindings-typescript/src/server/http_handlers.ts b/crates/bindings-typescript/src/server/http_handlers.ts index a092f513611..68d42a267f8 100644 --- a/crates/bindings-typescript/src/server/http_handlers.ts +++ b/crates/bindings-typescript/src/server/http_handlers.ts @@ -211,11 +211,19 @@ export class Request { } } +export type HandlerAliasViews = + SchemaDef extends { + namespaces: infer NS extends Record; + } + ? { readonly [K in keyof NS]: HandlerContext } + : {}; + export interface HandlerContext { readonly timestamp: Timestamp; readonly http: HttpClient; readonly identity: Identity; readonly random: Random; + readonly as: HandlerAliasViews; withTx(body: (ctx: TransactionCtx) => T): T; newUuidV4(): Uuid; newUuidV7(): Uuid; @@ -228,7 +236,8 @@ export type HandlerFn = ( export interface HttpHandlerExport< S extends UntypedSchemaDef = UntypedSchemaDef, -> extends ModuleExport { +> extends HandlerFn, + ModuleExport { [httpHandlerFn]: HandlerFn; } @@ -354,24 +363,27 @@ export function makeHttpHandlerExport( opts: HttpHandlerOpts | undefined, fn: HandlerFn ): HttpHandlerExport { - const handlerExport = { - [httpHandlerFn]: fn, - [exportContext]: ctx, - [registerExport](ctx: SchemaInner, exportName: string) { - if (exportedHttpHandlerObjects.has(handlerExport)) { - throw new TypeError( - `HTTP handler '${exportName}' was exported more than once` + const handlerExport: HttpHandlerExport = Object.assign( + (...args: Parameters>) => fn(...args), + { + [httpHandlerFn]: fn, + [exportContext]: ctx, + [registerExport](ctx: SchemaInner, exportName: string) { + if (exportedHttpHandlerObjects.has(handlerExport)) { + throw new TypeError( + `HTTP handler '${exportName}' was exported more than once` + ); + } + exportedHttpHandlerObjects.add(handlerExport); + registerHttpHandler(ctx, exportName, fn, opts); + ctx.httpHandlerExports.set( + handlerExport as HttpHandlerExport, + exportName ); - } - exportedHttpHandlerObjects.add(handlerExport); - registerHttpHandler(ctx, exportName, fn, opts); - ctx.httpHandlerExports.set( - handlerExport as HttpHandlerExport, - exportName - ); - }, - }; - return handlerExport as HttpHandlerExport; + }, + } + ); + return handlerExport; } export function makeHttpRouterExport( diff --git a/crates/bindings-typescript/src/server/index.ts b/crates/bindings-typescript/src/server/index.ts index a840be4a59d..ae084f672d4 100644 --- a/crates/bindings-typescript/src/server/index.ts +++ b/crates/bindings-typescript/src/server/index.ts @@ -33,5 +33,6 @@ export { type ResponseInit, } from './http'; export type { HandlerContext, HttpHandlerExport } from './http'; +export { ScheduleAt } from '../lib/schedule_at'; import './polyfills'; // Ensure polyfills are loaded diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index d07b71f5185..3f396674f75 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -22,10 +22,18 @@ import { Uuid } from '../lib/uuid'; import { httpClient, type HttpClient } from './http_internal'; import type { DbView } from './db_view'; import { makeRandom, type Random } from './rng'; -import { callUserFunction, ReducerCtxImpl, runWithTx, sys } from './runtime'; +import { + assignTxAliasViews, + buildProcedureAliasCtxMap, + callUserFunction, + ReducerCtxImpl, + runWithTx, + sys, +} from './runtime'; import { exportContext, registerExport, + type SubmoduleDispatchInfo, type ModuleExport, type SchemaInner, } from './schema'; @@ -73,6 +81,13 @@ export interface ProcedureOpts { name: string; } +export type ProcedureAliasViews = + SchemaDef extends { + namespaces: infer NS extends Record; + } + ? { readonly [K in keyof NS]: ProcedureCtx } + : {}; + export interface ProcedureCtx { readonly sender: Identity; readonly databaseIdentity: Identity; @@ -82,6 +97,7 @@ export interface ProcedureCtx { readonly connectionId: ConnectionId | null; readonly http: HttpClient; readonly random: Random; + readonly as: ProcedureAliasViews; withTx(body: (ctx: TransactionCtx) => T): T; newUuidV4(): Uuid; newUuidV7(): Uuid; @@ -154,23 +170,27 @@ export type Procedures = Array<{ }>; export function callProcedure( - moduleCtx: SchemaInner, + procedures: Procedures, id: number, sender: Identity, connectionId: ConnectionId | null, timestamp: Timestamp, argsBuf: Uint8Array, - dbView: () => DbView + dbView: () => DbView, + dispatches: SubmoduleDispatchInfo[] = [], + parentPrefix = '' ): Uint8Array { const { fn, deserializeArgs, serializeReturn, returnTypeBaseSize } = - moduleCtx.procedures[id]; + procedures[id]; const args = deserializeArgs(new BinaryReader(argsBuf)); const ctx: ProcedureCtx = new ProcedureCtxImpl( sender, timestamp, connectionId, - dbView + dbView, + dispatches, + parentPrefix ); const ret = callUserFunction(fn, ctx, args); @@ -187,14 +207,21 @@ const ProcedureCtxImpl = class ProcedureCtx #uuidCounter: { value: 0 } | undefined; #random: Random | undefined; #dbView: () => DbView; + #dispatches: SubmoduleDispatchInfo[]; + #parentPrefix: string; + #asViews: object | undefined; constructor( readonly sender: Identity, readonly timestamp: Timestamp, readonly connectionId: ConnectionId | null, - dbView: () => DbView + dbView: () => DbView, + dispatches: SubmoduleDispatchInfo[] = [], + parentPrefix = '' ) { this.#dbView = dbView; + this.#dispatches = dispatches; + this.#parentPrefix = parentPrefix; } get databaseIdentity() { @@ -213,17 +240,27 @@ const ProcedureCtxImpl = class ProcedureCtx return httpClient; } + get as() { + return (this.#asViews ??= buildProcedureAliasCtxMap( + this, + this.#dispatches, + this.#parentPrefix + )) as any; + } + withTx(body: (ctx: TransactionCtx) => T): T { - return runWithTx( - timestamp => - new TransactionCtxImpl( - this.sender, - timestamp, - this.connectionId, - this.#dbView() - ) as TransactionCtx, - body - ); + const dispatches = this.#dispatches; + const parentPrefix = this.#parentPrefix; + return runWithTx(timestamp => { + const tx = new TransactionCtxImpl( + this.sender, + timestamp, + this.connectionId, + this.#dbView() + ); + assignTxAliasViews(tx, dispatches, parentPrefix); + return tx as unknown as TransactionCtx; + }, body); } newUuidV4(): Uuid { diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index e5d782a71e3..2c95815c63e 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -10,6 +10,8 @@ import { import { RawModuleDef, ViewResultHeader, + type RawProcedureDefV10, + type RawReducerDefV10, type RawTableDefV10, type Typespace, } from '../lib/autogen/types'; @@ -27,6 +29,8 @@ import { type UniqueIndex, } from '../lib/indexes'; import { callProcedure } from './procedures'; +import type { Procedures } from './procedures'; +import type { Reducers } from './reducers'; import { type HandlerContext, Request, @@ -40,6 +44,7 @@ import { serializeHeaders, } from './http_shared'; import { + type AliasViews, type AuthCtx, type JsonObject, type JwtClaims, @@ -48,13 +53,18 @@ import { import { type UntypedSchemaDef } from '../lib/schema'; import { type RowType, type Table, type TableMethods } from '../lib/table'; import { bsatnBaseSize, hasOwn } from '../lib/util'; -import { type AnonymousViewCtx, type ViewCtx } from './views'; +import { + type AnonymousViewCtx, + type AnonViews, + type ViewCtx, + type Views, +} from './views'; import { isRowTypedQuery, makeQueryBuilder, toSql } from './query'; -import type { DbView } from './db_view'; +import type { DbView, ReadonlyDbView } from './db_view'; import { getErrorConstructor, SenderError } from './errors'; import { Range, type Bound } from './range'; import { makeRandom, type Random } from './rng'; -import type { SchemaInner } from './schema'; +import type { SubmoduleDispatchInfo, SchemaInner } from './schema'; import { HttpRequest, HttpResponse } from '../lib/autogen/types'; const { freeze } = Object; @@ -226,18 +236,21 @@ export const ReducerCtxImpl = class ReducerCtx< timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + as: AliasViews; constructor( sender: Identity, timestamp: Timestamp, connectionId: ConnectionId | null, - dbView: DbView + dbView: DbView, + asViews: object = {} ) { Object.seal(this); this.sender = sender; this.timestamp = timestamp; this.connectionId = connectionId; - this.db = dbView; + this.db = dbView as unknown as DbView; + this.as = asViews as AliasViews; } /** Reset the `ReducerCtx` to be used for a new transaction */ @@ -245,13 +258,21 @@ export const ReducerCtxImpl = class ReducerCtx< me: InstanceType, sender: Identity, timestamp: Timestamp, - connectionId: ConnectionId | null + connectionId: ConnectionId | null, + dbView?: DbView, + asViews?: object ) { me.sender = sender; me.timestamp = timestamp; me.connectionId = connectionId; me.#uuidCounter = undefined; me.#senderAuth = undefined; + if (dbView !== undefined) { + me.db = dbView; + } + if (asViews !== undefined) { + me.as = asViews as AliasViews; + } } get databaseIdentity() { @@ -337,31 +358,124 @@ export function runWithTx( } } +type FlatSubmoduleDispatch = { + reducerFns: Reducers; + reducerDefs: RawReducerDefV10[]; + procedureFns: Procedures; + procedureDefs: RawProcedureDefV10[]; + anonViewFns: AnonViews; + viewFns: Views; + tables: Array<{ accessorName: string; tableDef: RawTableDefV10 }>; + typespace: Typespace; + dbView_: DbView | undefined; + /** e.g. "alias." for a submodule with namespace alias "alias" */ + namePrefix: string; + subDispatches: SubmoduleDispatchInfo[]; +}; + +function flattenSubmoduleDispatches( + dispatches: SubmoduleDispatchInfo[], + parentPrefix = '' +): FlatSubmoduleDispatch[] { + const result: FlatSubmoduleDispatch[] = []; + for (const d of dispatches) { + const namePrefix = parentPrefix + d.namespace + '.'; + result.push({ + reducerFns: d.reducerFns, + reducerDefs: d.reducerDefs, + procedureFns: d.procedureFns, + procedureDefs: d.procedureDefs, + anonViewFns: d.anonViewFns, + viewFns: d.viewFns, + tables: d.tables, + typespace: d.typespace, + dbView_: undefined, + namePrefix, + subDispatches: d.subDispatches, + }); + result.push(...flattenSubmoduleDispatches(d.subDispatches, namePrefix)); + } + return result; +} + export const makeHooks = (schema: SchemaInner): ModuleHooks => new ModuleHooksImpl(schema); class ModuleHooksImpl implements ModuleHooks { #schema: SchemaInner; #dbView_: DbView | undefined; + #consumerAs_: object | undefined; #reducerArgsDeserializers; - /** Cache the `ReducerCtx` object to avoid allocating anew for ever reducer call. */ + #consumerReducerCount: number; + #consumerProcedureCount: number; + #flatSubmodules: FlatSubmoduleDispatch[]; + #consumerAnonViewCount: number; + #consumerViewCount: number; + /** Cache the `ReducerCtx` object to avoid allocating anew for every reducer call. */ #reducerCtx_: InstanceType | undefined; + /** Per-submodule alias ctx maps, cached lazily (parallel to #flatSubmodules). */ + #submoduleAsViews_: (object | undefined)[] = []; constructor(schema: SchemaInner) { this.#schema = schema; - this.#reducerArgsDeserializers = schema.moduleDef.reducers.map( - ({ params }) => ProductType.makeDeserializer(params, schema.typespace) + this.#consumerReducerCount = schema.reducers.length; + this.#consumerProcedureCount = schema.procedures.length; + this.#consumerAnonViewCount = schema.anonViews.length; + this.#consumerViewCount = schema.views.length; + this.#flatSubmodules = flattenSubmoduleDispatches( + schema.submoduleDispatchInfos + ); + + const consumerDeserializers = schema.moduleDef.reducers.map(({ params }) => + ProductType.makeDeserializer(params, schema.typespace) ); + const submoduleDeserializers = this.#flatSubmodules.flatMap( + ({ reducerDefs, typespace }) => + reducerDefs.map(({ params }) => + ProductType.makeDeserializer(params, typespace) + ) + ); + this.#reducerArgsDeserializers = [ + ...consumerDeserializers, + ...submoduleDeserializers, + ]; } get #dbView() { - return (this.#dbView_ ??= freeze( + if (this.#dbView_ !== undefined) return this.#dbView_; + const rootTables = Object.values(this.#schema.schemaType.tables).map( + table => [ + table.accessorName, + makeTableView(this.#schema.typespace, table.tableDef), + ] + ); + const submoduleNs = this.#schema.submoduleDispatchInfos.map(dispatch => [ + dispatch.namespace, + buildDbViewForDispatch(dispatch, dispatch.namespace + '.'), + ]); + this.#dbView_ = freeze( + Object.fromEntries([...rootTables, ...submoduleNs]) + ) as DbView; + return this.#dbView_; + } + + #getSubmoduleDbView(submoduleIdx: number): DbView { + const m = this.#flatSubmodules[submoduleIdx]; + return (m.dbView_ ??= freeze( Object.fromEntries( - Object.values(this.#schema.schemaType.tables).map(table => [ - table.accessorName, - makeTableView(this.#schema.typespace, table.tableDef), + m.tables.map(({ accessorName, tableDef }) => [ + accessorName, + makeTableView(m.typespace, tableDef, m.namePrefix), ]) - ) + ) as DbView + )); + } + + #getSubmoduleAsViews(submoduleIdx: number): object { + return (this.#submoduleAsViews_[submoduleIdx] ??= buildAliasCtxMap( + this.#reducerCtx, + this.#flatSubmodules[submoduleIdx].subDispatches, + this.#flatSubmodules[submoduleIdx].namePrefix )); } @@ -374,6 +488,14 @@ class ModuleHooksImpl implements ModuleHooks { )); } + get #consumerAs() { + return (this.#consumerAs_ ??= buildAliasCtxMap( + this.#reducerCtx, + this.#schema.submoduleDispatchInfos, + '' + )); + } + __describe_module__() { const writer = new BinaryWriter(128); RawModuleDef.serialize( @@ -398,19 +520,46 @@ class ModuleHooksImpl implements ModuleHooks { timestamp: bigint, argsBuf: DataView ): void { - const moduleCtx = this.#schema; const deserializeArgs = this.#reducerArgsDeserializers[reducerId]; BINARY_READER.reset(argsBuf); const args = deserializeArgs(BINARY_READER); const senderIdentity = new Identity(sender); + + let fn: ((...args: any[]) => any) | undefined; + let dbView: DbView; + let asViews: object; + + if (reducerId < this.#consumerReducerCount) { + fn = this.#schema.reducers[reducerId]; + dbView = this.#dbView; + asViews = this.#consumerAs; + } else { + let offset = this.#consumerReducerCount; + for (let i = 0; i < this.#flatSubmodules.length; i++) { + const m = this.#flatSubmodules[i]; + if (reducerId < offset + m.reducerFns.length) { + fn = m.reducerFns[reducerId - offset]; + dbView = this.#getSubmoduleDbView(i); + asViews = this.#getSubmoduleAsViews(i); + break; + } + offset += m.reducerFns.length; + } + if (fn === undefined) { + throw new RangeError(`unknown reducerId ${reducerId}`); + } + } + const ctx = this.#reducerCtx; ReducerCtxImpl.reset( ctx, senderIdentity, new Timestamp(timestamp), - ConnectionId.nullIfZero(new ConnectionId(connId)) + ConnectionId.nullIfZero(new ConnectionId(connId)), + dbView!, + asViews! ); - callUserFunction(moduleCtx.reducers[reducerId], ctx, args); + callUserFunction(fn, ctx, args); } __call_view__( @@ -419,14 +568,36 @@ class ModuleHooksImpl implements ModuleHooks { argsBuf: Uint8Array ): { data: Uint8Array } { const moduleCtx = this.#schema; + let viewFns: Views; + let localId: number; + let dbView: ReadonlyDbView; + + if (id < this.#consumerViewCount) { + viewFns = moduleCtx.views; + localId = id; + dbView = this.#dbView as ReadonlyDbView; + } else { + let offset = this.#consumerViewCount; + let found = false; + for (let i = 0; i < this.#flatSubmodules.length; i++) { + const m = this.#flatSubmodules[i]; + if (id < offset + m.viewFns.length) { + viewFns = m.viewFns; + localId = id - offset; + dbView = this.#getSubmoduleDbView(i) as ReadonlyDbView; + found = true; + break; + } + offset += m.viewFns.length; + } + if (!found) throw new RangeError(`unknown viewId ${id}`); + } + const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = - moduleCtx.views[id]; + viewFns![localId!]; const ctx: ViewCtx = freeze({ sender: new Identity(sender), - // this is the non-readonly DbView, but the typing for the user will be - // the readonly one, and if they do call mutating functions it will fail - // at runtime - db: this.#dbView, + db: dbView!, from: makeQueryBuilder(moduleCtx.schemaType), }); const args = deserializeParams(new BinaryReader(argsBuf)); @@ -444,13 +615,35 @@ class ModuleHooksImpl implements ModuleHooks { __call_view_anon__(id: u32, argsBuf: Uint8Array): { data: Uint8Array } { const moduleCtx = this.#schema; + let anonViewFns: AnonViews; + let localId: number; + let dbView: ReadonlyDbView; + + if (id < this.#consumerAnonViewCount) { + anonViewFns = moduleCtx.anonViews; + localId = id; + dbView = this.#dbView as ReadonlyDbView; + } else { + let offset = this.#consumerAnonViewCount; + let found = false; + for (let i = 0; i < this.#flatSubmodules.length; i++) { + const m = this.#flatSubmodules[i]; + if (id < offset + m.anonViewFns.length) { + anonViewFns = m.anonViewFns; + localId = id - offset; + dbView = this.#getSubmoduleDbView(i) as ReadonlyDbView; + found = true; + break; + } + offset += m.anonViewFns.length; + } + if (!found) throw new RangeError(`unknown anonViewId ${id}`); + } + const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = - moduleCtx.anonViews[id]; + anonViewFns![localId!]; const ctx: AnonymousViewCtx = freeze({ - // this is the non-readonly DbView, but the typing for the user will be - // the readonly one, and if they do call mutating functions it will fail - // at runtime - db: this.#dbView, + db: dbView!, from: makeQueryBuilder(moduleCtx.schemaType), }); const args = deserializeParams(new BinaryReader(argsBuf)); @@ -473,15 +666,43 @@ class ModuleHooksImpl implements ModuleHooks { timestamp: bigint, args: Uint8Array ): Uint8Array { - return callProcedure( - this.#schema, - id, - new Identity(sender), - ConnectionId.nullIfZero(new ConnectionId(connection_id)), - new Timestamp(timestamp), - args, - () => this.#dbView - ); + const senderIdentity = new Identity(sender); + const connId = ConnectionId.nullIfZero(new ConnectionId(connection_id)); + const ts = new Timestamp(timestamp); + + if (id < this.#consumerProcedureCount) { + return callProcedure( + this.#schema.procedures, + id, + senderIdentity, + connId, + ts, + args, + () => this.#dbView as DbView, + this.#schema.submoduleDispatchInfos + ); + } + + let offset = this.#consumerProcedureCount; + for (let i = 0; i < this.#flatSubmodules.length; i++) { + const m = this.#flatSubmodules[i]; + if (id < offset + m.procedureFns.length) { + return callProcedure( + m.procedureFns, + id - offset, + senderIdentity, + connId, + ts, + args, + () => this.#getSubmoduleDbView(i), + m.subDispatches, + m.namePrefix + ); + } + offset += m.procedureFns.length; + } + + throw new RangeError(`unknown procedureId ${id}`); } __call_http_handler__( @@ -494,7 +715,8 @@ class ModuleHooksImpl implements ModuleHooks { const handler = moduleCtx.httpHandlers[id]; const ctx = new HandlerContextImpl( new Timestamp(timestamp), - () => this.#dbView + () => this.#dbView, + this.#schema.submoduleDispatchInfos ); const requestMetadata = HttpRequest.deserialize(new BinaryReader(request)); const response = callUserFunction( @@ -521,14 +743,18 @@ class HandlerContextImpl #uuidCounter: { value: number } | undefined; #random: Random | undefined; #dbView: () => DbView; + #dispatches: SubmoduleDispatchInfo[]; + #asViews: object | undefined; readonly http = httpClient; constructor( readonly timestamp: Timestamp, - dbView: () => DbView + dbView: () => DbView, + dispatches: SubmoduleDispatchInfo[] = [] ) { this.#dbView = dbView; + this.#dispatches = dispatches; } get identity() { @@ -539,12 +765,28 @@ class HandlerContextImpl return (this.#random ??= makeRandom(this.timestamp)); } + get as() { + return (this.#asViews ??= buildHandlerAliasCtxMap( + this, + this.#dispatches, + '' + )) as any; + } + withTx(body: (ctx: any) => T): T { - return runWithTx( - timestamp => - new ReducerCtxImpl(Identity.zero(), timestamp, null, this.#dbView()), - body - ); + const dispatches = this.#dispatches; + return runWithTx(timestamp => { + const tx = new ReducerCtxImpl( + Identity.zero(), + timestamp, + null, + this.#dbView() + ); + if (dispatches.length > 0) { + tx.as = buildAliasCtxMap(tx, dispatches, '') as any; + } + return tx; + }, body); } newUuidV4(): Uuid { @@ -559,11 +801,247 @@ class HandlerContextImpl } } +function buildDbViewForDispatch( + dispatch: SubmoduleDispatchInfo, + namePrefix: string +): object { + const tableEntries = dispatch.tables.map(({ accessorName, tableDef }) => [ + accessorName, + makeTableView(dispatch.typespace, tableDef, namePrefix), + ]); + const subNsEntries = dispatch.subDispatches.map(sub => [ + sub.namespace, + buildDbViewForDispatch(sub, namePrefix + sub.namespace + '.'), + ]); + return freeze(Object.fromEntries([...tableEntries, ...subNsEntries])); +} + +function buildAliasCtx( + parent: InstanceType, + dispatch: SubmoduleDispatchInfo, + namePrefix: string +): object { + const nsDb = buildDbViewForDispatch(dispatch, namePrefix); + const subAs = buildAliasCtxMap(parent, dispatch.subDispatches, namePrefix); + return { + get sender() { + return parent.sender; + }, + get databaseIdentity() { + return parent.databaseIdentity; + }, + get identity() { + return parent.identity; + }, + get timestamp() { + return parent.timestamp; + }, + get connectionId() { + return parent.connectionId; + }, + get senderAuth() { + return parent.senderAuth; + }, + get random() { + return parent.random; + }, + newUuidV4() { + return parent.newUuidV4(); + }, + newUuidV7() { + return parent.newUuidV7(); + }, + db: nsDb, + as: subAs, + }; +} + +function buildAliasCtxMap( + parent: InstanceType, + dispatches: SubmoduleDispatchInfo[], + parentPrefix: string +): object { + return freeze( + Object.fromEntries( + dispatches.map(d => [ + d.namespace, + buildAliasCtx(parent, d, parentPrefix + d.namespace + '.'), + ]) + ) + ); +} + +function buildHandlerAliasCtx( + parent: HandlerContextImpl, + dispatch: SubmoduleDispatchInfo, + namePrefix: string +): object { + // nsDb is built lazily inside withTx so that sys.table_id_from_name is called + // only after a transaction has been started (sys.procedure_start_mut_tx). + let nsDb_: DbView | undefined; + const subAs = buildHandlerAliasCtxMap( + parent, + dispatch.subDispatches, + namePrefix + ); + return { + get timestamp() { + return parent.timestamp; + }, + get http() { + return parent.http; + }, + get identity() { + return parent.identity; + }, + get random() { + return parent.random; + }, + as: subAs, + withTx(body: any) { + return runWithTx( + (ts: Timestamp) => + new ReducerCtxImpl( + Identity.zero(), + ts, + null, + (nsDb_ ??= buildDbViewForDispatch( + dispatch, + namePrefix + ) as DbView) + ), + body + ); + }, + newUuidV4() { + return parent.newUuidV4(); + }, + newUuidV7() { + return parent.newUuidV7(); + }, + }; +} + +function buildHandlerAliasCtxMap( + parent: HandlerContextImpl, + dispatches: SubmoduleDispatchInfo[], + parentPrefix: string +): object { + return freeze( + Object.fromEntries( + dispatches.map(d => [ + d.namespace, + buildHandlerAliasCtx(parent, d, parentPrefix + d.namespace + '.'), + ]) + ) + ); +} + +type ProcCtxRef = { + sender: Identity; + connectionId: ConnectionId | null; + timestamp: Timestamp; + get databaseIdentity(): Identity; + get identity(): Identity; + get http(): typeof httpClient; + get random(): Random; + newUuidV4(): Uuid; + newUuidV7(): Uuid; +}; + +function buildProcedureAliasCtx( + parent: ProcCtxRef, + dispatch: SubmoduleDispatchInfo, + namePrefix: string +): object { + // nsDb is built lazily inside withTx so that sys.table_id_from_name is called + // only after a transaction has been started (sys.procedure_start_mut_tx). + let nsDb_: DbView | undefined; + const subAs = buildProcedureAliasCtxMap( + parent, + dispatch.subDispatches, + namePrefix + ); + return { + get sender() { + return parent.sender; + }, + get databaseIdentity() { + return parent.databaseIdentity; + }, + get identity() { + return parent.identity; + }, + get timestamp() { + return parent.timestamp; + }, + get connectionId() { + return parent.connectionId; + }, + get http() { + return parent.http; + }, + get random() { + return parent.random; + }, + as: subAs, + withTx(body: any) { + return runWithTx( + (ts: Timestamp) => + new ReducerCtxImpl( + parent.sender, + ts, + parent.connectionId, + (nsDb_ ??= buildDbViewForDispatch( + dispatch, + namePrefix + ) as DbView) + ), + body + ); + }, + newUuidV4() { + return parent.newUuidV4(); + }, + newUuidV7() { + return parent.newUuidV7(); + }, + }; +} + +export function buildProcedureAliasCtxMap( + parent: ProcCtxRef, + dispatches: SubmoduleDispatchInfo[], + parentPrefix: string +): object { + return freeze( + Object.fromEntries( + dispatches.map(d => [ + d.namespace, + buildProcedureAliasCtx(parent, d, parentPrefix + d.namespace + '.'), + ]) + ) + ); +} + +/** Builds and assigns reducer-style alias views onto a freshly created TransactionCtx. + * Must be called while inside a transaction (after sys.procedure_start_mut_tx). */ +export function assignTxAliasViews( + tx: InstanceType, + dispatches: SubmoduleDispatchInfo[], + parentPrefix = '' +): void { + if (dispatches.length > 0) { + tx.as = buildAliasCtxMap(tx, dispatches, parentPrefix) as any; + } +} + export function makeTableView( typespace: Typespace, - table: RawTableDefV10 + table: RawTableDefV10, + namePrefix = '' ): Table { - const table_id = sys.table_id_from_name(table.sourceName); + const table_id = sys.table_id_from_name(namePrefix + table.sourceName); const rowType = typespace.types[table.productTypeRef]; if (rowType.tag !== 'Product') { throw 'impossible'; @@ -659,7 +1137,7 @@ export function makeTableView( for (const indexDef of table.indexes) { const accessorName = indexDef.accessorName!; - const index_id = sys.index_id_from_name(indexDef.sourceName!); + const index_id = sys.index_id_from_name(namePrefix + indexDef.sourceName!); let column_ids: number[]; let isHashIndex = false; diff --git a/crates/bindings-typescript/src/server/schema.ts b/crates/bindings-typescript/src/server/schema.ts index 384e7ebcd0c..5e00e73c896 100644 --- a/crates/bindings-typescript/src/server/schema.ts +++ b/crates/bindings-typescript/src/server/schema.ts @@ -3,6 +3,11 @@ import { CaseConversionPolicy, Lifecycle, type MethodOrAny, + type RawModuleDefV10, + type RawProcedureDefV10, + type RawReducerDefV10, + type RawTableDefV10, + type Typespace, } from '../lib/autogen/types'; import { type ParamsAsObject, @@ -18,6 +23,7 @@ import { } from '../lib/schema'; import type { UntypedTableSchema } from '../lib/table_schema'; import { ColumnBuilder, TypeBuilder } from '../lib/type_builders'; +import { hasOwn } from '../lib/util'; import { Router, type HandlerFn, @@ -55,10 +61,25 @@ import { } from './views'; import type { UntypedTableDef } from '../lib/table'; +export type SubmoduleDispatchInfo = { + namespace: string; + reducerFns: Reducers; + reducerDefs: RawReducerDefV10[]; + procedureFns: Procedures; + procedureDefs: RawProcedureDefV10[]; + anonViewFns: AnonViews; + viewFns: Views; + typespace: Typespace; + tables: Array<{ accessorName: string; tableDef: RawTableDefV10 }>; + subDispatches: SubmoduleDispatchInfo[]; +}; + export class SchemaInner< S extends UntypedSchemaDef = UntypedSchemaDef, > extends ModuleContext { schemaType: S; + exportsRegistered = false; + schedulesResolved = false; existingFunctions = new Set(); existingHttpHandlers = new Set(); reducers: Reducers = []; @@ -79,6 +100,7 @@ export class SchemaInner< new Map(); pendingSchedules: PendingSchedule[] = []; pendingHttpRoutes: PendingHttpRoute[] = []; + submoduleDispatchInfos: SubmoduleDispatchInfo[] = []; constructor(getSchemaType: (ctx: SchemaInner) => S) { super(); @@ -104,6 +126,10 @@ export class SchemaInner< } resolveSchedules() { + if (this.schedulesResolved) { + return; + } + this.schedulesResolved = true; for (const { reducer, scheduleAtCol, tableName } of this.pendingSchedules) { const functionName = this.functionExports.get(reducer()); if (functionName === undefined) { @@ -186,23 +212,9 @@ export class Schema implements ModuleDefaultExport { } [moduleHooks](exports: object) { - // if (!(hasOwn(exports, 'default') && exports.default instanceof Schema)) { - // throw new TypeError('must export schema as default export'); - // } - const registeredSchema = this.#ctx; - for (const [name, moduleExport] of Object.entries(exports)) { - if (name === 'default') continue; - if (!isModuleExport(moduleExport)) { - throw new TypeError( - 'exporting something that is not a spacetime export' - ); - } - checkExportContext(moduleExport, registeredSchema); - moduleExport[registerExport](registeredSchema, name); - } - registeredSchema.resolveSchedules(); - registeredSchema.resolveHttpRoutes(); - return makeHooks(registeredSchema); + this.buildRawModuleDefV10(exports); + this.#ctx.resolveHttpRoutes(); + return makeHooks(this.#ctx); } get schemaType(): S { @@ -217,6 +229,55 @@ export class Schema implements ModuleDefaultExport { return this.#ctx.typespace; } + get submoduleDispatchInfos(): SubmoduleDispatchInfo[] { + return this.#ctx.submoduleDispatchInfos; + } + + /** Internal: register exports and materialize the RawModuleDefV10 for upload. */ + buildRawModuleDefV10( + exports: object, + opts?: { ignoreNonModuleExports?: boolean } + ): RawModuleDefV10 { + registerModuleExports(this.#ctx, exports, { + ignoreNonModuleExports: opts?.ignoreNonModuleExports ?? false, + }); + this.#ctx.resolveSchedules(); + return this.#ctx.rawModuleDefV10(); + } + + /** + * @internal – called by schema() when processing a submodule namespace entry. + * Registers the library's exports and returns both the serialized module def + * and the runtime dispatch info needed by ModuleHooksImpl for __call_reducer__. + */ + buildSubmoduleDispatch(exports: object): { + rawDef: RawModuleDefV10; + dispatch: SubmoduleDispatchInfo; + } { + const rawDef = this.buildRawModuleDefV10(exports, { + ignoreNonModuleExports: true, + }); + this.#ctx.resolveHttpRoutes(); + return { + rawDef, + dispatch: { + namespace: '', + reducerFns: [...this.#ctx.reducers], + reducerDefs: [...this.#ctx.moduleDef.reducers], + procedureFns: [...this.#ctx.procedures], + procedureDefs: [...this.#ctx.moduleDef.procedures], + anonViewFns: [...this.#ctx.anonViews], + viewFns: [...this.#ctx.views], + typespace: this.#ctx.moduleDef.typespace, + tables: Object.values(this.#ctx.schemaType.tables).map(t => ({ + accessorName: t.accessorName, + tableDef: t.tableDef, + })), + subDispatches: [...this.#ctx.submoduleDispatchInfos], + }, + }; + } + /** * Defines a SpacetimeDB reducer function. * @@ -625,18 +686,101 @@ export interface ModuleSettings { CASE_CONVERSION_POLICY?: CaseConversionPolicy; } -export function schema>( - tables: H, +type SubmoduleNamespace = { + default: Schema; + [key: string]: unknown; +}; + +type SchemaEntry = UntypedTableSchema | SubmoduleNamespace; + +type ExtractTableEntries> = { + [K in keyof H as H[K] extends UntypedTableSchema ? K : never]: Extract< + H[K], + UntypedTableSchema + >; +}; + +type ExtractSubmoduleSchemas> = { + [K in keyof H as H[K] extends { default: Schema } + ? K + : never]: H[K] extends { default: Schema } + ? S + : never; +}; + +type SchemaDefForEntries> = + TablesToSchema> & { + namespaces: ExtractSubmoduleSchemas; + }; + +function isUntypedTableSchema(x: unknown): x is UntypedTableSchema { + return typeof x === 'object' && x !== null && hasOwn(x, 'tableDef'); +} + +function isSubmoduleNamespace(x: unknown): x is SubmoduleNamespace { + return ( + typeof x === 'object' && + x !== null && + hasOwn(x, 'default') && + x.default instanceof Schema + ); +} + +function registerModuleExports( + schema: SchemaInner, + exports: object, + opts?: { ignoreNonModuleExports?: boolean } +) { + if (schema.exportsRegistered) { + return; + } + schema.exportsRegistered = true; + + for (const [name, moduleExport] of Object.entries(exports)) { + if (name === 'default') continue; + if (!isModuleExport(moduleExport)) { + if (opts?.ignoreNonModuleExports) { + continue; + } + throw new TypeError('exporting something that is not a spacetime export'); + } + checkExportContext(moduleExport, schema); + moduleExport[registerExport](schema, name); + } +} + +export function schema>( + entries: H, moduleSettings?: ModuleSettings -): Schema> { - const ctx = new SchemaInner>(ctx => { +): Schema> { + const ctx = new SchemaInner>(ctx => { // Apply module settings. if (moduleSettings?.CASE_CONVERSION_POLICY != null) { ctx.setCaseConversionPolicy(moduleSettings.CASE_CONVERSION_POLICY); } const tableSchemas: Record = {}; - for (const [accName, table] of Object.entries(tables)) { + for (const [accName, entry] of Object.entries(entries)) { + if (entry instanceof Schema) { + throw new TypeError( + `schema entry '${accName}' looks like a default import; use \`import * as ${accName} from '...'\` so the submodule can see the library's named reducer exports.` + ); + } + if (isSubmoduleNamespace(entry)) { + const { rawDef, dispatch } = + entry.default.buildSubmoduleDispatch(entry); + dispatch.namespace = accName; + ctx.addSubmodule({ namespace: accName, module: rawDef }); + ctx.submoduleDispatchInfos.push(dispatch); + continue; + } + if (!isUntypedTableSchema(entry)) { + throw new TypeError( + `schema entry '${accName}' must be a table or a submodule namespace object` + ); + } + + const table = entry; const tableDef = table.tableDef(ctx, accName); tableSchemas[accName] = tableToSchema(accName, table, tableDef); ctx.moduleDef.tables.push(tableDef); @@ -656,7 +800,7 @@ export function schema>( }); } } - return { tables: tableSchemas } as TablesToSchema; + return { tables: tableSchemas } as SchemaDefForEntries; }); return new Schema(ctx); diff --git a/crates/bindings-typescript/src/server/views.ts b/crates/bindings-typescript/src/server/views.ts index 58e77062db0..e8420fd4b56 100644 --- a/crates/bindings-typescript/src/server/views.ts +++ b/crates/bindings-typescript/src/server/views.ts @@ -279,7 +279,7 @@ export function registerView< } (anon ? ctx.anonViews : ctx.views).push({ - fn, + fn: fn as unknown as ViewFn, deserializeParams: ProductType.makeDeserializer(paramType, typespace), serializeReturn: AlgebraicType.makeSerializer(returnType, typespace), returnTypeBaseSize: bsatnBaseSize(typespace, returnType), diff --git a/crates/bindings-typescript/tests/ctx_as.test.ts b/crates/bindings-typescript/tests/ctx_as.test.ts new file mode 100644 index 00000000000..e764d881b76 --- /dev/null +++ b/crates/bindings-typescript/tests/ctx_as.test.ts @@ -0,0 +1,97 @@ +import { beforeAll, describe, expect, it, vi } from 'vitest'; + +vi.mock( + 'spacetime:sys@2.0', + () => ({ + moduleHooks: Symbol('moduleHooks'), + table_id_from_name: () => 1, + index_id_from_name: () => 1, + row_iter_bsatn_close: () => {}, + }), + { virtual: true } +); + +describe('ctx.as alias proxy', () => { + let schema: typeof import('../src/server/schema').schema; + let table: typeof import('../src/lib/table').table; + let t: typeof import('../src/lib/type_builders').t; + let moduleHooks: symbol; + + beforeAll(async () => { + ({ schema } = await import('../src/server/schema')); + ({ table } = await import('../src/lib/table')); + ({ t } = await import('../src/lib/type_builders')); + ({ moduleHooks } = (await import('spacetime:sys@2.0')) as any); + }); + + it('ctx.as. provides a narrowed ctx with library db and delegating sender', async () => { + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + const authSchema = schema({ sessions }); + const authLib = { default: authSchema }; + + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + + let capturedCtx: any; + const myReducer = consumer.reducer((ctx: any) => { + capturedCtx = ctx; + }); + + const hooks = (consumer as any)[moduleHooks]({ myReducer }); + hooks.__call_reducer__(0, 0n, 0n, 0n, new DataView(new ArrayBuffer(0))); + + expect(capturedCtx.as).toBeDefined(); + expect(capturedCtx.as.myauth).toBeDefined(); + expect(capturedCtx.as.myauth.db).toBeDefined(); + expect(capturedCtx.as.myauth.db.sessions).toBeDefined(); + expect(capturedCtx.as.myauth.sender).toBe(capturedCtx.sender); + expect(capturedCtx.as.myauth.timestamp).toBe(capturedCtx.timestamp); + }); + + it('ctx.as is empty object when there are no submodules', async () => { + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players }); + + let capturedCtx: any; + const myReducer = consumer.reducer((ctx: any) => { + capturedCtx = ctx; + }); + + const hooks = (consumer as any)[moduleHooks]({ myReducer }); + hooks.__call_reducer__(0, 0n, 0n, 0n, new DataView(new ArrayBuffer(0))); + + expect(capturedCtx.as).toBeDefined(); + expect(Object.keys(capturedCtx.as)).toHaveLength(0); + }); + + it('ctx.as..as carries nested submodule aliases', async () => { + const bazTable = table({ name: 'baz_items' }, { id: t.u32().primaryKey() }); + const bazSchema = schema({ bazTable }); + const bazLib = { default: bazSchema }; + + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + const authSchema = schema({ sessions, baz: bazLib }); + const authLib = { default: authSchema }; + + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + + let capturedCtx: any; + const myReducer = consumer.reducer((ctx: any) => { + capturedCtx = ctx; + }); + + const hooks = (consumer as any)[moduleHooks]({ myReducer }); + hooks.__call_reducer__(0, 0n, 0n, 0n, new DataView(new ArrayBuffer(0))); + + expect(capturedCtx.as.myauth.as.baz).toBeDefined(); + expect(capturedCtx.as.myauth.as.baz.db.bazTable).toBeDefined(); + expect(capturedCtx.as.myauth.as.baz.sender).toBe(capturedCtx.sender); + }); +}); diff --git a/crates/bindings-typescript/tests/schema_submodules.test.ts b/crates/bindings-typescript/tests/schema_submodules.test.ts new file mode 100644 index 00000000000..7c430ec3dc1 --- /dev/null +++ b/crates/bindings-typescript/tests/schema_submodules.test.ts @@ -0,0 +1,220 @@ +import { beforeAll, describe, expect, it, vi } from 'vitest'; + +vi.mock( + 'spacetime:sys@2.0', + () => ({ + moduleHooks: Symbol('moduleHooks'), + }), + { virtual: true } +); + +vi.mock('../src/server/runtime', () => ({ + makeHooks: () => ({}), + callProcedure: () => new Uint8Array(), + callUserFunction: (fn: (...args: any[]) => any, ...args: any[]) => + fn(...args), + ReducerCtxImpl: class {}, + sys: { + row_iter_bsatn_close: () => {}, + }, +})); + +describe('schema submodules', () => { + let schema: typeof import('../src/server/schema').schema; + let table: typeof import('../src/lib/table').table; + let t: typeof import('../src/lib/type_builders').t; + + beforeAll(async () => { + ({ schema } = await import('../src/server/schema')); + ({ table } = await import('../src/lib/table')); + ({ t } = await import('../src/lib/type_builders')); + }); + + it('emits submodule module defs and resolves submodule schedules', () => { + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + + const sessionCleanupTick = table( + { + name: 'session_cleanup_tick', + scheduled: (): any => cleanExpiredSessions, + }, + { + scheduledId: t.u64().primaryKey().autoInc(), + scheduledAt: t.scheduleAt(), + } + ); + + const sessions = table( + { name: 'sessions' }, + { + id: t.u64().primaryKey().autoInc(), + } + ); + + const authSchema = schema({ + sessions, + sessionCleanupTick, + }); + + const cleanExpiredSessions = authSchema.reducer(() => {}); + const authLib = { + default: authSchema, + cleanExpiredSessions, + }; + + const consumer = schema({ + players, + myauth: authLib, + }); + + const raw = consumer.buildRawModuleDefV10({}); + const submodules = raw.sections.find( + section => section.tag === 'Submodules' + )?.value; + + expect(submodules).toHaveLength(1); + expect(submodules?.[0]?.namespace).toBe('myauth'); + + const submoduleSections = submodules?.[0]?.module.sections ?? []; + const submoduleReducers = submoduleSections.find( + section => section.tag === 'Reducers' + )?.value; + const submoduleSchedules = submoduleSections.find( + section => section.tag === 'Schedules' + )?.value; + + expect(submoduleReducers).toEqual( + expect.arrayContaining([ + expect.objectContaining({ sourceName: 'cleanExpiredSessions' }), + ]) + ); + expect(submoduleSchedules).toEqual([ + expect.objectContaining({ + tableName: 'sessionCleanupTick', + functionName: 'cleanExpiredSessions', + }), + ]); + }); + + it('rejects default-import style submodules with a clear error', () => { + const sessions = table( + { name: 'sessions' }, + { + id: t.u64().primaryKey().autoInc(), + } + ); + + const authSchema = schema({ sessions }); + + expect(() => + schema({ + myauth: authSchema as any, + }) + ).toThrow(/looks like a default import/); + }); + + it('populates submoduleDispatchInfos with reducer fns and table metadata', () => { + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + + const authSchema = schema({ sessions }); + const cleanExpiredSessions = authSchema.reducer(() => {}); + const authLib = { default: authSchema, cleanExpiredSessions }; + + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + + const infos = consumer.submoduleDispatchInfos; + expect(infos).toHaveLength(1); + + const info = infos[0]; + expect(info.reducerFns).toHaveLength(1); + expect(info.reducerDefs).toHaveLength(1); + expect(info.reducerDefs[0].sourceName).toBe('cleanExpiredSessions'); + expect(info.tables).toHaveLength(1); + expect(info.tables[0].accessorName).toBe('sessions'); + expect(info.subDispatches).toHaveLength(0); + }); + + it('flattens nested submodule dispatches depth-first', () => { + // baz library: 1 reducer + const bazTable = table({ name: 'baz_items' }, { id: t.u32().primaryKey() }); + const bazSchema = schema({ bazTable }); + const bazReducer = bazSchema.reducer(() => {}); + const bazLib = { default: bazSchema, bazReducer }; + + // auth library: 1 own reducer, uses baz as a submodule + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + const authSchema = schema({ sessions, baz: bazLib }); + const authReducer = authSchema.reducer(() => {}); + const authLib = { default: authSchema, authReducer }; + + // consumer: 1 own reducer, uses auth as a submodule + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + const consumerReducer = consumer.reducer(() => {}); + + // Verify depth-first structure: + // consumer.submoduleDispatchInfos[0] = myauth (authReducer) + // consumer.submoduleDispatchInfos[0].subDispatches[0] = myauth.baz (bazReducer) + const infos = consumer.submoduleDispatchInfos; + expect(infos).toHaveLength(1); + + const authInfo = infos[0]; + expect(authInfo.reducerFns).toHaveLength(1); + expect(authInfo.reducerDefs[0].sourceName).toBe('authReducer'); + expect(authInfo.subDispatches).toHaveLength(1); + + const bazInfo = authInfo.subDispatches[0]; + expect(bazInfo.reducerFns).toHaveLength(1); + expect(bazInfo.reducerDefs[0].sourceName).toBe('bazReducer'); + expect(bazInfo.subDispatches).toHaveLength(0); + + // Unused variable check + void consumerReducer; + }); + + it('submoduleDispatchInfos carry namespace and nested namespace dispatches propagate', () => { + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + const authSchema = schema({ sessions }); + const authLib = { default: authSchema }; + + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + + const infos = consumer.submoduleDispatchInfos; + expect(infos).toHaveLength(1); + expect(infos[0].namespace).toBe('myauth'); + expect(infos[0].tables[0].accessorName).toBe('sessions'); + }); + + it('nested submodules carry their own namespace on subDispatches', () => { + const bazTable = table({ name: 'baz_items' }, { id: t.u32().primaryKey() }); + const bazSchema = schema({ bazTable }); + const bazLib = { default: bazSchema }; + + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + const authSchema = schema({ sessions, baz: bazLib }); + const authLib = { default: authSchema }; + + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + + const authInfo = consumer.submoduleDispatchInfos[0]; + expect(authInfo.namespace).toBe('myauth'); + expect(authInfo.subDispatches).toHaveLength(1); + expect(authInfo.subDispatches[0].namespace).toBe('baz'); + expect(authInfo.subDispatches[0].tables[0].accessorName).toBe('bazTable'); + }); +}); diff --git a/crates/cli/src/api.rs b/crates/cli/src/api.rs index d40b03ee87e..a608d220335 100644 --- a/crates/cli/src/api.rs +++ b/crates/cli/src/api.rs @@ -4,7 +4,7 @@ use std::ops::Add; use reqwest::{header, Client, RequestBuilder}; use serde::Deserialize; -use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; use spacetimedb_lib::de::serde::DeserializeWrapper; use spacetimedb_lib::Identity; @@ -61,11 +61,11 @@ impl ClientApi { } /// Reads the `ModuleDef` from the `schema` endpoint. - pub async fn module_def(&self) -> anyhow::Result { + pub async fn module_def(&self) -> anyhow::Result { let res = self .client .get(self.con.db_uri("schema")) - .query(&[("version", "9")]) + .query(&[("version", "10")]) .send() .await?; let DeserializeWrapper(module_def) = res.json_or_error().await?; diff --git a/crates/cli/src/subcommands/build.rs b/crates/cli/src/subcommands/build.rs index 5ef4cea7300..f0347fbbc36 100644 --- a/crates/cli/src/subcommands/build.rs +++ b/crates/cli/src/subcommands/build.rs @@ -2,7 +2,9 @@ use crate::util::find_module_path; use crate::Config; use clap::ArgAction::SetTrue; use clap::{Arg, ArgMatches}; +use colored::Colorize; use std::ffi::OsString; +use std::io::IsTerminal; use std::path::{Path, PathBuf}; pub fn cli() -> clap::Command { @@ -86,6 +88,25 @@ pub fn run_build( } let result = crate::tasks::build(&module_path, lint_dir.as_deref(), build_debug, features.as_ref())?; + + // For TypeScript modules, extract the schema from the bundle and emit + // advisory warnings (e.g. HTTP handlers in mounted sub-modules are unsupported). + // If the standalone binary is unavailable or extraction fails, skip silently. + if result.1 == "Js" + && let Ok(module_def) = crate::subcommands::generate::extract_descriptions(&result.0) + { + let tty = std::io::stderr().is_terminal(); + for warning in module_def.collect_warnings() { + let prefix = if tty { + "warning:".yellow().to_string() + } else { + "warning:".to_string() + }; + eprintln!("{prefix} {warning}"); + println!(); + } + } + println!("Build finished successfully."); Ok(result) diff --git a/crates/cli/src/subcommands/call.rs b/crates/cli/src/subcommands/call.rs index 3e5f8840aa8..a273b97ebbf 100644 --- a/crates/cli/src/subcommands/call.rs +++ b/crates/cli/src/subcommands/call.rs @@ -45,14 +45,8 @@ enum CallDef<'a> { impl<'a> CallDef<'a> { fn params(&self) -> &'a sats::ProductType { match self { - CallDef::Reducer(reducer_def) => &reducer_def.params, - CallDef::Procedure(procedure_def) => &procedure_def.params, - } - } - fn name(&self) -> &str { - match self { - CallDef::Reducer(reducer_def) => &reducer_def.name, - CallDef::Procedure(procedure_def) => &procedure_def.name, + CallDef::Reducer(r) => &r.params, + CallDef::Procedure(p) => &p.params, } } fn kind(&self) -> &str { @@ -103,10 +97,13 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), Error> { let module_def: ModuleDef = api.module_def().await?.try_into()?; - let call_def = match module_def.reducer(&**reducer_procedure_name) { - Some(reducer_def) => CallDef::Reducer(reducer_def), - None => match module_def.procedure(&**reducer_procedure_name) { - Some(procedure_def) => CallDef::Procedure(procedure_def), + // Dot-qualified names (e.g. `lib.my_reducer`) route to submodules. + // Keep the owning module def around: type refs in the params must be + // resolved against the owning module's typespace, not the consumer's. + let (call_def, owning_def) = match module_def.reducer_by_name_with_module(reducer_procedure_name) { + Some((_, reducer_def, owning_def)) => (CallDef::Reducer(reducer_def), owning_def), + None => match module_def.procedure_by_name_with_module(reducer_procedure_name) { + Some((_, procedure_def, owning_def)) => (CallDef::Procedure(procedure_def), owning_def), None => { return Err(anyhow::Error::msg(no_such_reducer_or_procedure( &database_identity, @@ -152,7 +149,14 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), Error> { if response_text.starts_with("no such reducer") || response_text.starts_with("no such procedure") { no_such_reducer_or_procedure(&database_identity, database, reducer_procedure_name, &module_def) } else if response_text.starts_with("invalid arguments") { - invalid_arguments(&database_identity, database, &response_text, &module_def, call_def) + invalid_arguments( + &database_identity, + database, + &response_text, + owning_def.typespace(), + reducer_procedure_name, + call_def, + ) } else { return error; }; @@ -169,11 +173,20 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), Error> { } /// Returns an error message for when `reducer` is called with wrong arguments. -fn invalid_arguments(identity: &Identity, db: &str, text: &str, module_def: &ModuleDef, call_def: CallDef) -> String { +/// +/// `full_name` is the (possibly dot-qualified) name the user invoked. +fn invalid_arguments( + identity: &Identity, + db: &str, + text: &str, + typespace: &Typespace, + full_name: &str, + call_def: CallDef, +) -> String { let mut error = format!( "Invalid arguments provided for {} `{}` for database `{}` resolving to identity `{}`.", call_def.kind(), - call_def.name(), + full_name, db, identity ); @@ -190,7 +203,10 @@ fn invalid_arguments(identity: &Identity, db: &str, text: &str, module_def: &Mod error, "\n\nThe {} has the following signature:\n\t{}", call_def.kind(), - CallSignature(module_def.typespace().with_type(&call_def)) + CallSignature { + name: full_name, + call: typespace.with_type(&call_def) + } ) .unwrap(); @@ -218,13 +234,16 @@ fn split_at_first_substring<'t>(text: &'t str, substring: &str) -> Option<(&'t s /// Provided the `schema_json` for the database, /// returns the signature for a reducer OR procedure with `name`. -struct CallSignature<'a>(sats::WithTypespace<'a, CallDef<'a>>); +struct CallSignature<'a> { + name: &'a str, + call: sats::WithTypespace<'a, CallDef<'a>>, +} impl std::fmt::Display for CallSignature<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let call_def = self.0.ty(); - let typespace = self.0.typespace(); + let call_def = self.call.ty(); + let typespace = self.call.typespace(); - write!(f, "{}(", call_def.name())?; + write!(f, "{}(", self.name)?; // Print the arguments to `args`. let mut comma = false; @@ -258,17 +277,24 @@ const CALL_PRINT_LIMIT: usize = 10; /// Provided the schema for the database, /// decorate `error` with more helpful info about reducers and procedures. +/// +/// Reducers and procedures from submodules are listed under their +/// full dot-qualified names (e.g. `lib.my_reducer`). fn add_reducer_procedure_ctx_to_err(error: &mut String, module_def: &ModuleDef, reducer_name: &str) { - let reducers = module_def - .reducers() - .filter(|reducer| reducer.lifecycle.is_none()) - .map(|reducer| &*reducer.name) + let reducer_names = module_def + .all_reducers_with_prefix() + .into_iter() + .filter(|(_, _, r)| r.lifecycle.is_none()) + .map(|(prefix, _, r)| format!("{prefix}{}", &*r.name)) .collect::>(); + let reducers = reducer_names.iter().map(String::as_str).collect::>(); - let procedures = module_def - .procedures() - .map(|reducer| &*reducer.name) + let procedure_names = module_def + .all_procedures_with_prefix() + .into_iter() + .map(|(prefix, _, p)| format!("{prefix}{}", &*p.name)) .collect::>(); + let procedures = procedure_names.iter().map(String::as_str).collect::>(); if let Some(best) = find_best_match_for_name(&reducers, reducer_name, None) { write!(error, "\n\nA reducer with a similar name exists: `{best}`").unwrap(); diff --git a/crates/cli/src/subcommands/describe.rs b/crates/cli/src/subcommands/describe.rs index e774224855c..2dd8448cc5f 100644 --- a/crates/cli/src/subcommands/describe.rs +++ b/crates/cli/src/subcommands/describe.rs @@ -6,7 +6,9 @@ use crate::util::UNSTABLE_WARNING; use crate::util::{database_identity, get_auth_header}; use anyhow::Context; use clap::{Arg, ArgAction, ArgMatches}; +use spacetimedb_lib::db::raw_def::v10::{RawReducerDefV10, RawTableDefV10}; use spacetimedb_lib::sats; +use spacetimedb_schema::def::ModuleDef; pub fn cli() -> clap::Command { clap::Command::new("describe") @@ -97,30 +99,35 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error }; let api = ClientApi::new(conn); - let module_def = api.module_def().await?; + let raw = api.module_def().await?; if json { fn sats_to_json(v: &T) -> serde_json::Result { serde_json::to_string_pretty(sats::serde::SerdeWrapper::from_ref(v)) } let json = match entity { + // Entity lookups go through the validated `ModuleDef`, which resolves + // canonical names and dot-qualified names from submodules + // (e.g. `lib.my_reducer`). Some((EntityType::Reducer, reducer_name)) => { - let reducer = module_def - .reducers - .iter() - .find(|r| *r.name == *reducer_name) - .context("no such reducer")?; - sats_to_json(reducer)? + let module_def: ModuleDef = raw.try_into()?; + let (_, reducer) = module_def.reducer_by_name(reducer_name).context("no such reducer")?; + sats_to_json(&RawReducerDefV10::from(reducer.clone()))? } Some((EntityType::Table, table_name)) => { - let table = module_def - .tables - .iter() - .find(|t| *t.name == *table_name) + let module_def: ModuleDef = raw.try_into()?; + let (_, _, table) = module_def + .all_tables_with_prefix() + .into_iter() + .find(|(prefix, _, t)| { + table_name + .strip_prefix(prefix.as_str()) + .is_some_and(|rest| rest == &*t.name) + }) .context("no such table")?; - sats_to_json(table)? + sats_to_json(&RawTableDefV10::from(table.clone()))? } - None => sats_to_json(&module_def)?, + None => sats_to_json(&raw)?, }; // TODO: validate the JSON output diff --git a/crates/cli/src/subcommands/subscribe.rs b/crates/cli/src/subcommands/subscribe.rs index 8fc6f18abda..ff19623a7f3 100644 --- a/crates/cli/src/subcommands/subscribe.rs +++ b/crates/cli/src/subcommands/subscribe.rs @@ -7,12 +7,12 @@ use reqwest::Url; use serde_json::Value; use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2, v3 as ws_v3}; use spacetimedb_data_structures::map::HashMap; -use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::de::serde::{DeserializeWrapper, SeedWrapper}; use spacetimedb_lib::de::DeserializeSeed as BsatnDeserializeSeed; use spacetimedb_lib::sats::WithTypespace; use spacetimedb_lib::ser::serde::SerializeWrapper; use spacetimedb_lib::{bsatn, AlgebraicType}; +use spacetimedb_schema::def::ModuleDef; use std::collections::VecDeque; use std::io; use std::sync::atomic::AtomicU32; @@ -116,7 +116,7 @@ impl SubscribeConnection { } /// Wait for the initial subscription result and optionally print it. - async fn await_initial_update(&mut self, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> { + async fn await_initial_update(&mut self, module_def: Option<&ModuleDef>) -> Result<(), Error> { match self { Self::V3 { ws, pending } => await_initial_update_v3(ws, pending, module_def).await, Self::V1 { ws } => await_initial_update_v1(ws, module_def).await, @@ -127,7 +127,7 @@ impl SubscribeConnection { async fn consume_transaction_updates( &mut self, num: Option, - module_def: &RawModuleDefV9, + module_def: &ModuleDef, num_received: &UpdateCounter, ) -> Result<(), Error> { match self { @@ -183,7 +183,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error database: resolved.database.clone(), }; let api = ClientApi::new(conn); - let module_def = api.module_def().await?; + let module_def: ModuleDef = api.module_def().await?.try_into()?; let mut conn = connect_with_fallback(&api, confirmed).await?; let num_received = UpdateCounter::new(); @@ -450,7 +450,7 @@ fn parse_msg_json(msg: &WsMessage) -> Option(ws: &mut S, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> +async fn await_initial_update_v1(ws: &mut S, module_def: Option<&ModuleDef>) -> Result<(), Error> where S: TryStream + Unpin, { @@ -494,7 +494,7 @@ where async fn await_initial_update_v3( ws: &mut S, pending: &mut VecDeque, - module_def: Option<&RawModuleDefV9>, + module_def: Option<&ModuleDef>, ) -> Result<(), Error> where S: TryStream + Unpin, @@ -530,7 +530,7 @@ where async fn consume_transaction_updates_v1( ws: &mut S, num: Option, - module_def: &RawModuleDefV9, + module_def: &ModuleDef, num_received: &UpdateCounter, ) -> Result<(), Error> where @@ -575,7 +575,7 @@ async fn consume_transaction_updates_v3( ws: &mut S, pending: &mut VecDeque, num: Option, - module_def: &RawModuleDefV9, + module_def: &ModuleDef, num_received: &UpdateCounter, ) -> Result<(), Error> where @@ -665,25 +665,19 @@ fn decode_server_payload(msg: Bytes, pending: &mut VecDeque, - schema: &RawModuleDefV9, -) -> Result { +fn format_output_json_v1(msg: &ws_v1::DatabaseUpdate, schema: &ModuleDef) -> Result { let formatted = reformat_update_v1(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } /// Format initial v3 subscription rows using the CLI's existing JSON output shape. -fn format_output_json_query_rows(msg: &ws_v2::QueryRows, schema: &RawModuleDefV9) -> Result { +fn format_output_json_query_rows(msg: &ws_v2::QueryRows, schema: &ModuleDef) -> Result { let formatted = reformat_query_rows(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } /// Format a v3 transaction update using the CLI's existing JSON output shape. -fn format_output_json_transaction_update( - msg: &ws_v2::TransactionUpdate, - schema: &RawModuleDefV9, -) -> Result { +fn format_output_json_transaction_update(msg: &ws_v2::TransactionUpdate, schema: &ModuleDef) -> Result { let formatted = reformat_transaction_update(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } @@ -694,19 +688,42 @@ fn format_output_json_from_tables(formatted: &HashMap<&str, SubscriptionTable>) Ok(output) } +/// Resolve the row type for a table or view by its wire name. +/// +/// Wire names of tables and views from submodules are dot-qualified +/// (e.g. `lib.my_table`). The returned type is resolved against the owning +/// module's typespace, since that is where its type refs point. +fn type_for_table_like<'a>(module_def: &'a ModuleDef, name: &str) -> Option> { + let matches_wire_name = + |prefix: &str, plain_name: &str| name.strip_prefix(prefix).is_some_and(|rest| rest == plain_name); + + let (owning_def, type_ref) = module_def + .all_tables_with_prefix() + .into_iter() + .find_map(|(prefix, owning_def, table)| { + matches_wire_name(&prefix, &table.name).then_some((owning_def, table.product_type_ref)) + }) + .or_else(|| { + module_def + .all_views_with_prefix() + .into_iter() + .find_map(|(prefix, owning_def, view)| { + matches_wire_name(&prefix, &view.name).then_some((owning_def, view.product_type_ref)) + }) + })?; + + Some(owning_def.typespace().resolve(type_ref)) +} + /// Convert a v1 JSON-format database update to the normalized table output map. fn reformat_update_v1<'a>( msg: &'a ws_v1::DatabaseUpdate, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { msg.tables .iter() .map(|upd| { - let table_ty = schema.typespace.resolve( - schema - .type_ref_for_table_like(&upd.table_name) - .context("table not found in schema")?, - ); + let table_ty = type_for_table_like(schema, &upd.table_name).context("table not found in schema")?; let reformat_row = |row: &str| -> anyhow::Result { // TODO: can the following two calls be merged into a single call to reduce allocations? @@ -736,16 +753,12 @@ fn reformat_update_v1<'a>( /// Convert v3 initial subscription rows to the normalized table output map. fn reformat_query_rows<'a>( msg: &'a ws_v2::QueryRows, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { let mut formatted = HashMap::default(); for table in &msg.tables { - let table_ty = schema.typespace.resolve( - schema - .type_ref_for_table_like(&table.table) - .context("table not found in schema")?, - ); + let table_ty = type_for_table_like(schema, &table.table).context("table not found in schema")?; let table_output = formatted.entry(&*table.table).or_insert_with(|| SubscriptionTable { deletes: Vec::new(), inserts: Vec::new(), @@ -759,17 +772,13 @@ fn reformat_query_rows<'a>( /// Convert a v3 transaction update to the normalized table output map. fn reformat_transaction_update<'a>( msg: &'a ws_v2::TransactionUpdate, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { let mut formatted = HashMap::default(); for query_set in &msg.query_sets { for table in &query_set.tables { - let table_ty = schema.typespace.resolve( - schema - .type_ref_for_table_like(&table.table_name) - .context("table not found in schema")?, - ); + let table_ty = type_for_table_like(schema, &table.table_name).context("table not found in schema")?; let table_output = formatted .entry(&*table.table_name) .or_insert_with(|| SubscriptionTable { diff --git a/crates/codegen/examples/regen-typescript-moduledef.rs b/crates/codegen/examples/regen-typescript-moduledef.rs index 6ebad479090..a3fb1b30f76 100644 --- a/crates/codegen/examples/regen-typescript-moduledef.rs +++ b/crates/codegen/examples/regen-typescript-moduledef.rs @@ -72,6 +72,14 @@ fn main() -> anyhow::Result<()> { let code = regex_replace!(&code, r"SubscriptionBuilderImpl as __SubscriptionBuilderImpl,", r""); let code = regex_replace!(&code, r"TableCache as __TableCache,", r""); let code = regex_replace!(&code, r"ClientCache as __ClientCache,", r""); + // Fix a TypeScript type inference error (TS7022/TS7023) caused by the recursive type + // RawModuleDefV10 -> RawSubmoduleV10 -> RawModuleDefV10. Annotating the `module` getter + // with `: any` breaks the cycle without affecting other generated types. + let code = regex_replace!( + &code, + r"(export const RawSubmoduleV10 = [\s\S]*?)get module\(\) \{", + r"${1}get module(): any {" + ); fs::write(dir.join(filename), code.as_bytes()) })?; diff --git a/crates/codegen/src/cpp.rs b/crates/codegen/src/cpp.rs index 9ddbeae8bb0..ced50bddc0b 100644 --- a/crates/codegen/src/cpp.rs +++ b/crates/codegen/src/cpp.rs @@ -47,7 +47,7 @@ impl<'opts> Cpp<'opts> { } fn is_recursive_mount_module_field(&self, type_name: &str, field_name: &str) -> bool { - type_name == "RawModuleMountV10" && field_name == "module" + type_name == "RawSubmoduleV10" && field_name == "module" } fn write_header_comment(&self, output: &mut String) { @@ -180,7 +180,7 @@ impl<'opts> Cpp<'opts> { for (field_name, field_type) in &product.elements { write!(output, " ").unwrap(); if self.is_recursive_mount_module_field(type_name, field_name) { - // Temporary special-case to preserve the recursive RawModuleMountV10 -> + // Temporary special-case to preserve the recursive RawSubmoduleV10 -> // RawModuleDefV10 shape while breaking the include cycle in generated C++. write!(output, "std::shared_ptr<{}::RawModuleDefV10>", self.namespace).unwrap(); } else { @@ -213,7 +213,7 @@ impl<'opts> Cpp<'opts> { writeln!(output, " }}").unwrap(); // Generate equality method - if type_name == "RawModuleMountV10" { + if type_name == "RawSubmoduleV10" { // Pointer equality is sufficient for this internal autogen type. Mounts are not // emitted by the C++ module path yet; this exists to keep the schema shape aligned. writeln!(output, " SPACETIMEDB_PRODUCT_TYPE_EQUALITY(namespace_, module)").unwrap(); @@ -541,13 +541,13 @@ impl Lang for Cpp<'_> { let type_name = name.to_string(); for dep in deps { - if dep != type_name && !(type_name == "RawModuleMountV10" && dep == "RawModuleDefV10") { + if dep != type_name && !(type_name == "RawSubmoduleV10" && dep == "RawModuleDefV10") { writeln!(output, "#include \"{}.g.h\"", dep).unwrap(); } } writeln!(output).unwrap(); - if type_name == "RawModuleMountV10" { + if type_name == "RawSubmoduleV10" { writeln!(output, "namespace {} {{", self.namespace).unwrap(); writeln!(output, "struct RawModuleDefV10;").unwrap(); writeln!(output, "}} // namespace {}", self.namespace).unwrap(); diff --git a/crates/codegen/src/lib.rs b/crates/codegen/src/lib.rs index 28d4fb8a5a4..e8ec880e6f7 100644 --- a/crates/codegen/src/lib.rs +++ b/crates/codegen/src/lib.rs @@ -1,3 +1,4 @@ +use spacetimedb_lib::db::raw_def::v9::TableAccess; use spacetimedb_schema::def::{ModuleDef, ProcedureDef, ReducerDef, TableDef, TypeDef, ViewDef}; use spacetimedb_schema::schema::{Schema, TableSchema}; mod code_indenter; @@ -33,10 +34,35 @@ pub fn generate(module: &ModuleDef, lang: &dyn Lang, options: &CodegenOptions) - itertools::chain!( util::iter_tables(module, options.visibility).map(|tbl| lang.generate_table_file(module, tbl)), module.views().map(|view| lang.generate_view_file(module, view)), + // Public tables from submodules + module + .all_tables_with_prefix() + .into_iter() + .filter(|(prefix, _, table)| !prefix.is_empty() && table.table_access == TableAccess::Public) + .map(|(prefix, owning_def, table)| lang.generate_submodule_table_file(owning_def, &prefix, table)), + // Views from submodules (views are currently always public) + module + .all_views_with_prefix() + .into_iter() + .filter(|(prefix, _, _)| !prefix.is_empty()) + .map(|(prefix, owning_def, view)| lang.generate_submodule_view_file(owning_def, &prefix, view)), module.types().flat_map(|typ| lang.generate_type_files(module, typ)), util::iter_reducers(module, options.visibility).map(|reducer| lang.generate_reducer_file(module, reducer)), util::iter_procedures(module, options.visibility) .map(|procedure| lang.generate_procedure_file(module, procedure)), + // Reducers from submodules + module + .all_reducers_with_prefix() + .into_iter() + .filter(|(prefix, _, reducer)| !prefix.is_empty() && !reducer.visibility.is_private()) + .map(|(prefix, owning_def, reducer)| lang.generate_submodule_reducer_file(owning_def, &prefix, reducer)), + // Procedures from submodules + module + .all_procedures_with_prefix() + .into_iter() + .filter(|(prefix, _, procedure)| !prefix.is_empty() && !procedure.visibility.is_private()) + .map(|(prefix, owning_def, procedure)| lang + .generate_submodule_procedure_file(owning_def, &prefix, procedure)), lang.generate_global_files(module, options), ) .collect() @@ -68,4 +94,59 @@ pub trait Lang { .expect("Failed to generate table due to validation errors"); self.generate_table_file_from_schema(module, &tbl, schema) } + + /// Generate a row-type file for a public table from a submodule. + /// Uses `owning_def`'s typespace for type resolution. + /// Filename goes in a subdirectory named after the namespace: + /// e.g. `alias/table_name_table.ts` for namespace `"alias."`, table `tableName`. + fn generate_submodule_table_file(&self, owning_def: &ModuleDef, namespace: &str, table: &TableDef) -> OutputFile { + let schema = TableSchema::from_module_def(owning_def, table, (), 0.into()) + .validated() + .expect("Failed to generate submodule table file"); + let mut file = self.generate_table_file_from_schema(owning_def, table, schema); + let ns_path = namespace.trim_end_matches('.').replace('.', "/"); + file.filename = format!("{}/{}", ns_path, file.filename); + file + } + + /// Generate a row-type file for a view from a submodule. + /// Filename goes in a subdirectory named after the namespace prefix. + fn generate_submodule_view_file(&self, owning_def: &ModuleDef, namespace: &str, view: &ViewDef) -> OutputFile { + let tbl = TableDef::from(view.clone()); + let schema = TableSchema::from_view_def_for_codegen(owning_def, view) + .validated() + .expect("Failed to generate submodule view file"); + let mut file = self.generate_table_file_from_schema(owning_def, &tbl, schema); + let ns_path = namespace.trim_end_matches('.').replace('.', "/"); + file.filename = format!("{}/{}", ns_path, file.filename); + file + } + + /// Generate an arg-schema file for a reducer from a submodule. + /// Filename goes in a subdirectory named after the namespace prefix. + fn generate_submodule_reducer_file( + &self, + owning_def: &ModuleDef, + prefix: &str, + reducer: &ReducerDef, + ) -> OutputFile { + let mut file = self.generate_reducer_file(owning_def, reducer); + let ns_path = prefix.trim_end_matches('.').replace('.', "/"); + file.filename = format!("{}/{}", ns_path, file.filename); + file + } + + /// Generate an arg-schema file for a procedure from a submodule. + /// Filename goes in a subdirectory named after the namespace prefix. + fn generate_submodule_procedure_file( + &self, + owning_def: &ModuleDef, + prefix: &str, + procedure: &ProcedureDef, + ) -> OutputFile { + let mut file = self.generate_procedure_file(owning_def, procedure); + let ns_path = prefix.trim_end_matches('.').replace('.', "/"); + file.filename = format!("{}/{}", ns_path, file.filename); + file + } } diff --git a/crates/codegen/src/typescript.rs b/crates/codegen/src/typescript.rs index a2d1cbf8e6c..3fa65a8a8bf 100644 --- a/crates/codegen/src/typescript.rs +++ b/crates/codegen/src/typescript.rs @@ -6,16 +6,19 @@ use crate::{CodegenOptions, OutputFile}; use super::util::{collect_case, print_auto_generated_file_comment, type_ref_name}; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt::{self, Write}; use std::iter; use std::ops::Deref; use convert_case::{Case, Casing}; +use spacetimedb_lib::db::raw_def::v9::TableAccess; use spacetimedb_lib::sats::layout::PrimitiveType; use spacetimedb_lib::sats::AlgebraicTypeRef; use spacetimedb_primitives::ColId; -use spacetimedb_schema::def::{ColumnDef, ConstraintDef, IndexDef, ModuleDef, ReducerDef, TableDef, TypeDef}; +use spacetimedb_schema::def::{ + ConstraintDef, IndexDef, ModuleDef, ProcedureDef, ReducerDef, TableDef, TypeDef, ViewDef, +}; use spacetimedb_schema::identifier::Identifier; use spacetimedb_schema::reducer_name::ReducerName; use spacetimedb_schema::schema::TableSchema; @@ -85,15 +88,7 @@ impl Lang for TypeScript { writeln!(out, "export default __t.row({{"); out.indent(1); - write_object_type_builder_fields( - module, - out, - &product_def.elements, - table.primary_key, - true, - Some(&table.columns), - ) - .unwrap(); + write_object_type_builder_fields(module, out, &product_def.elements, table.primary_key, true, true).unwrap(); out.dedent(1); writeln!(out, "}});"); OutputFile { @@ -151,7 +146,7 @@ impl Lang for TypeScript { writeln!(out, "export const params = {{"); out.with_indent(|out| { - write_object_type_builder_fields(module, out, &procedure.params_for_generate.elements, None, true, None) + write_object_type_builder_fields(module, out, &procedure.params_for_generate.elements, None, true, false) .unwrap() }); writeln!(out, "}};"); @@ -201,6 +196,67 @@ impl Lang for TypeScript { writeln!(out, "import {table_name_pascalcase}Row from \"./{table_module_name}\";"); } + // Import row types for submodule namespace tables (public only) + let ns_tables: Vec<_> = module + .all_tables_with_prefix() + .into_iter() + .filter(|(prefix, _, table)| !prefix.is_empty() && table.table_access == TableAccess::Public) + .collect(); + let ns_views: Vec<_> = module + .all_views_with_prefix() + .into_iter() + .filter(|(prefix, _, _)| !prefix.is_empty()) + .collect(); + let ns_reducers: Vec<_> = module + .all_reducers_with_prefix() + .into_iter() + .filter(|(prefix, _, reducer)| !prefix.is_empty() && !reducer.visibility.is_private()) + .collect(); + let ns_procedures: Vec<_> = module + .all_procedures_with_prefix() + .into_iter() + .filter(|(prefix, _, procedure)| !prefix.is_empty() && !procedure.visibility.is_private()) + .collect(); + if !ns_tables.is_empty() || !ns_views.is_empty() { + writeln!(out); + writeln!(out, "// Import namespace table schema definitions"); + for (prefix, _, table) in &ns_tables { + let ns_path = submodule_ns_path(prefix); + let file_stem = table_module_name(&table.accessor_name); + let row_type = submodule_row_type_name(prefix, table.accessor_name.deref()); + writeln!(out, "import {row_type}Row from \"./{ns_path}/{file_stem}\";"); + } + for (prefix, _, view) in &ns_views { + let ns_path = submodule_ns_path(prefix); + let file_stem = table_module_name(&view.accessor_name); + let row_type = submodule_row_type_name(prefix, view.accessor_name.deref()); + writeln!(out, "import {row_type}Row from \"./{ns_path}/{file_stem}\";"); + } + } + if !ns_reducers.is_empty() { + writeln!(out); + writeln!(out, "// Import namespace reducer arg schemas"); + for (prefix, _, reducer) in &ns_reducers { + if !is_reducer_invokable(reducer) { + continue; + } + let ns_path = submodule_ns_path(prefix); + let module_name = reducer_module_name(&reducer.accessor_name); + let args_type = submodule_reducer_args_type_name(prefix, &reducer.accessor_name); + writeln!(out, "import {args_type} from \"./{ns_path}/{module_name}\";"); + } + } + if !ns_procedures.is_empty() { + writeln!(out); + writeln!(out, "// Import namespace procedure arg schemas"); + for (prefix, _, procedure) in &ns_procedures { + let ns_path = submodule_ns_path(prefix); + let module_name = procedure_module_name(&procedure.accessor_name); + let args_type = submodule_procedure_args_type_name(prefix, &procedure.accessor_name); + writeln!(out, "import * as {args_type} from \"./{ns_path}/{module_name}\";"); + } + } + writeln!(out); writeln!(out, "/** Type-only namespace exports for generated type groups. */"); @@ -224,7 +280,7 @@ impl Lang for TypeScript { module, out, type_ref, - &table.name, + table.name.deref(), iter_indexes(table), iter_constraints(table), table.is_event, @@ -242,16 +298,59 @@ impl Lang for TypeScript { } writeln!(out, "{view_accessor}: __table({{"); out.indent(1); - write_table_opts(module, out, type_ref, &view.name, iter::empty(), iter::empty(), false); + write_table_opts( + module, + out, + type_ref, + view.name.deref(), + iter::empty(), + iter::empty(), + false, + ); out.dedent(1); writeln!(out, "}}, {}Row),", view_name_pascalcase); } + // Namespace tables from submodules + for (prefix, owning_def, table) in &ns_tables { + let source_name = submodule_source_name(prefix, table.accessor_name.deref()); + let row_type = submodule_row_type_name(prefix, table.accessor_name.deref()); + let type_ref = table.product_type_ref; + writeln!(out, "\"{source_name}\": __table({{"); + out.indent(1); + write_table_opts( + owning_def, + out, + type_ref, + &source_name, + iter_indexes(table), + iter_constraints(table), + table.is_event, + ); + out.dedent(1); + writeln!(out, "}}, {row_type}Row),"); + } + // Namespace views from submodules + for (prefix, owning_def, view) in &ns_views { + let source_name = submodule_source_name(prefix, view.accessor_name.deref()); + let row_type = submodule_row_type_name(prefix, view.accessor_name.deref()); + let type_ref = view.product_type_ref; + writeln!(out, "\"{source_name}\": __table({{"); + out.indent(1); + write_table_opts( + owning_def, + out, + type_ref, + &source_name, + iter::empty(), + iter::empty(), + false, + ); + out.dedent(1); + writeln!(out, "}}, {row_type}Row),"); + } out.dedent(1); writeln!(out, "}});"); - table_accessor_aliases.retain(|(deprecated_accessor, _)| !table_accessor_names.contains(deprecated_accessor)); - let has_table_accessor_aliases = !table_accessor_aliases.is_empty(); - writeln!(out); writeln!(out, "/** The schema information for all reducers in this module. This is defined the same way as the reducers would have been defined in the server, except the body of the reducer is omitted in code generation. */"); writeln!(out, "const reducersSchema = __reducers("); @@ -264,6 +363,14 @@ impl Lang for TypeScript { let args_type = reducer_args_type_name(&reducer.accessor_name); writeln!(out, "__reducerSchema(\"{}\", {}),", reducer.name, args_type); } + for (prefix, _, reducer) in &ns_reducers { + if !is_reducer_invokable(reducer) { + continue; + } + let wire_name = format!("{}{}", prefix, reducer.name); + let args_type = submodule_reducer_args_type_name(prefix, &reducer.accessor_name); + writeln!(out, "__reducerSchema(\"{wire_name}\", {args_type}),"); + } out.dedent(1); writeln!(out, ");"); @@ -282,9 +389,20 @@ impl Lang for TypeScript { procedure.name, ); } + for (prefix, _, procedure) in &ns_procedures { + let wire_name = format!("{}{}", prefix, procedure.name); + let args_type = submodule_procedure_args_type_name(prefix, &procedure.accessor_name); + writeln!( + out, + "__procedureSchema(\"{wire_name}\", {args_type}.params, {args_type}.returnType)," + ); + } out.dedent(1); writeln!(out, ");"); + table_accessor_aliases.retain(|(deprecated_accessor, _)| !table_accessor_names.contains(deprecated_accessor)); + let has_table_accessor_aliases = !table_accessor_aliases.is_empty(); + if has_table_accessor_aliases { writeln!(out); writeln!( @@ -448,36 +566,95 @@ impl Lang for TypeScript { writeln!(out); writeln!(out, "/** The tables available in this remote SpacetimeDB module. Each table reference doubles as a query builder. */"); - if has_table_accessor_aliases { - writeln!( - out, - "const tablesBase: __TablesBase = __makeQueryBuilder(tablesSchema.schemaType);" - ); + if ns_tables.is_empty() && ns_views.is_empty() { + if has_table_accessor_aliases { + writeln!( + out, + "const tablesBase: __TablesBase = __makeQueryBuilder(tablesSchema.schemaType);" + ); + writeln!( + out, + "export const tables: Tables = __withTableAccessorAliases(tablesBase, true) as Tables;" + ); + } else { + writeln!( + out, + "export const tables: __QueryBuilder = __makeQueryBuilder(tablesSchema.schemaType);" + ); + } + } else { + writeln!(out, "const _qb = __makeQueryBuilder(tablesSchema.schemaType);"); + writeln!(out, "export const tables = {{"); + out.indent(1); + // Root tables (use camelCase accessor, matching tablesSchema keys) + for table in iter_tables(module, options.visibility) { + let key = table.accessor_name.deref().to_case(Case::Camel); + writeln!(out, "{key}: _qb.{key},"); + } + // Root views + for view in iter_views(module) { + let key = view.accessor_name.deref().to_case(Case::Camel); + writeln!(out, "{key}: _qb.{key},"); + } + // Build and emit namespace tree + let tree = build_ns_tree(&ns_tables, &ns_views); + emit_ns_tree(out, &tree); + out.dedent(1); + writeln!(out, "}} as const;"); + } + writeln!(out); + writeln!(out, "/** The reducers available in this remote SpacetimeDB module. */"); + if ns_reducers.is_empty() { writeln!( out, - "export const tables: Tables = __withTableAccessorAliases(tablesBase, true) as Tables;" + "export const reducers = __convertToAccessorMap(reducersSchema.reducersType.reducers);" ); } else { writeln!( out, - "export const tables: __QueryBuilder = __makeQueryBuilder(tablesSchema.schemaType);" + "const _reducers = __convertToAccessorMap(reducersSchema.reducersType.reducers);" ); + writeln!(out, "export const reducers = {{"); + out.indent(1); + for reducer in iter_reducers(module, options.visibility) { + if !is_reducer_invokable(reducer) { + continue; + } + let key = reducer.accessor_name.deref().to_case(Case::Camel); + writeln!(out, "{key}: _reducers.{key},"); + } + let tree = build_reducer_ns_tree(&ns_reducers); + emit_fn_ns_tree(out, "_reducers", &tree); + out.dedent(1); + writeln!(out, "}} as const;"); } - writeln!(out); - writeln!(out, "/** The reducers available in this remote SpacetimeDB module. */"); - writeln!( - out, - "export const reducers = __convertToAccessorMap(reducersSchema.reducersType.reducers);" - ); + writeln!(out); writeln!( out, "/** The procedures available in this remote SpacetimeDB module. */" ); - writeln!( - out, - "export const procedures = __convertToAccessorMap(proceduresSchema.procedures);" - ); + if ns_procedures.is_empty() { + writeln!( + out, + "export const procedures = __convertToAccessorMap(proceduresSchema.procedures);" + ); + } else { + writeln!( + out, + "const _procedures = __convertToAccessorMap(proceduresSchema.procedures);" + ); + writeln!(out, "export const procedures = {{"); + out.indent(1); + for procedure in iter_procedures(module, options.visibility) { + let key = procedure.accessor_name.deref().to_case(Case::Camel); + writeln!(out, "{key}: _procedures.{key},"); + } + let tree = build_procedure_ns_tree(&ns_procedures); + emit_fn_ns_tree(out, "_procedures", &tree); + out.dedent(1); + writeln!(out, "}} as const;"); + } // Write type aliases for EventContext, ReducerEventContext, SubscriptionEventContext, ErrorContext writeln!(out); @@ -619,7 +796,20 @@ impl Lang for TypeScript { let procedures_file = generate_procedures_file(module, options); let types_file = generate_types_file(module); - vec![index_file, reducers_file, procedures_file, types_file] + let mut files = vec![index_file, reducers_file, procedures_file, types_file]; + + // Generate types.ts for each submodule namespace so that the + // namespace-scoped reducer/procedure/table files can resolve their + // `import { … } from "./types"` imports. + let mut submodule_namespaces: BTreeMap = BTreeMap::new(); + collect_submodule_namespaces(module, "", &mut submodule_namespaces); + for (prefix, owning_def) in &submodule_namespaces { + let ns_path = submodule_ns_path(prefix); + let filename = format!("{ns_path}/types.ts"); + files.push(generate_types_file_with_path(owning_def, filename)); + } + + files } } @@ -694,6 +884,10 @@ fn generate_procedures_file(module: &ModuleDef, options: &CodegenOptions) -> Out } fn generate_types_file(module: &ModuleDef) -> OutputFile { + generate_types_file_with_path(module, "types.ts".to_string()) +} + +fn generate_types_file_with_path(module: &ModuleDef, filename: String) -> OutputFile { let mut output = CodeIndenter::new(String::new(), INDENT); let out = &mut output; @@ -727,11 +921,22 @@ fn generate_types_file(module: &ModuleDef) -> OutputFile { } OutputFile { - filename: "types.ts".to_string(), + filename, code: output.into_inner(), } } +/// Recursively collect all submodule namespaces in depth-first order. +/// Keys are dot-terminated prefix strings (e.g. `"lib."`, `"lib.sublib."`). +/// Values are references to the `ModuleDef` that owns that namespace. +fn collect_submodule_namespaces<'a>(module: &'a ModuleDef, prefix: &str, out: &mut BTreeMap) { + for (ns, submodule_def) in module.submodules() { + let full_prefix = format!("{prefix}{ns}."); + out.insert(full_prefix.clone(), submodule_def); + collect_submodule_namespaces(submodule_def, &full_prefix, out); + } +} + fn print_index_imports(out: &mut Indenter) { // All library imports are prefixed with `__` to avoid // clashing with the names of user generated types. @@ -823,7 +1028,7 @@ fn define_body_for_reducer(module: &ModuleDef, out: &mut Indenter, params: &[(Id writeln!(out, "}};"); } else { writeln!(out); - out.with_indent(|out| write_object_type_builder_fields(module, out, params, None, true, None).unwrap()); + out.with_indent(|out| write_object_type_builder_fields(module, out, params, None, true, false).unwrap()); writeln!(out, "}};"); } } @@ -848,7 +1053,7 @@ fn define_body_for_product( writeln!(out, "}});"); } else { writeln!(out); - out.with_indent(|out| write_object_type_builder_fields(module, out, elements, None, true, None).unwrap()); + out.with_indent(|out| write_object_type_builder_fields(module, out, elements, None, true, false).unwrap()); writeln!(out, "}});"); } writeln!(out, "export type {name} = __Infer;"); @@ -859,13 +1064,13 @@ fn write_table_opts<'a>( module: &ModuleDef, out: &mut Indenter, type_ref: AlgebraicTypeRef, - name: &Identifier, + name: &str, indexes: impl Iterator, constraints: impl Iterator, is_event: bool, ) { let product_def = module.typespace_for_generate()[type_ref].as_product().unwrap(); - writeln!(out, "name: '{}',", name.deref()); + writeln!(out, "name: '{}',", name); writeln!(out, "indexes: ["); out.indent(1); for index_def in indexes { @@ -940,7 +1145,7 @@ fn write_object_type_builder_fields( elements: &[(Identifier, AlgebraicTypeUse)], primary_key: Option, convert_case: bool, - columns: Option<&[ColumnDef]>, + write_original_name: bool, ) -> anyhow::Result<()> { for (i, (ident, ty)) in elements.iter().enumerate() { let name = if convert_case { @@ -953,14 +1158,7 @@ fn write_object_type_builder_fields( Some(pk) => pk.idx() == i, None => false, }; - // The `.name(..)` value is the in-database (canonical) column name, which may - // differ from the generated camelCase accessor key. Emit it only when the - // canonical name differs, so the client maps to the correct wire/column name - // regardless of the source identifier's casing. - let original_name = columns - .and_then(|columns| columns.get(i)) - .map(|column| column.name.deref()) - .filter(|canonical| convert_case && *canonical != name.as_str()); + let original_name = (write_original_name && convert_case && *name != **ident).then_some(&**ident); write_type_builder_field(module, out, &name, original_name, ty, is_primary_key)?; } @@ -1098,7 +1296,7 @@ fn define_body_for_sum( (Identifier::for_test(pascal), ty.clone()) }) .collect(); - out.with_indent(|out| write_object_type_builder_fields(module, out, &pascal_variants, None, false, None).unwrap()); + out.with_indent(|out| write_object_type_builder_fields(module, out, &pascal_variants, None, false, false).unwrap()); writeln!(out, "}});"); writeln!(out, "export type {name} = __Infer;"); out.newline(); @@ -1108,6 +1306,22 @@ fn table_module_name(table_name: &Identifier) -> String { table_name.deref().to_case(Case::Snake) + "_table" } +/// Combined accessor name for a submodule namespace table/view. +/// E.g. namespace="alias.", accessor_name="tableName" → "aliasTableName" +/// Source name (wire name) for a submodule namespace table/view. +/// E.g. namespace="alias.", accessor_name="tableName" → "alias.tableName" +fn submodule_source_name(namespace: &str, accessor_name: &str) -> String { + format!("{}{}", namespace, accessor_name) +} + +/// TypeScript import symbol for a submodule namespace table/view row type. +/// Uses `_` separator to avoid colliding with root tables that share the same PascalCase prefix. +/// E.g. namespace="lib.", accessor_name="library_table" → "Lib_LibraryTable" +fn submodule_row_type_name(namespace: &str, accessor_name: &str) -> String { + let ns_part = namespace.trim_end_matches('.').replace('.', "_").to_case(Case::Pascal); + format!("{}_{}", ns_part, accessor_name.to_case(Case::Pascal)) +} + fn reducer_args_type_name(reducer_name: &ReducerName) -> String { reducer_name.deref().to_case(Case::Pascal) + "Reducer" } @@ -1124,6 +1338,154 @@ fn procedure_module_name(procedure_name: &Identifier) -> String { procedure_name.deref().to_case(Case::Snake) + "_procedure" } +/// Converts a dot-terminated namespace like `"lib."` or `"lib.sublib."` to a path like `"lib"` or `"lib/sublib"`. +fn submodule_ns_path(namespace: &str) -> String { + namespace.trim_end_matches('.').replace('.', "/") +} + +/// TypeScript import symbol for a submodule namespace reducer/procedure. +/// Uses `_` separator to avoid colliding with root reducers/procedures sharing the same prefix. +/// E.g. prefix="lib.", accessor_name="library_reducer" → "Lib_LibraryReducer" +fn submodule_fn_type_name(prefix: &str, accessor_name: &str) -> String { + let ns_part = prefix.trim_end_matches('.').replace('.', "_").to_case(Case::Pascal); + format!("{}_{}", ns_part, accessor_name.to_case(Case::Pascal)) +} + +fn submodule_reducer_args_type_name(prefix: &str, accessor_name: &ReducerName) -> String { + submodule_fn_type_name(prefix, accessor_name.deref()) + "Reducer" +} + +fn submodule_procedure_args_type_name(prefix: &str, accessor_name: &Identifier) -> String { + submodule_fn_type_name(prefix, accessor_name.deref()) + "Procedure" +} + +/// A node in the recursive namespace tree used to emit the nested `tables` export. +struct NsTree { + /// (combined_qb_key, local_ts_key) for table/view entries at this level. + entries: Vec<(String, String)>, + /// Child namespace nodes keyed by namespace segment. + children: BTreeMap, +} + +impl NsTree { + fn new() -> Self { + NsTree { + entries: Vec::new(), + children: BTreeMap::new(), + } + } + + fn insert(&mut self, path_segs: &[&str], combined_qb_key: String, local_ts_key: String) { + if path_segs.is_empty() { + self.entries.push((combined_qb_key, local_ts_key)); + } else { + self.children + .entry(path_segs[0].to_string()) + .or_insert_with(NsTree::new) + .insert(&path_segs[1..], combined_qb_key, local_ts_key); + } + } +} + +/// Build the namespace tree from all submodule tables and views. +fn build_ns_tree<'a>( + ns_tables: &[(String, &'a ModuleDef, &'a TableDef)], + ns_views: &[(String, &'a ModuleDef, &'a ViewDef)], +) -> BTreeMap { + let mut tree: BTreeMap = BTreeMap::new(); + for (prefix, _, table) in ns_tables { + let source_name = submodule_source_name(prefix, table.accessor_name.deref()); + let local = table.accessor_name.deref().to_case(Case::Camel); + // prefix like "lib." → segments ["lib"], or "lib.sublib." → ["lib", "sublib"] + let segs: Vec<&str> = prefix.trim_end_matches('.').split('.').collect(); + if let Some((first, rest)) = segs.split_first() { + tree.entry(first.to_string()) + .or_insert_with(NsTree::new) + .insert(rest, source_name, local); + } + } + for (prefix, _, view) in ns_views { + let source_name = submodule_source_name(prefix, view.accessor_name.deref()); + let local = view.accessor_name.deref().to_case(Case::Camel); + let segs: Vec<&str> = prefix.trim_end_matches('.').split('.').collect(); + if let Some((first, rest)) = segs.split_first() { + tree.entry(first.to_string()) + .or_insert_with(NsTree::new) + .insert(rest, source_name, local); + } + } + tree +} + +/// Recursively emit the namespace tree as nested TypeScript object blocks. +fn emit_ns_tree(out: &mut Indenter, tree: &BTreeMap) { + for (ns, node) in tree { + writeln!(out, "{ns}: {{"); + out.indent(1); + for (qb_key, local_key) in &node.entries { + writeln!(out, "{local_key}: _qb[\"{qb_key}\"],"); + } + emit_ns_tree(out, &node.children); + out.dedent(1); + writeln!(out, "}},"); + } +} + +/// Build namespace tree for submodule reducers (uses `.` path separator). +/// `flat_key` matches SDK's `accessorName = toCamelCase(wireName)`. +/// SDK toCamelCase only splits on `_`/`-`, so `/` is kept verbatim: +/// `"lib.library_reducer"` → `"lib.libraryReducer"`. Bracket notation is required. +fn build_reducer_ns_tree<'a>(ns_reducers: &[(String, &'a ModuleDef, &'a ReducerDef)]) -> BTreeMap { + let mut tree: BTreeMap = BTreeMap::new(); + for (prefix, _, reducer) in ns_reducers { + if !is_reducer_invokable(reducer) { + continue; + } + let flat_key = format!("{}{}", prefix, reducer.accessor_name.deref().to_case(Case::Camel)); + let local = reducer.accessor_name.deref().to_case(Case::Camel); + let segs: Vec<&str> = prefix.trim_end_matches('.').split('.').collect(); + if let Some((first, rest)) = segs.split_first() { + tree.entry(first.to_string()) + .or_insert_with(NsTree::new) + .insert(rest, flat_key, local); + } + } + tree +} + +/// Build namespace tree for submodule procedures (uses `.` path separator). +fn build_procedure_ns_tree<'a>( + ns_procedures: &[(String, &'a ModuleDef, &'a ProcedureDef)], +) -> BTreeMap { + let mut tree: BTreeMap = BTreeMap::new(); + for (prefix, _, procedure) in ns_procedures { + let flat_key = format!("{}{}", prefix, procedure.accessor_name.deref().to_case(Case::Camel)); + let local = procedure.accessor_name.deref().to_case(Case::Camel); + let segs: Vec<&str> = prefix.trim_end_matches('.').split('.').collect(); + if let Some((first, rest)) = segs.split_first() { + tree.entry(first.to_string()) + .or_insert_with(NsTree::new) + .insert(rest, flat_key, local); + } + } + tree +} + +/// Emit a namespace tree for reducers/procedures using bracket notation. +/// Flat keys contain `/` (e.g. `"lib/libraryReducer"`) so dot notation is invalid JS. +fn emit_fn_ns_tree(out: &mut Indenter, map_var: &str, tree: &BTreeMap) { + for (ns, node) in tree { + writeln!(out, "{ns}: {{"); + out.indent(1); + for (flat_key, local_key) in &node.entries { + writeln!(out, "{local_key}: {map_var}[\"{flat_key}\"],"); + } + emit_fn_ns_tree(out, map_var, &node.children); + out.dedent(1); + writeln!(out, "}},"); + } +} + pub fn type_name(module: &ModuleDef, ty: &AlgebraicTypeUse) -> String { let mut s = String::new(); write_type(module, &mut s, ty, None, None).unwrap(); diff --git a/crates/core/src/client/message_handlers_v1.rs b/crates/core/src/client/message_handlers_v1.rs index 60d8eafdb28..32421dd808a 100644 --- a/crates/core/src/client/message_handlers_v1.rs +++ b/crates/core/src/client/message_handlers_v1.rs @@ -49,7 +49,7 @@ pub async fn handle(client: &ClientConnection, message: DataMessage, timer: Inst res.map_err(|e| { ( Some(reducer), - mod_info.module_def.reducer_full(&**reducer).map(|(id, _)| id), + mod_info.module_def.reducer_by_name(reducer).map(|(id, _)| id), e.into(), ) }) diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 982ab6dcbff..9aa128a41e2 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -21,12 +21,12 @@ use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, IndexScanPointOr use spacetimedb_datastore::traits::IsolationLevel; use spacetimedb_lib::{http as st_http, ConnectionId, Identity, Timestamp}; use spacetimedb_primitives::{ColId, ColList, IndexId, TableId}; +use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::{ bsatn::{self, ToBsatn}, buffer::CountWriter, AlgebraicValue, ProductValue, }; -use spacetimedb_schema::identifier::Identifier; use spacetimedb_table::indexes::RowPointer; use spacetimedb_table::table::RowRef; use std::fmt::Display; @@ -48,7 +48,7 @@ pub struct InstanceEnv { /// The type of the last, including current, function to be executed by this environment. pub func_type: FuncCallType, /// The name of the last, including current, function to be executed by this environment. - pub func_name: Option, + pub func_name: Option, /// Are we in an anonymous tx context? in_anon_tx: bool, /// A procedure's last known transaction offset. @@ -246,7 +246,7 @@ impl InstanceEnv { } /// Signal to this `InstanceEnv` that a function call is beginning. - pub fn start_funcall(&mut self, name: Identifier, ts: Timestamp, func_type: FuncCallType) { + pub fn start_funcall(&mut self, name: RawIdentifier, ts: Timestamp, func_type: FuncCallType) { self.start_time = ts; self.start_instant = Instant::now(); self.func_type = func_type; diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 8a1bbc6b414..ccc7af69dd8 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -1,6 +1,6 @@ use super::{ - ArgsTuple, FunctionArgs, InvalidProcedureArguments, InvalidReducerArguments, ReducerCallResult, - ReducerCallResultWithTxOffset, ReducerId, ReducerOutcome, Scheduler, + ArgsTuple, FunctionArgs, InvalidProcedureArguments, InvalidReducerArguments, ReducerCallResult, ReducerId, + ReducerOutcome, Scheduler, }; use crate::client::messages::{OneOffQueryResponseMessage, ProcedureResultMessage, SerializableMessage}; use crate::client::{ClientActorId, ClientConnectionSender, WsVersion}; @@ -10,7 +10,7 @@ use crate::db::sql::ast::SchemaViewer; use crate::error::DBError; use crate::estimation::{check_row_limit, estimate_rows_scanned}; use crate::hash::Hash; -use crate::host::host_controller::{CallProcedureReturn, ProcedureCallResult}; +use crate::host::host_controller::{CallProcedureReturn, ProcedureCallResult, ReducerCallResultWithTxOffset}; use crate::host::scheduler::{CallScheduledFunctionError, CallScheduledFunctionResult, ScheduledFunctionParams}; use crate::host::v8::{JsFatalHook, JsMainInstance, JsProcedureCallCompletion, JsProcedureInstance}; pub use crate::host::wasm_common::module_host_actor::{InstanceCommon, WasmInstance}; @@ -63,7 +63,7 @@ use spacetimedb_lib::{bsatn, ConnectionId, TimeDuration, Timestamp}; use spacetimedb_primitives::{HttpHandlerId, ProcedureId, TableId, ViewFnPtr, ViewId}; use spacetimedb_query::compile_subscription; use spacetimedb_sats::raw_identifier::RawIdentifier; -use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, ProductValue}; +use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, ProductValue, Typespace}; use spacetimedb_schema::auto_migrate::{AutoMigrateError, MigrationPolicy}; use spacetimedb_schema::def::{ModuleDef, ProcedureDef, ReducerDef, ViewDef}; use spacetimedb_schema::identifier::Identifier; @@ -549,18 +549,6 @@ impl GenericModuleInstance for super::v8::JsProcedureInstance { } } -/// Creates the table for `view_def` in `stdb`. -pub fn create_table_from_view_def( - stdb: &RelationalDB, - tx: &mut MutTxId, - module_def: &ModuleDef, - view_def: &ViewDef, -) -> anyhow::Result<()> { - stdb.create_view(tx, module_def, view_def) - .with_context(|| format!("failed to create table for view {}", &view_def.name))?; - Ok(()) -} - /// Moves out the `trapped: bool` from `res`. fn extract_trapped(res: Result<(T, bool), E>) -> (Result, bool) { match res { @@ -595,21 +583,37 @@ fn init_database_inner( let auth_ctx = AuthCtx::for_current(owner_identity); let (tx, ()) = stdb .with_auto_rollback(tx, |tx| { - // Create all in-memory tables defined by the module, - // with IDs ordered lexicographically by the table names. - let mut table_defs: Vec<_> = module_def.tables().collect(); - table_defs.sort_by_key(|x| &x.name); - for def in table_defs { - logger.info(&format!("Creating table `{}`", &def.name)); - spacetimedb_engine::update::create_table_from_def(stdb, tx, module_def, def)?; + // Create all in-memory tables defined by the module (including submodules), + // with IDs ordered lexicographically by their full namespaced names. + let mut table_defs = module_def.all_tables_with_prefix(); + table_defs.sort_by(|(p1, _, d1), (p2, _, d2)| { + let n1 = format!("{}{}", p1, d1.name); + let n2 = format!("{}{}", p2, d2.name); + n1.cmp(&n2) + }); + for (prefix, owning_def, def) in table_defs { + let display_name = format!("{}{}", prefix, def.name); + logger.info(&format!("Creating table `{}`", display_name)); + spacetimedb_engine::update::create_table_from_def_with_prefix(stdb, tx, owning_def, def, &prefix)?; } - // Create all in-memory views defined by the module. - let mut view_defs: Vec<_> = module_def.views().collect(); - view_defs.sort_by_key(|x| &x.name); - for def in view_defs { - logger.info(&format!("Creating table for view `{}`", &def.name)); - create_table_from_view_def(stdb, tx, module_def, def)?; + // Create all in-memory views defined by the module (root + submodule). + let mut view_defs: Vec<(String, &ModuleDef, &ViewDef)> = module_def.all_views_with_prefix(); + view_defs.sort_by(|(p1, _, d1), (p2, _, d2)| { + let n1 = format!("{}{}", p1, d1.name); + let n2 = format!("{}{}", p2, d2.name); + n1.cmp(&n2) + }); + for (prefix, owning_def, def) in view_defs { + let display_name = format!("{}{}", prefix, def.name); + logger.info(&format!("Creating table for view `{}`", display_name)); + if prefix.is_empty() { + spacetimedb_engine::update::create_table_from_view_def(stdb, tx, owning_def, def)?; + } else { + spacetimedb_engine::update::create_table_from_view_def_with_prefix( + stdb, tx, owning_def, def, &prefix, + )?; + } } // Insert the late-bound row-level security expressions. @@ -707,7 +711,7 @@ pub fn call_identity_connected( // abort the connection: we can't really recover. let tx = Some(ScopeGuard::into_inner(mut_tx)); let params = ModuleHost::call_reducer_params( - module, + &module.module_def, caller_auth.claims.identity, Some(caller_connection_id), None, @@ -1075,7 +1079,7 @@ impl ProcedureResultTarget { } pub struct CallViewParams { - pub view_name: Identifier, + pub view_name: RawIdentifier, pub view_id: ViewId, pub table_id: TableId, pub fn_ptr: ViewFnPtr, @@ -1088,6 +1092,10 @@ pub struct CallViewParams { pub args: ArgsTuple, pub row_type: AlgebraicTypeRef, pub timestamp: Timestamp, + /// The typespace of the module that owns this view. + /// For root views this equals the top-level typespace; + /// for submodule views this is the submodule's own typespace. + pub view_typespace: Typespace, } pub(crate) struct ResolvedViewForRefresh<'a> { @@ -1112,14 +1120,15 @@ pub(crate) fn resolve_view_for_refresh<'a>( .table_id .ok_or_else(|| anyhow::anyhow!("view {:?} does not have a backing table", view_id))?; - let view_name: Identifier = st_view.view_name.into(); - let view_def = module_def.view(&view_name).ok_or_else(|| { - anyhow::anyhow!( - "view `{}` for view id `{}` not found in current module", - view_name, - view_id - ) - })?; + let (_, view_def, _) = module_def + .view_by_name_with_global_fn_ptr(st_view.view_name.as_ref()) + .ok_or_else(|| { + anyhow::anyhow!( + "view `{}` for view id `{}` not found in current module", + st_view.view_name, + view_id + ) + })?; let is_anonymous = view_def.is_anonymous; @@ -1128,7 +1137,7 @@ pub(crate) fn resolve_view_for_refresh<'a>( "found is_anonymous={} in st_view, but {} in module when updating view `{}`", st_view.is_anonymous, is_anonymous, - view_name, + st_view.view_name, )); } @@ -2118,7 +2127,7 @@ impl ModuleHost { // that `st_client` is updated appropriately. let tx = Some(mut_tx); let result = Self::call_reducer_params( - info, + &info.module_def, caller_identity, Some(caller_connection_id), None, @@ -2205,7 +2214,7 @@ impl ModuleHost { } fn call_reducer_params( - module: &ModuleInfo, + owning_def: &ModuleDef, caller_identity: Identity, caller_connection_id: Option, client: Option>, @@ -2216,7 +2225,7 @@ impl ModuleHost { args: FunctionArgs, ) -> Result { let args = args - .into_tuple_for_def(&module.module_def, reducer_def) + .into_tuple_for_def(owning_def, reducer_def) .map_err(InvalidReducerArguments)?; let caller_connection_id = caller_connection_id.unwrap_or(ConnectionId::ZERO); Ok(CallReducerParams { @@ -2241,10 +2250,10 @@ impl ModuleHost { reducer_name: &str, args: FunctionArgs, ) -> Result<(&'a ReducerDef, CallReducerParams), ReducerCallError> { - let (reducer_id, reducer_def) = self + let (reducer_id, reducer_def, owning_def) = self .info .module_def - .reducer_full(reducer_name) + .reducer_by_name_with_module(reducer_name) .ok_or(ReducerCallError::NoSuchReducer)?; if let Some(lifecycle) = reducer_def.lifecycle { return Err(ReducerCallError::LifecycleReducer(lifecycle)); @@ -2257,7 +2266,7 @@ impl ModuleHost { Ok(( reducer_def, Self::call_reducer_params( - &self.info, + owning_def, caller_identity, caller_connection_id, client, @@ -2754,10 +2763,10 @@ impl ModuleHost { procedure_name: &str, args: FunctionArgs, ) -> Result<(&'a ProcedureDef, CallProcedureParams), ProcedureCallError> { - let (procedure_id, procedure_def) = self + let (procedure_id, procedure_def, owning_def) = self .info .module_def - .procedure_full(procedure_name) + .procedure_by_name_with_module(procedure_name) .ok_or(ProcedureCallError::NoSuchProcedure)?; if procedure_def.visibility.is_private() && !self.is_database_owner(caller_identity) { @@ -2765,7 +2774,7 @@ impl ModuleHost { } let args = args - .into_tuple_for_def(&self.info.module_def, procedure_def) + .into_tuple_for_def(owning_def, procedure_def) .map_err(InvalidProcedureArguments)?; let caller_connection_id = caller_connection_id.unwrap_or(ConnectionId::ZERO); @@ -2856,10 +2865,11 @@ impl ModuleHost { } /// Materializes the views return by the `view_collector`, if not already materialized, - /// and updates view lifecycle state accordingly. + /// and updates `st_view_sub` accordingly. /// - /// Passing [`Workload::Sql`] will update the instance's last-used timestamp. - /// Passing [`Workload::Subscribe`] will also increment the subscriber's refcount. + /// Passing [`Workload::Sql`] will update `st_view_sub.last_called`. + /// Passing [`Workload::Subscribe`] will also increment `st_view_sub.num_subscribers`, + /// in addition to updating `st_view_sub.last_called`. pub fn materialize_views( mut tx: MutTxId, instance: &mut RefInstance<'_, I>, @@ -2872,7 +2882,7 @@ impl ModuleHost { view_collector.collect_views(&mut view_ids); for view_id in view_ids { let st_view_row = tx.lookup_st_view(view_id)?; - let view_name = st_view_row.view_name.into(); + let view_name: RawIdentifier = st_view_row.view_name.into(); let view_id = st_view_row.view_id; let table_id = st_view_row.table_id.ok_or(ViewCallError::TableDoesNotExist(view_id))?; let is_anonymous = st_view_row.is_anonymous; @@ -2956,8 +2966,14 @@ impl ModuleHost { table_id, view_def, } = resolved; - let view_name = &view_def.name; - let args = match FunctionArgs::Nullary.into_tuple_for_def(module_def, view_def) { + let Some((fn_ptr, view_def, owning_def)) = + module_def.view_by_name_with_global_fn_ptr(view_def.name.as_raw().as_ref()) + else { + outcome = ViewOutcome::Failed(format!("view for view id `{}` not found in module", view_id)); + break; + }; + let view_name = view_def.name.as_raw(); + let args = match FunctionArgs::Nullary.into_tuple_for_def(owning_def, view_def) { Ok(args) => args, Err(err) => { outcome = ViewOutcome::Failed(format!("failed to build view args: {err}")); @@ -2971,12 +2987,13 @@ impl ModuleHost { view_name, view_id, table_id, - view_def.fn_ptr, + fn_ptr, caller, sender, args, view_def.product_type_ref, timestamp, + owning_def.typespace().clone(), ); // Increment execution stats @@ -3007,7 +3024,7 @@ impl ModuleHost { fn call_view( instance: &mut RefInstance<'_, I>, tx: MutTxId, - view_name: &Identifier, + view_name: &RawIdentifier, view_id: ViewId, table_id: TableId, args: FunctionArgs, @@ -3030,7 +3047,7 @@ impl ModuleHost { fn call_view_at( instance: &mut RefInstance<'_, I>, tx: MutTxId, - view_name: &Identifier, + view_name: &RawIdentifier, view_id: ViewId, table_id: TableId, args: FunctionArgs, @@ -3039,22 +3056,34 @@ impl ModuleHost { timestamp: Timestamp, ) -> Result<(ViewCallResult, bool), ViewCallError> { let module_def = &instance.common.info().module_def; - let view_def = module_def.view(view_name).ok_or(ViewCallError::NoSuchView)?; - let fn_ptr = view_def.fn_ptr; + let (global_fn_ptr, view_def, owning_def) = module_def + .view_by_name_with_global_fn_ptr(view_name.as_ref()) + .ok_or(ViewCallError::NoSuchView)?; let row_type = view_def.product_type_ref; let args = args - .into_tuple_for_def(module_def, view_def) + .into_tuple_for_def(owning_def, view_def) .map_err(InvalidViewArguments)?; Ok(Self::call_view_inner( - instance, tx, view_name, view_id, table_id, fn_ptr, caller, sender, args, row_type, timestamp, + instance, + tx, + view_name, + view_id, + table_id, + global_fn_ptr, + caller, + sender, + args, + row_type, + timestamp, + owning_def.typespace().clone(), )) } fn call_view_inner( instance: &mut RefInstance<'_, I>, tx: MutTxId, - name: &Identifier, + name: &RawIdentifier, view_id: ViewId, table_id: TableId, fn_ptr: ViewFnPtr, @@ -3063,6 +3092,7 @@ impl ModuleHost { args: ArgsTuple, row_type: AlgebraicTypeRef, timestamp: Timestamp, + view_typespace: Typespace, ) -> (ViewCallResult, bool) { let view_name = name.clone(); let params = CallViewParams { @@ -3075,6 +3105,7 @@ impl ModuleHost { sender, args, row_type, + view_typespace, }; instance.common.call_view_with_tx(tx, params, instance.instance) diff --git a/crates/core/src/host/scheduler.rs b/crates/core/src/host/scheduler.rs index 295574606e2..393fb8a75c3 100644 --- a/crates/core/src/host/scheduler.rs +++ b/crates/core/src/host/scheduler.rs @@ -306,7 +306,7 @@ impl ScheduledFunctionParams { } fn kind(&self, module: &ModuleInfo) -> ScheduledFunctionKind { - if module.module_def.procedure_full(self.function_name()).is_some() { + if module.module_def.procedure_by_name(self.function_name()).is_some() { ScheduledFunctionKind::Procedure } else { ScheduledFunctionKind::Reducer @@ -775,11 +775,14 @@ fn function_to_reducer_call_params( ) -> anyhow::Result<(Timestamp, Instant, CallReducerParams)> { let identity = module.database_identity; + // Find the reducer and deserialize the arguments. + // Use the owning module's typespace (not necessarily the root's) so that type-index + // references inside the def are resolved correctly for submodules. let module = &module.module_def; - let Some((id, def)) = module.reducer_full(name) else { + let Some((id, def, owning)) = module.reducer_by_name_with_module(name) else { return Err(anyhow!("Reducer `{name}` not found")); }; - let args = args.into_tuple_for_def(module, def).map_err(InvalidReducerArguments)?; + let args = args.into_tuple_for_def(owning, def).map_err(InvalidReducerArguments)?; let (ts, instant) = scheduled_call_time(at); Ok((ts, instant, CallReducerParams::from_system(ts, identity, id, args))) @@ -794,11 +797,11 @@ fn function_to_procedure_call_params( let identity = module.database_identity; let module = &module.module_def; - let Some((id, def)) = module.procedure_full(name) else { + let Some((id, def, owning)) = module.procedure_by_name_with_module(name) else { return Err(anyhow!("Procedure `{name}` not found")); }; let args = args - .into_tuple_for_def(module, def) + .into_tuple_for_def(owning, def) .map_err(InvalidProcedureArguments)?; let (ts, instant) = scheduled_call_time(at); diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 52abf373e48..b425868efa5 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -102,9 +102,9 @@ use spacetimedb_client_api_messages::energy::FunctionBudget; use spacetimedb_datastore::locking_tx_datastore::FuncCallType; use spacetimedb_datastore::traits::Program; use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp}; +use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_schema::auto_migrate::MigrationPolicy; use spacetimedb_schema::def::ModuleDef; -use spacetimedb_schema::identifier::Identifier; use spacetimedb_table::static_assert_size; use std::cell::Cell; use std::num::NonZeroUsize; @@ -381,7 +381,7 @@ impl JsInstanceEnv { /// /// Returns the handle used by reducers to read from `args` /// as well as the handle used to write the error message, if any. - fn start_funcall(&mut self, name: Identifier, ts: Timestamp, func_type: FuncCallType) { + fn start_funcall(&mut self, name: RawIdentifier, ts: Timestamp, func_type: FuncCallType) { self.instance_env.start_funcall(name, ts, func_type); } @@ -1406,7 +1406,7 @@ fn handle_main_worker_request( handle_worker_request("call_reducer", reply_tx, || { let mut call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); let (res, trapped) = call_reducer(None, params); - (res, trapped) + (res.result, trapped) }) } JsMainWorkerRequest::CallReducerDetached { params, on_panic } => { @@ -1458,7 +1458,10 @@ fn handle_main_worker_request( caller_auth, caller_connection_id, } => handle_worker_request("call_identity_connected", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let call_reducer = |tx, params| { + let (res, trapped) = instance_common.call_reducer_with_tx(tx, params, inst); + (res.result, trapped) + }; let mut trapped = false; let res = call_identity_connected(caller_auth, caller_connection_id, &info, call_reducer, &mut trapped); (res, trapped) @@ -1468,7 +1471,10 @@ fn handle_main_worker_request( caller_identity, caller_connection_id, } => handle_worker_request("call_identity_disconnected", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let call_reducer = |tx, params| { + let (res, trapped) = instance_common.call_reducer_with_tx(tx, params, inst); + (res.result, trapped) + }; let mut trapped = false; let res = ModuleHost::call_identity_disconnected_inner( caller_identity, @@ -1481,7 +1487,10 @@ fn handle_main_worker_request( }), JsMainWorkerRequest::DisconnectClient { reply_tx, client_id } => { handle_worker_request("disconnect_client", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let call_reducer = |tx, params| { + let (res, trapped) = instance_common.call_reducer_with_tx(tx, params, inst); + (res.result, trapped) + }; let mut trapped = false; let res = ModuleHost::disconnect_client_inner(client_id, &info, call_reducer, &mut trapped); (res, trapped) @@ -1489,7 +1498,7 @@ fn handle_main_worker_request( } JsMainWorkerRequest::InitDatabase { reply_tx, program } => { handle_worker_request("init_database", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx_offset(tx, params, inst); + let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); let (res, trapped): (Result, bool) = init_database(replica_ctx, &info.module_def, program, call_reducer); (res, trapped) diff --git a/crates/core/src/host/v8/syscall/common.rs b/crates/core/src/host/v8/syscall/common.rs index e6e287c545b..c4ff6ed6347 100644 --- a/crates/core/src/host/v8/syscall/common.rs +++ b/crates/core/src/host/v8/syscall/common.rs @@ -27,8 +27,8 @@ use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCal use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp}; use spacetimedb_primitives::{ColId, IndexId, ProcedureId, TableId, ViewFnPtr}; use spacetimedb_sats::bsatn; +use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_schema::def::ModuleDef; -use spacetimedb_schema::identifier::Identifier; use v8::{FunctionCallbackArguments, Isolate, Local, PinScope, Value}; /// Calls the `__call_procedure__` function `fun`. @@ -761,7 +761,7 @@ fn refresh_views( let table_id = resolved.table_id; let view_def = resolved.view_def; - let view_name = &view_def.name; + let view_name = view_def.name.as_raw(); let fn_ptr = view_def.fn_ptr; let sender = tx .as_ref() @@ -860,7 +860,7 @@ fn call_view( scope: &mut PinScope<'_, '_>, hooks: &HookFunctions<'_>, view_call: &ViewCallInfo, - view_name: &Identifier, + view_name: &RawIdentifier, table_id: TableId, fn_ptr: ViewFnPtr, sender: Option, diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 97951829d16..8380ef46d4f 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -6,18 +6,19 @@ use crate::db::sql::ast::SchemaViewer; use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; use crate::error::DBError; use crate::host::host_controller::CallProcedureReturn; +use crate::host::host_controller::ReducerCallResultWithTxOffset; use crate::host::instance_env::{InstanceEnv, TxSlot}; use crate::host::module_common::{build_common_module_from_raw, ModuleCommon}; use crate::host::module_host::{ call_identity_connected, init_database, CallHttpHandlerParams, CallProcedureParams, CallReducerParams, - CallViewParams, ClientConnectedError, DatabaseUpdate, EventStatus, HttpHandlerCallError, ModuleEvent, - ModuleFunctionCall, ModuleInfo, RefInstance, SqlCommand, SqlCommandResult, ViewCallResult, ViewCommand, - ViewCommandResult, ViewOutcome, + CallViewParams, ClientConnectedError, DatabaseUpdate, EventStatus, HttpHandlerCallError, InitDatabaseResult, + ModuleEvent, ModuleFunctionCall, ModuleInfo, RefInstance, SqlCommand, SqlCommandResult, ViewCallResult, + ViewCommand, ViewCommandResult, ViewOutcome, }; use crate::host::scheduler::{CallScheduledFunctionResult, ScheduledFunctionParams}; use crate::host::{ - ArgsTuple, InitDatabaseResult, ModuleHost, ProcedureCallError, ProcedureCallResult, ReducerCallError, - ReducerCallResult, ReducerCallResultWithTxOffset, ReducerId, ReducerOutcome, Scheduler, UpdateDatabaseResult, + ArgsTuple, ModuleHost, ProcedureCallError, ProcedureCallResult, ReducerCallError, ReducerCallResult, ReducerId, + ReducerOutcome, Scheduler, UpdateDatabaseResult, }; use crate::identity::Identity; use crate::messages::control_db::HostType; @@ -37,7 +38,7 @@ use spacetimedb_auth::identity::ConnectionAuthCtx; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::error::{DatastoreError, ViewError}; use spacetimedb_datastore::execution_context::{self, ReducerContext, Workload}; -use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo}; +use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo, ViewInstanceArgs}; use spacetimedb_datastore::traits::{IsolationLevel, Program}; use spacetimedb_execution::ExecutionParams; use spacetimedb_lib::buffer::DecodeError; @@ -48,6 +49,7 @@ use spacetimedb_lib::metrics::ExecutionMetrics; use spacetimedb_lib::{bsatn, http as st_http, ConnectionId, Hash, ProductType, RawModuleDef, Timestamp}; use spacetimedb_primitives::{HttpHandlerId, ProcedureId, TableId, ViewFnPtr, ViewId}; use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type; +use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, Deserialize, ProductValue, Typespace, WithTypespace}; use spacetimedb_schema::auto_migrate::{MigratePlan, MigrationPolicy, MigrationPolicyError}; use spacetimedb_schema::def::deserialize::FunctionDef; @@ -589,7 +591,9 @@ impl WasmModuleInstance { impl WasmModuleInstance { #[tracing::instrument(level = "trace", skip_all)] fn call_reducer_with_tx(&mut self, tx: Option, params: CallReducerParams) -> (ReducerCallResult, bool) { - let (res, trapped) = self.call_reducer_with_tx_offset(tx, params); + let (res, trapped) = crate::callgrind_flag::invoke_allowing_callgrind(|| { + self.common.call_reducer_with_tx(tx, params, &mut self.instance) + }); (res.result, trapped) } @@ -600,7 +604,7 @@ impl WasmModuleInstance { params: CallReducerParams, ) -> (ReducerCallResultWithTxOffset, bool) { crate::callgrind_flag::invoke_allowing_callgrind(|| { - self.common.call_reducer_with_tx_offset(tx, params, &mut self.instance) + self.common.call_reducer_with_tx(tx, params, &mut self.instance) }) } @@ -758,7 +762,7 @@ impl InstanceCommon { } } - /// Re-evaluates all materialized view instances tracked in view lifecycle state. + /// Re-evaluates all views which have entries in `st_view_subs`. fn evaluate_subscribed_views( &mut self, tx: MutTxId, @@ -955,16 +959,6 @@ impl InstanceCommon { tx: Option, params: CallReducerParams, inst: &mut I, - ) -> (ReducerCallResult, bool) { - let (res, trapped) = self.call_reducer_with_tx_offset(tx, params, inst); - (res.result, trapped) - } - - pub(crate) fn call_reducer_with_tx_offset( - &mut self, - tx: Option, - params: CallReducerParams, - inst: &mut I, ) -> (ReducerCallResultWithTxOffset, bool) { let CallReducerParams { timestamp, @@ -1314,6 +1308,7 @@ impl InstanceCommon { args, row_type, timestamp, + view_typespace, } = params; let _outer_span = start_call_function_span(&view_name, &caller, None); @@ -1368,19 +1363,18 @@ impl InstanceCommon { // This is wrapped in a closure to simplify error handling. let outcome: Result = (|| { let view_call = match sender { - Some(sender) => ViewCallInfo::sender(view_id, sender), + Some(s) => ViewCallInfo::sender(view_id, s), None => ViewCallInfo::anonymous(view_id), }; let result = ViewResult::from_return_data(raw).context("Error parsing view result")?; - let typespace = self.info.module_def.typespace(); - let row_product_type = typespace + let row_product_type = view_typespace .resolve(row_type) .resolve_refs()? .into_product() .map_err(|_| anyhow!("Error resolving row type for view"))?; let rows = match result { - ViewResult::Rows(bytes) => deserialize_view_rows(row_type, bytes, typespace) + ViewResult::Rows(bytes) => deserialize_view_rows(row_type, bytes, &view_typespace) .context("Error deserializing rows returned by view".to_string())?, ViewResult::RawSql(query) => self .run_query_for_view(&mut tx, &query, &row_product_type, &view_call) @@ -1503,57 +1497,71 @@ fn collect_subscribed_view_calls( ) -> Result, anyhow::Error> { let mut view_calls = Vec::new(); - for view in module_def.views() { + for (prefix, owning_def, view) in module_def.all_views_with_prefix() { let ViewDef { - name: view_name, + accessor_name, + name: local_name, is_anonymous, - fn_ptr, product_type_ref, .. } = view; + let display_name = if prefix.is_empty() { + local_name.to_string() + } else { + format!("{}{}", prefix, accessor_name) + }; + + let (global_fn_ptr, _, _) = module_def + .view_by_name_with_global_fn_ptr(&display_name) + .ok_or_else(|| anyhow::anyhow!("view {} not found in module_def", display_name))?; + let st_view = tx - .view_from_name(view_name)? - .ok_or_else(|| anyhow::anyhow!("view {} not found in database", &view_name))?; + .view_from_name(&display_name)? + .ok_or_else(|| anyhow::anyhow!("view {} not found in database", display_name))?; let view_id = st_view.view_id; let table_id = st_view .table_id - .ok_or_else(|| anyhow::anyhow!("view {} does not have a backing table in database", &view_name))?; - let view_instances = tx.materialized_view_instances_for_view(view_id); + .ok_or_else(|| anyhow::anyhow!("view {} does not have a backing table in database", display_name))?; + let subs = tx.materialized_view_instances_for_view(view_id); + let view_name = RawIdentifier::from(display_name); + let view_typespace = owning_def.typespace().clone(); if *is_anonymous { - if view_instances.is_empty() { + if !subs.iter().any(|a| matches!(a, ViewInstanceArgs::Anonymous)) { continue; } view_calls.push(CallViewParams { view_name: view_name.clone(), view_id, table_id, - fn_ptr: *fn_ptr, + fn_ptr: global_fn_ptr, caller: owner_identity, sender: None, args: ArgsTuple::nullary(), row_type: *product_type_ref, timestamp: Timestamp::now(), + view_typespace: view_typespace.clone(), }); continue; } - for args in view_instances { - let Some(sender) = args.sender() else { + for sub in subs { + let ViewInstanceArgs::Sender(identity) = sub else { continue; }; view_calls.push(CallViewParams { view_name: view_name.clone(), view_id, table_id, - fn_ptr: *fn_ptr, + fn_ptr: global_fn_ptr, caller: owner_identity, - sender: Some(sender), + sender: Some(identity), args: ArgsTuple::nullary(), row_type: *product_type_ref, timestamp: Timestamp::now(), + view_typespace: view_typespace.clone(), }); } } @@ -1580,7 +1588,7 @@ impl AllVmMetrics { let def = &info.module_def; let reducers = def.reducer_ids_and_defs(); let num_reducers = reducers.len() as u32; - let reducers = reducers.map(|(_, def)| def.name()); + let reducers = reducers.into_iter().map(|(_, def)| def.name()); // These are the views: let views = def.views().map(|def| def.name()); @@ -1770,7 +1778,7 @@ fn lifecyle_modifications_to_tx( */ pub trait InstanceOp { - fn name(&self) -> &Identifier; + fn name(&self) -> &RawIdentifier; fn timestamp(&self) -> Timestamp; fn call_type(&self) -> FuncCallType; } @@ -1778,7 +1786,7 @@ pub trait InstanceOp { /// Describes a view call in a cheaply shareable way. #[derive(Clone, Debug)] pub struct ViewOp<'a> { - pub name: &'a Identifier, + pub name: &'a RawIdentifier, pub view_id: ViewId, pub table_id: TableId, pub fn_ptr: ViewFnPtr, @@ -1788,7 +1796,7 @@ pub struct ViewOp<'a> { } impl InstanceOp for ViewOp<'_> { - fn name(&self) -> &Identifier { + fn name(&self) -> &RawIdentifier { self.name } @@ -1804,7 +1812,7 @@ impl InstanceOp for ViewOp<'_> { /// Describes an anonymous view call in a cheaply shareable way. #[derive(Clone, Debug)] pub struct AnonymousViewOp<'a> { - pub name: &'a Identifier, + pub name: &'a RawIdentifier, pub view_id: ViewId, pub table_id: TableId, pub fn_ptr: ViewFnPtr, @@ -1813,7 +1821,7 @@ pub struct AnonymousViewOp<'a> { } impl InstanceOp for AnonymousViewOp<'_> { - fn name(&self) -> &Identifier { + fn name(&self) -> &RawIdentifier { self.name } @@ -1839,8 +1847,8 @@ pub struct ReducerOp<'a> { } impl InstanceOp for ReducerOp<'_> { - fn name(&self) -> &Identifier { - self.name.as_identifier() + fn name(&self) -> &RawIdentifier { + self.name.as_identifier().as_raw() } fn timestamp(&self) -> Timestamp { self.timestamp @@ -1883,8 +1891,8 @@ pub struct ProcedureOp { } impl InstanceOp for ProcedureOp { - fn name(&self) -> &Identifier { - &self.name + fn name(&self) -> &RawIdentifier { + self.name.as_raw() } fn timestamp(&self) -> Timestamp { self.timestamp @@ -1905,8 +1913,8 @@ pub struct HttpHandlerOp { } impl InstanceOp for HttpHandlerOp { - fn name(&self) -> &Identifier { - &self.name + fn name(&self) -> &RawIdentifier { + self.name.as_raw() } fn timestamp(&self) -> Timestamp { self.timestamp @@ -1960,9 +1968,16 @@ mod tests { let mut tx = begin_mut_tx(&stdb); let (view_id, _table_id) = stdb.create_view(&mut tx, &module_def, view_def)?; - let view_call = ViewCallInfo::anonymous(view_id); - tx.subscribe_view(view_call.clone(), ViewInstanceArgs::Anonymous, Identity::ZERO)?; - tx.subscribe_view(view_call, ViewInstanceArgs::Anonymous, Identity::ONE)?; + tx.subscribe_view( + ViewCallInfo::anonymous(view_id), + ViewInstanceArgs::Anonymous, + Identity::ZERO, + )?; + tx.subscribe_view( + ViewCallInfo::anonymous(view_id), + ViewInstanceArgs::Anonymous, + Identity::ONE, + )?; // Two subscriber rows exist, but anonymous views should still be reevaluated once // because they share a single materialization. @@ -1990,10 +2005,16 @@ mod tests { let mut tx = begin_mut_tx(&stdb); let (view_id, _table_id) = stdb.create_view(&mut tx, &module_def, view_def)?; - let zero_args = ViewInstanceArgs::Sender(Identity::ZERO); - let one_args = ViewInstanceArgs::Sender(Identity::ONE); - tx.subscribe_view(ViewCallInfo::from_args(view_id, zero_args), zero_args, Identity::ZERO)?; - tx.subscribe_view(ViewCallInfo::from_args(view_id, one_args), one_args, Identity::ONE)?; + tx.subscribe_view( + ViewCallInfo::sender(view_id, Identity::ZERO), + ViewInstanceArgs::Sender(Identity::ZERO), + Identity::ZERO, + )?; + tx.subscribe_view( + ViewCallInfo::sender(view_id, Identity::ONE), + ViewInstanceArgs::Sender(Identity::ONE), + Identity::ONE, + )?; // Sender-backed views keep one materialization per sender, so reevaluation must // preserve both callers. diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index e429be2056e..d450c21898a 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -23,8 +23,8 @@ use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCal use spacetimedb_lib::{bsatn, ConnectionId, Identity, Timestamp}; use spacetimedb_primitives::errno::HOST_CALL_FAILURE; use spacetimedb_primitives::{errno, ColId, ViewFnPtr}; +use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_schema::def::ModuleDef; -use spacetimedb_schema::identifier::Identifier; use std::future::Future; use std::num::NonZeroU32; use std::sync::Arc; @@ -333,7 +333,7 @@ impl WasmInstanceEnv { /// as well as the handle used to write the reducer error message or procedure return value. pub fn start_funcall( &mut self, - name: Identifier, + name: RawIdentifier, args: bytes::Bytes, ts: Timestamp, func_type: FuncCallType, @@ -1775,7 +1775,7 @@ impl WasmInstanceEnv { let table_id = resolved.table_id; let view_def = resolved.view_def; - let view_name = &view_def.name; + let view_name = view_def.name.as_raw(); let fn_ptr = view_def.fn_ptr; let sender = tx .as_ref() @@ -1846,7 +1846,7 @@ impl WasmInstanceEnv { fn call_view<'a>( caller: &mut Caller<'a, Self>, view_call: &ViewCallInfo, - view_name: &Identifier, + view_name: &RawIdentifier, fn_ptr: ViewFnPtr, sender: Option, ) -> anyhow::Result { diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index 1ec062ff1d5..6d951ebb15d 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -19,8 +19,8 @@ use futures_util::FutureExt; use spacetimedb_datastore::locking_tx_datastore::FuncCallType; use spacetimedb_lib::{bsatn, ConnectionId, Identity, RawModuleDef}; use spacetimedb_primitives::errno::HOST_CALL_FAILURE; +use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_schema::def::ModuleDef; -use spacetimedb_schema::identifier::Identifier; use wasmtime::{ AsContext, AsContextMut, Caller, ExternType, Instance, InstancePre, Linker, Store, TypedFunc, WasmBacktrace, WasmParams, WasmResults, @@ -244,7 +244,7 @@ pub(super) fn call_view_export( mut ctx: impl AsContextMut, call_view: Option, call_view_anon: Option, - view_name: &Identifier, + view_name: &RawIdentifier, fn_ptr: u32, sender: Option, args_source: u32, @@ -630,7 +630,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { // Prepare arguments to the reducer + the error sink & start timings. let args_bytes = op.args.get_bsatn().clone(); - let reducer_name = op.name.clone().into(); + let reducer_name: RawIdentifier = op.name.clone().into(); let (args_source, errors_sink) = store .data_mut() @@ -754,10 +754,12 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let [conn_id_0, conn_id_1] = prepare_connection_id_for_call(op.caller_connection_id); // Prepare arguments to the reducer + the error sink & start timings. - let (args_source, result_sink) = - store - .data_mut() - .start_funcall(op.name.clone(), op.arg_bytes, op.timestamp, FuncCallType::Procedure); + let (args_source, result_sink) = store.data_mut().start_funcall( + op.name.clone().into(), + op.arg_bytes, + op.timestamp, + FuncCallType::Procedure, + ); let Some(call_procedure) = self.call_procedure.as_ref() else { let res = module_host_actor::ProcedureExecuteResult { @@ -825,7 +827,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let (request_source, response_sink) = store .data_mut() - .start_funcall(op.name.clone(), op.request_bytes, op.timestamp, call_type); + .start_funcall(op.name.clone().into(), op.request_bytes, op.timestamp, call_type); let request_body_source = store .data_mut() .create_extra_bytes_source(op.request_body_bytes) diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index 3debc259afe..6ea0f94cfad 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -789,6 +789,48 @@ impl MutTxId { Ok((view_id, table_id)) } + /// Like [`create_view`] but registers the view under `name_prefix + view_def.accessor_name` + /// (e.g. `"lib.library_view"`), using `owning_def` for type resolution. + /// + /// Used for submodule views whose canonical names are dot-namespaced. + pub fn create_view_with_prefix( + &mut self, + owning_def: &ModuleDef, + view_def: &ViewDef, + name_prefix: &str, + ) -> Result<(ViewId, TableId)> { + let mut table_schema = TableSchema::from_view_def_for_datastore(owning_def, view_def); + let prefixed_name = format!("{}{}", name_prefix, &*view_def.accessor_name); + table_schema.table_name = TableName::new_raw(RawIdentifier::from(prefixed_name.clone())); + + // Clear alias so st_table_accessor doesn't get the bare (un-prefixed) accessor name, + // which would conflict when two mounts have views with the same local name. + // The namespaced canonical name is already the unique lookup key (same as tables). + table_schema.alias = None; + + // Prefix index and constraint names so they remain globally unique across mounts. + for index in &mut table_schema.indexes { + index.index_name = RawIdentifier::from(format!("{}{}", name_prefix, index.index_name)); + } + for constraint in &mut table_schema.constraints { + constraint.constraint_name = RawIdentifier::from(format!("{}{}", name_prefix, constraint.constraint_name)); + } + + let table_id = self.create_table(table_schema)?; + + let view_name = RawIdentifier::from(prefixed_name); + let view_id = self + .view_id_from_name(&view_name)? + .ok_or(ViewError::NotFound(view_name))?; + + self.insert_into_st_view_param(view_id, &view_def.param_columns)?; + self.insert_into_st_view_column(view_id, &view_def.return_columns)?; + + self.committed_state_write_lock.ephemeral_tables.insert(table_id); + + Ok((view_id, table_id)) + } + /// Drop the backing table of a view and update the system tables. pub fn drop_view(&mut self, view_id: ViewId) -> Result<()> { let st_view_row = self.lookup_st_view(view_id)?; diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index 5ba71da15cb..d8b15b4b571 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -1203,6 +1203,16 @@ impl RelationalDB { Ok(tx.create_view(module_def, view_def)?) } + pub fn create_view_with_prefix( + &self, + tx: &mut MutTx, + owning_def: &ModuleDef, + view_def: &ViewDef, + name_prefix: &str, + ) -> Result<(ViewId, TableId), DBError> { + Ok(tx.create_view_with_prefix(owning_def, view_def, name_prefix)?) + } + pub fn drop_view(&self, tx: &mut MutTx, view_id: ViewId) -> Result<(), DBError> { Ok(tx.drop_view(view_id)?) } diff --git a/crates/engine/src/update.rs b/crates/engine/src/update.rs index 98094a69947..8ce0c9b89d4 100644 --- a/crates/engine/src/update.rs +++ b/crates/engine/src/update.rs @@ -8,12 +8,15 @@ use spacetimedb_datastore::system_tables::{StViewFields, StViewRow, ST_VIEW_ID}; use spacetimedb_lib::db::auth::StTableType; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::AlgebraicValue; -use spacetimedb_primitives::{ColSet, TableId}; +use spacetimedb_primitives::{ColSet, ConstraintId, TableId}; +use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_schema::auto_migrate::{AutoMigratePlan, AutoMigrateStep, ManualMigratePlan, MigratePlan}; -use spacetimedb_schema::def::{ModuleDef, ModuleDefLookup, TableDef, ViewDef}; +use spacetimedb_schema::def::{ModuleDef, TableDef, ViewDef}; +use spacetimedb_schema::identifier::{Identifier, NamespacedIdentifier}; use spacetimedb_schema::schema::{ column_schemas_from_defs, ConstraintSchema, IndexSchema, Schema, SequenceSchema, TableSchema, }; +use spacetimedb_schema::table_name::TableName; /// The logger used for by [`update_database`] and friends. pub trait UpdateLogger { @@ -48,18 +51,19 @@ pub fn stale_view_backing_table_recreate_plan<'def>( }))) } -fn stale_view_backing_table_recreate_steps<'def>( +fn stale_view_backing_table_recreate_steps( stdb: &RelationalDB, - module_def: &'def ModuleDef, -) -> anyhow::Result>> { + module_def: &ModuleDef, +) -> anyhow::Result> { stdb.with_read_only(Workload::Internal, |tx| -> anyhow::Result<_> { let mut steps = Vec::new(); for view in module_def.views() { if view_backing_table_needs_recreate(stdb, tx, module_def, view)? { + let name = NamespacedIdentifier::new_assume_valid(view.name.to_string()); steps.extend([ - AutoMigrateStep::RemoveView(view.key()), - AutoMigrateStep::AddView(view.key()), + AutoMigrateStep::RemoveView(name.clone()), + AutoMigrateStep::AddView(name), ]); } } @@ -105,7 +109,7 @@ fn view_backing_row_layout_changed(actual: &TableSchema, expected: &TableSchema) actual.row_type != expected.row_type } -fn ensure_disconnect_all_users(steps: &mut Vec>) { +fn ensure_disconnect_all_users(steps: &mut Vec) { if !steps .iter() .any(|step| matches!(step, AutoMigrateStep::DisconnectAllUsers)) @@ -135,15 +139,28 @@ pub fn update_database( // TODO: consider using `ErrorStream` here. let old_module_def = plan.old_def(); + + // Build a map from full-name (namespaced) -> (owning_def, table_def) covering root and all + // submodule tables. Submodule tables are stored in the DB with prefixed names like + // "lib.library_procedure_timer", but `ModuleDef::table()` only has the current level. + // `all_tables_with_prefix()` returns the owning submodule alongside each def, which is also + // needed so that `check_compatible` resolves column type refs against the correct + // (sub)module typespace. + let old_tables_by_name: std::collections::HashMap = old_module_def + .all_tables_with_prefix() + .into_iter() + .map(|(prefix, owning_def, table_def)| (format!("{}{}", prefix, &table_def.name[..]), (owning_def, table_def))) + .collect(); + for table in existing_tables .iter() .filter(|table| table.table_type != StTableType::System && !table.is_view()) { - let old_def = old_module_def - .table(&table.table_name[..]) + let (owning_def, old_def) = old_tables_by_name + .get(table.table_name.as_ref()) .ok_or_else(|| anyhow::anyhow!("table {} not found in old_module_def", table.table_name))?; - table.check_compatible(old_module_def, old_def)?; + table.check_compatible(owning_def, old_def)?; } match plan { @@ -186,9 +203,12 @@ fn auto_migrate_database( for precheck in plan.prechecks { match precheck { spacetimedb_schema::auto_migrate::AutoMigratePrecheck::CheckAddSequenceRangeValid(sequence_name) => { - let table_def = plan.new.stored_in_table_def(sequence_name).unwrap(); - let sequence_def = &table_def.sequences[sequence_name]; - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + let (prefix, _owning_def, table_def, sequence_def) = + plan.new.find_sequence_by_full_name(&sequence_name).ok_or_else(|| { + anyhow::anyhow!("Precheck: sequence `{sequence_name}` not found in new module def") + })?; + let table_full_name = format!("{}{}", prefix, &*table_def.accessor_name); + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let ty = table_def .get_column(sequence_def.column) @@ -226,7 +246,7 @@ fn auto_migrate_database( for step in plan.steps { match step { spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveTable(table_name) => { - let table_id = stdb.table_id_from_name_mut(tx, table_name)?.unwrap(); + let table_id = stdb.table_id_from_name_mut(tx, &table_name)?.unwrap(); if stdb.table_row_count_mut(tx, table_id).unwrap_or(0) > 0 { anyhow::bail!( @@ -239,22 +259,22 @@ fn auto_migrate_database( stdb.drop_table(tx, table_id)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::AddTable(table_name) => { - let table_def: &TableDef = plan.new.expect_lookup(table_name); - - // Recursively sets IDs to 0. - // They will be initialized by the database when the table is created. - let table_schema = TableSchema::from_module_def(plan.new, table_def, (), TableId::SENTINEL); - + let (prefix, owning_def, table_def) = plan + .new + .find_table_by_full_name(&table_name) + .ok_or_else(|| anyhow::anyhow!("AddTable: table `{table_name}` not found in new module def"))?; log!(logger, "Creating table `{table_name}`"); - - stdb.create_table(tx, table_schema)?; + create_table_from_def_with_prefix(stdb, tx, owning_def, table_def, &prefix)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::AddView(view_name) => { - let view_def: &ViewDef = plan.new.expect_lookup(view_name); - stdb.create_view(tx, plan.new, view_def)?; + let (prefix, owning_def, view_def) = plan + .new + .find_view_by_full_name(&view_name) + .ok_or_else(|| anyhow::anyhow!("AddView: view `{view_name}` not found in new module def"))?; + create_table_from_view_def_with_prefix(stdb, tx, owning_def, view_def, &prefix)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveView(view_name) => { - let view_id = stdb.view_id_from_name_mut(tx, view_name)?.unwrap(); + let view_id = stdb.view_id_from_name_mut(tx, &view_name)?.unwrap(); stdb.drop_view(tx, view_id)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::UpdateView(_) => { @@ -265,9 +285,12 @@ fn auto_migrate_database( } } spacetimedb_schema::auto_migrate::AutoMigrateStep::AddIndex(index_name) => { - let table_def = plan.new.stored_in_table_def(index_name).unwrap(); - let index_def = table_def.indexes.get(index_name).unwrap(); - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + let (prefix, owning_def, table_def, index_def) = plan + .new + .find_index_by_full_name(&index_name) + .ok_or_else(|| anyhow::anyhow!("AddIndex: index `{index_name}` not found in new module def"))?; + let table_full_name = format!("{}{}", prefix, &*table_def.accessor_name); + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let index_cols = ColSet::from(index_def.algorithm.columns()); @@ -277,131 +300,148 @@ fn auto_migrate_database( .filter_map(|(_, c)| c.data.unique_columns()) .any(|unique_cols| unique_cols == &index_cols); - log!(logger, "Creating index `{}` on table `{}`", index_name, table_def.name); + log!(logger, "Creating index `{index_name}` on table `{table_full_name}`"); + + let mut index_schema = IndexSchema::from_module_def(owning_def, index_def, table_id, 0.into()); - let index_schema = IndexSchema::from_module_def(plan.new, index_def, table_id, 0.into()); + // Apply namespace prefix for submodule indexes + if !prefix.is_empty() { + index_schema.index_name = RawIdentifier::from(format!("{}{}", prefix, index_schema.index_name)); + if let Some(alias) = &index_schema.alias { + index_schema.alias = Some(RawIdentifier::from(format!("{}{}", prefix, alias))); + } + } stdb.create_index(tx, index_schema, is_unique)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveIndex(index_name) => { - let table_def = plan.old.stored_in_table_def(index_name).unwrap(); - - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + let (prefix, _owning_def, table_def, _index_def) = plan + .old + .find_index_by_full_name(&index_name) + .ok_or_else(|| anyhow::anyhow!("RemoveIndex: index `{index_name}` not found in old module def"))?; + let table_full_name = format!("{}{}", prefix, &*table_def.accessor_name); + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let table_schema = stdb.schema_for_table_mut(tx, table_id)?; let index_schema = table_schema .indexes .iter() - .find(|index| index.index_name[..] == index_name[..]) + .find(|index| index.index_name[..] == *index_name) .unwrap(); - log!(logger, "Dropping index `{}` on table `{}`", index_name, table_def.name); + log!(logger, "Dropping index `{index_name}` on table `{table_full_name}`"); stdb.drop_index(tx, index_schema.index_id)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveConstraint(constraint_name) => { - let table_def = plan.old.stored_in_table_def(constraint_name).unwrap(); - - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + let (prefix, _owning_def, table_def, _constraint_def) = + plan.old.find_constraint_by_full_name(&constraint_name).ok_or_else(|| { + anyhow::anyhow!("RemoveConstraint: constraint `{constraint_name}` not found in old module def") + })?; + let table_full_name = format!("{}{}", prefix, &*table_def.accessor_name); + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let table_schema = stdb.schema_for_table_mut(tx, table_id)?; + let constraint_schema = table_schema .constraints .iter() - .find(|constraint| constraint.constraint_name[..] == constraint_name[..]) + .find(|constraint| constraint.constraint_name[..] == *constraint_name) .unwrap(); log!( logger, - "Dropping constraint `{}` on table `{}`", - constraint_name, - table_def.name + "Dropping constraint `{constraint_name}` on table `{table_full_name}`" ); stdb.drop_constraint(tx, constraint_schema.constraint_id)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::AddConstraint(constraint_name) => { - let table_def = plan - .new - .stored_in_table_def(constraint_name) - .expect("AddConstraint references a table that should exist in the new module def"); - let constraint_def = &table_def.constraints[constraint_name]; + let (prefix, owning_def, table_def, constraint_def) = + plan.new.find_constraint_by_full_name(&constraint_name).ok_or_else(|| { + anyhow::anyhow!("AddConstraint: constraint `{constraint_name}` not found in new module def") + })?; + let table_full_name = format!("{}{}", prefix, &*table_def.accessor_name); let table_id = stdb - .table_id_from_name_mut(tx, &table_def.name)? + .table_id_from_name_mut(tx, &table_full_name)? .expect("table should exist in the database for AddConstraint"); - let constraint_schema = ConstraintSchema::from_module_def( - plan.new, - constraint_def, - table_id, - spacetimedb_primitives::ConstraintId::SENTINEL, - ); + let constraint_schema = + ConstraintSchema::from_module_def(owning_def, constraint_def, table_id, ConstraintId::SENTINEL); log!( logger, - "Adding constraint `{}` on table `{}`", - constraint_name, - table_def.name + "Adding constraint `{constraint_name}` on table `{table_full_name}`" ); stdb.create_constraint(tx, constraint_schema)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::AddSequence(sequence_name) => { - let table_def = plan.new.stored_in_table_def(sequence_name).unwrap(); - let sequence_def = table_def.sequences.get(sequence_name).unwrap(); - - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + let (prefix, owning_def, table_def, sequence_def) = + plan.new.find_sequence_by_full_name(&sequence_name).ok_or_else(|| { + anyhow::anyhow!("AddSequence: sequence `{sequence_name}` not found in new module def") + })?; + let table_full_name = format!("{}{}", prefix, &*table_def.accessor_name); + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let table_schema = stdb.schema_for_table_mut(tx, table_id)?; - log!( - logger, - "Adding sequence `{}` to table `{}`", - sequence_name, - table_def.name - ); + log!(logger, "Adding sequence `{sequence_name}` to table `{table_full_name}`"); let sequence_schema = - SequenceSchema::from_module_def(plan.new, sequence_def, table_schema.table_id, 0.into()); + SequenceSchema::from_module_def(owning_def, sequence_def, table_schema.table_id, 0.into()); stdb.create_sequence(tx, sequence_schema)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveSequence(sequence_name) => { - let table_def = plan.old.stored_in_table_def(sequence_name).unwrap(); - - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + let (prefix, _owning_def, table_def, _sequence_def) = + plan.old.find_sequence_by_full_name(&sequence_name).ok_or_else(|| { + anyhow::anyhow!("RemoveSequence: sequence `{sequence_name}` not found in old module def") + })?; + let table_full_name = format!("{}{}", prefix, &*table_def.accessor_name); + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let table_schema = stdb.schema_for_table_mut(tx, table_id)?; let sequence_schema = table_schema .sequences .iter() - .find(|sequence| sequence.sequence_name[..] == sequence_name[..]) + .find(|sequence| sequence.sequence_name[..] == *sequence_name) .unwrap(); log!( logger, - "Dropping sequence `{}` from table `{}`", - sequence_name, - table_def.name + "Dropping sequence `{sequence_name}` from table `{table_full_name}`" ); stdb.drop_sequence(tx, sequence_schema.sequence_id)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangeColumns(table_name) => { - let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); - let table_id = stdb.table_id_from_name_mut(tx, table_name).unwrap().unwrap(); - let column_schemas = column_schemas_from_defs(plan.new, &table_def.columns, table_id); + let (_prefix, owning_def, table_def) = + plan.new.find_table_by_full_name(&table_name).ok_or_else(|| { + anyhow::anyhow!("ChangeColumns: table `{table_name}` not found in new module def") + })?; + let table_id = stdb.table_id_from_name_mut(tx, &table_name).unwrap().unwrap(); + let column_schemas = column_schemas_from_defs(owning_def, &table_def.columns, table_id); - log!(logger, "Changing columns of table `{}`", table_name); + log!(logger, "Changing columns of table `{table_name}`"); stdb.alter_table_row_type(tx, table_id, column_schemas)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::ReschemaEventTable(table_name) => { - let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); - let table_id = stdb.table_id_from_name_mut(tx, table_name).unwrap().unwrap(); - let column_schemas = column_schemas_from_defs(plan.new, &table_def.columns, table_id); + let (_prefix, owning_def, table_def) = + plan.new.find_table_by_full_name(&table_name).ok_or_else(|| { + anyhow::anyhow!("ReschemaEventTable: table `{table_name}` not found in new module def") + })?; + let table_id = stdb.table_id_from_name_mut(tx, &table_name).unwrap().unwrap(); + let column_schemas = column_schemas_from_defs(owning_def, &table_def.columns, table_id); log!(logger, "Changing schema of event table `{}`", table_name); stdb.alter_event_table_row_type(tx, table_id, column_schemas)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangeAccess(table_name) => { - let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); - stdb.alter_table_access(tx, table_name, table_def.table_access.into())?; + let (_prefix, _owning_def, table_def) = plan + .new + .find_table_by_full_name(&table_name) + .ok_or_else(|| anyhow::anyhow!("ChangeAccess: table `{table_name}` not found in new module def"))?; + stdb.alter_table_access(tx, &table_name, table_def.table_access.into())?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangePrimaryKey(table_name) => { - let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); + let (_prefix, _owning_def, table_def) = + plan.new.find_table_by_full_name(&table_name).ok_or_else(|| { + anyhow::anyhow!("ChangePrimaryKey: table `{table_name}` not found in new module def") + })?; log!(logger, "Changing primary key for table `{table_name}`"); - stdb.alter_table_primary_key(tx, table_name, table_def.primary_key)?; + stdb.alter_table_primary_key(tx, &table_name, table_def.primary_key)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::AddSchedule(_) => { anyhow::bail!("Adding schedules is not yet implemented"); @@ -411,22 +451,28 @@ fn auto_migrate_database( } spacetimedb_schema::auto_migrate::AutoMigrateStep::AddRowLevelSecurity(sql_rls) => { log!(logger, "Adding row-level security `{sql_rls}`"); - let rls = plan.new.lookup_expect(sql_rls); + let rls = plan + .new + .row_level_security() + .find(|r| *r.sql == *sql_rls) + .ok_or_else(|| { + anyhow::anyhow!("AddRowLevelSecurity: RLS `{sql_rls}` not found in new module def") + })?; let rls = RowLevelExpr::build_row_level_expr(tx, &auth_ctx, rls)?; stdb.create_row_level_security(tx, rls.def)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveRowLevelSecurity(sql_rls) => { - log!(logger, "Removing-row level security `{sql_rls}`"); - stdb.drop_row_level_security(tx, sql_rls.clone())?; + log!(logger, "Removing row-level security `{sql_rls}`"); + stdb.drop_row_level_security(tx, sql_rls)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::AddColumns(table_name) => { - let table_def = plan + let (_prefix, owning_def, table_def) = plan .new - .stored_in_table_def(&table_name.clone().into()) - .expect("table must exist"); - let table_id = stdb.table_id_from_name_mut(tx, table_name).unwrap().unwrap(); - let column_schemas = column_schemas_from_defs(plan.new, &table_def.columns, table_id); + .find_table_by_full_name(&table_name) + .ok_or_else(|| anyhow::anyhow!("AddColumns: table `{table_name}` not found in new module def"))?; + let table_id = stdb.table_id_from_name_mut(tx, &table_name).unwrap().unwrap(); + let column_schemas = column_schemas_from_defs(owning_def, &table_def.columns, table_id); let default_values: Vec = table_def .columns @@ -455,9 +501,69 @@ pub fn create_table_from_def( module_def: &ModuleDef, table_def: &TableDef, ) -> anyhow::Result<()> { - let schema = TableSchema::from_module_def(module_def, table_def, (), TableId::SENTINEL); + create_table_from_def_with_prefix(stdb, tx, module_def, table_def, "") +} + +/// Creates a submodule table in `stdb`, applying the namespace to its canonical name. +/// `name_prefix` is the dot-terminated namespace string (e.g. `"alias."`). +pub fn create_table_from_def_with_prefix( + stdb: &RelationalDB, + tx: &mut MutTxId, + owning_def: &ModuleDef, + table_def: &TableDef, + name_prefix: &str, +) -> anyhow::Result<()> { + let mut schema = TableSchema::from_module_def(owning_def, table_def, (), TableId::SENTINEL); + if !name_prefix.is_empty() { + let prefixed_name = format!("{}{}", name_prefix, &*table_def.accessor_name); + schema.table_name = TableName::new_raw(RawIdentifier::from(prefixed_name)); + + // No alias needed + schema.alias = None; + + // Apply the namespace to the scheduled reducer/procedure name so the scheduler can + // resolve it via the namespaced reducer_by_name / procedure_by_name + if let Some(schedule) = &mut schema.schedule { + let prefixed_fn = format!("{}{}", name_prefix, &*schedule.function_name); + schedule.function_name = Identifier::new_assume_valid(RawIdentifier::from(prefixed_fn)); + } + + // Apply the namespace to index canonical names and aliases for global uniqueness. + for index in &mut schema.indexes { + index.index_name = RawIdentifier::from(format!("{}{}", name_prefix, index.index_name)); + if let Some(alias) = &index.alias { + index.alias = Some(RawIdentifier::from(format!("{}{}", name_prefix, alias))); + } + } + } stdb.create_table(tx, schema) - .with_context(|| format!("failed to create table {}", &table_def.name))?; + .with_context(|| format!("failed to create table {}{}", name_prefix, &*table_def.accessor_name))?; + Ok(()) +} + +/// Creates the table for `view_def` in `stdb`. +pub fn create_table_from_view_def( + stdb: &RelationalDB, + tx: &mut MutTxId, + module_def: &ModuleDef, + view_def: &ViewDef, +) -> anyhow::Result<()> { + stdb.create_view(tx, module_def, view_def) + .with_context(|| format!("failed to create table for view {}", &view_def.name))?; + Ok(()) +} + +/// Creates the table for a submodule `view_def` in `stdb`, applying the namespace prefix. +/// `name_prefix` is the dot-terminated namespace string (e.g. `"lib."`). +pub fn create_table_from_view_def_with_prefix( + stdb: &RelationalDB, + tx: &mut MutTxId, + owning_def: &ModuleDef, + view_def: &ViewDef, + name_prefix: &str, +) -> anyhow::Result<()> { + stdb.create_view_with_prefix(tx, owning_def, view_def, name_prefix) + .with_context(|| format!("failed to create table for view {}{}", name_prefix, &view_def.name))?; Ok(()) } @@ -663,86 +769,6 @@ mod test { .expect("should be a valid module definition") } - fn view_module() -> ModuleDef { - let mut builder = RawModuleDefV10Builder::new(); - let return_type_ref = builder.add_algebraic_type( - [], - "my_view_return_type", - AlgebraicType::product([("a", AlgebraicType::U64)]), - true, - ); - builder.add_view( - "my_view", - 0, - true, - true, - ProductType::unit(), - AlgebraicType::array(AlgebraicType::Ref(return_type_ref)), - ); - builder.add_view_primary_key("my_view", ["a"]); - builder - .finish() - .try_into() - .expect("should be a valid module definition") - } - - fn old_view_backing_schema(module_def: &ModuleDef) -> TableSchema { - let view = module_def.view("my_view").unwrap(); - let mut schema = TableSchema::from_view_def_for_datastore(module_def, view); - - schema.columns.remove(0); - for (pos, col) in schema.columns.iter_mut().enumerate() { - col.col_pos = pos.into(); - } - schema.indexes.clear(); - schema.constraints.clear(); - schema.reset(); - - schema - } - - fn create_backing_table(stdb: &TestDB, schema: TableSchema) -> anyhow::Result<()> { - let mut tx = begin_mut_tx(stdb); - stdb.create_table(&mut tx, schema)?; - stdb.commit_tx(tx)?; - Ok(()) - } - - #[test] - fn stale_view_backing_schema_generates_startup_repair_plan() -> anyhow::Result<()> { - let stdb = TestDB::durable()?; - let module_def = view_module(); - create_backing_table(&stdb, old_view_backing_schema(&module_def))?; - - let plan = stale_view_backing_table_recreate_plan(&stdb, &module_def)?.expect("expected repair plan"); - - let MigratePlan::Auto(plan) = &plan else { - panic!("expected auto migration"); - }; - let my_view = module_def.view("my_view").unwrap().key(); - assert!(plan.steps.contains(&AutoMigrateStep::RemoveView(my_view))); - assert!(plan.steps.contains(&AutoMigrateStep::AddView(my_view))); - assert!(plan.steps.contains(&AutoMigrateStep::DisconnectAllUsers)); - assert!(!plan.steps.contains(&AutoMigrateStep::UpdateView(my_view))); - - Ok(()) - } - - #[test] - fn current_view_backing_schema_skips_startup_repair_plan() -> anyhow::Result<()> { - let stdb = TestDB::durable()?; - let module_def = view_module(); - let view = module_def.view("my_view").unwrap(); - let backing_schema = TableSchema::from_view_def_for_datastore(&module_def, view); - create_backing_table(&stdb, backing_schema)?; - - let plan = stale_view_backing_table_recreate_plan(&stdb, &module_def)?; - - assert!(plan.is_none(), "{plan:#?}"); - - Ok(()) - } - enum TakeSnapshot { None, BeforeAutomigration, diff --git a/crates/lib/src/db/raw_def/v10.rs b/crates/lib/src/db/raw_def/v10.rs index 1ab04d7a6bd..b21d8470b84 100644 --- a/crates/lib/src/db/raw_def/v10.rs +++ b/crates/lib/src/db/raw_def/v10.rs @@ -98,6 +98,9 @@ pub enum RawModuleDefV10Section { /// Primary key metadata for views. ViewPrimaryKeys(Vec), + + /// Submodules, keyed by the namespace they are registered under. + Submodules(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -124,6 +127,14 @@ pub enum MethodOrAny { Method(crate::http::Method), } +#[derive(Debug, Clone, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawSubmoduleV10 { + pub namespace: String, + pub module: RawModuleDefV10, +} + #[derive(Debug, Clone, Copy, Default, SpacetimeType)] #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] #[sats(crate = crate)] @@ -558,6 +569,14 @@ pub struct RawViewPrimaryKeyDefV10 { } impl RawModuleDefV10 { + /// Get the submodules for this module definition. + pub fn submodules(&self) -> Option<&Vec> { + self.sections.iter().find_map(|s| match s { + RawModuleDefV10Section::Submodules(submodules) => Some(submodules), + _ => None, + }) + } + /// Get the types section, if present. pub fn types(&self) -> Option<&Vec> { self.sections.iter().find_map(|s| match s { diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/auto_migrate.rs index 63239de7ecc..32a3daa370d 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/auto_migrate.rs @@ -1,15 +1,17 @@ use core::{cmp::Ordering, ops::BitOr}; -use crate::{def::*, error::PrettyAlgebraicType, identifier::Identifier}; +use crate::{ + def::*, + error::PrettyAlgebraicType, + identifier::{Identifier, NamespacedIdentifier}, +}; use formatter::format_plan; +use spacetimedb_data_structures::map::HashMap; use spacetimedb_data_structures::{ error_stream::{CollectAllErrors, CombineErrors, ErrorStream}, map::{HashCollectionExt as _, HashSet}, }; -use spacetimedb_lib::{ - db::raw_def::v9::{RawRowLevelSecurityDefV9, TableType}, - hash_bytes, Identity, -}; +use spacetimedb_lib::{db::raw_def::v9::TableType, hash_bytes, Identity}; use spacetimedb_sats::{ layout::{HasLayout, SumTypeLayout}, raw_identifier::RawIdentifier, @@ -22,7 +24,7 @@ mod termcolor_formatter; pub type Result = std::result::Result>; -/// A plan for a migration. +// A plan for a migration. #[derive(Debug)] pub enum MigratePlan<'def> { Manual(ManualMigratePlan<'def>), @@ -208,10 +210,10 @@ pub struct AutoMigratePlan<'def> { pub new: &'def ModuleDef, /// The checks to perform before the automatic migration. /// There is also an implied check: that the schema in the database is compatible with the old ModuleDef. - pub prechecks: Vec>, + pub prechecks: Vec, /// The migration steps to perform. /// Order matters: `Remove`s of a particular `Def` must be ordered before `Add`s. - pub steps: Vec>, + pub steps: Vec, } impl AutoMigratePlan<'_> { @@ -235,99 +237,91 @@ impl AutoMigratePlan<'_> { /// Checks that must be performed before performing an automatic migration. /// These checks can access table contents and other database state. #[derive(PartialEq, Eq, Debug, PartialOrd, Ord)] -pub enum AutoMigratePrecheck<'def> { +pub enum AutoMigratePrecheck { /// Perform a check that adding a sequence is valid (the relevant column contains no values /// greater than the sequence's start value). - CheckAddSequenceRangeValid(::Key<'def>), + /// Payload is the full namespaced sequence name (e.g., `"lib.library_table_id_seq"`). + CheckAddSequenceRangeValid(NamespacedIdentifier), } /// A step in an automatic migration. +/// +/// All variant payloads are full namespaced names (e.g., `"lib.library_table"` for a submodule +/// table, or `"user"` for a root-level table). This allows submodule and root-level items to be +/// handled uniformly. Row-level security payloads are SQL text (`Box`) rather than +/// identifiers. +/// +/// IMPORTANT: Remove variants MUST be declared before Add variants in this enum. The ordering +/// is used to sort steps of an auto-migration so that removes precede adds for the same name. #[derive(PartialEq, Eq, Debug, PartialOrd, Ord)] -pub enum AutoMigrateStep<'def> { - // It is important FOR CORRECTNESS that `Remove` variants are declared before `Add` variants in this enum! - // - // The ordering is used to sort the steps of an auto-migration. - // If adds go before removes, and the user tries to remove an index and then re-add it with new configuration, - // the following can occur: - // - // 1. `AddIndex("indexname")` - // 2. `RemoveIndex("indexname")` - // - // This results in the existing index being re-added -- which, at time of writing, does nothing -- and then removed, - // resulting in the intended index not being created. - // - // For now, we just ensure that we declare all `Remove` variants before `Add` variants - // and let `#[derive(PartialOrd)]` take care of the rest. - // - // TODO: when this enum is made serializable, a more durable fix will be needed here. - // Probably we will want to have separate arrays of add and remove steps. - // - /// Remove an index. - RemoveIndex(::Key<'def>), - /// Remove a constraint. - RemoveConstraint(::Key<'def>), - /// Remove a sequence. - RemoveSequence(::Key<'def>), - /// Remove a schedule annotation from a table. - RemoveSchedule(::Key<'def>), - /// Remove a view and corresponding view table - RemoveView(::Key<'def>), - /// Remove a row-level security query. - RemoveRowLevelSecurity(::Key<'def>), +pub enum AutoMigrateStep { + /// Remove an index. Payload is the full namespaced index name. + RemoveIndex(NamespacedIdentifier), + /// Remove a constraint. Payload is the full namespaced constraint name. + RemoveConstraint(NamespacedIdentifier), + /// Remove a sequence. Payload is the full namespaced sequence name. + RemoveSequence(NamespacedIdentifier), + /// Remove a schedule annotation from a table. Payload is the full namespaced TABLE name. + RemoveSchedule(NamespacedIdentifier), + /// Remove a view and corresponding view table. Payload is the full namespaced view name. + RemoveView(NamespacedIdentifier), + /// Remove a row-level security query. Payload is the SQL text. + RemoveRowLevelSecurity(Box), + /// Remove an empty table and all its sub-objects (indexes, constraints, sequences). /// Validated at execution time: fails if the table contains data. - RemoveTable(::Key<'def>), + /// Payload is the full namespaced table name (e.g., `"lib.library_table"` or `"user"`). + RemoveTable(NamespacedIdentifier), /// Change the column types of a table, in a layout compatible way. - /// - /// This should be done before any new indices are added. - ChangeColumns(::Key<'def>), + /// Payload is the full namespaced table name. + ChangeColumns(NamespacedIdentifier), /// Change the column types of an event table, in a way that may not be layout-compatible. - ReschemaEventTable(::Key<'def>), + /// Payload is the full namespaced table name. + ReschemaEventTable(NamespacedIdentifier), /// Add columns to a table, in a layout-INCOMPATIBLE way. /// /// This is a destructive operation that requires first running a `DisconnectAllUsers`. - /// - /// The added columns are guaranteed to be contiguous - /// and at the end of the table. + /// The added columns are guaranteed to be contiguous and at the end of the table. /// They are also guaranteed to have default values set. - /// - /// When this step is present, - /// no `ChangeColumns` steps will be, for the same table. - AddColumns(::Key<'def>), + /// When this step is present, no `ChangeColumns` steps will be, for the same table. + /// Payload is the full namespaced table name. + AddColumns(NamespacedIdentifier), /// Add a table, including all indexes, constraints, and sequences. /// There will NOT be separate steps in the plan for adding indexes, constraints, and sequences. - AddTable(::Key<'def>), - /// Add an index. - AddIndex(::Key<'def>), + /// Payload is the full namespaced table name. + AddTable(NamespacedIdentifier), + /// Add an index. Payload is the full namespaced index name. + AddIndex(NamespacedIdentifier), /// Add a constraint to an existing table (with data validation precheck). - AddConstraint(::Key<'def>), - /// Add a sequence. - AddSequence(::Key<'def>), - /// Add a schedule annotation to a table. - AddSchedule(::Key<'def>), - /// Add a view and corresponding view table - AddView(::Key<'def>), - /// Add a row-level security query. - AddRowLevelSecurity(::Key<'def>), - - /// Change the access of a table. - ChangeAccess(::Key<'def>), + /// Payload is the full namespaced constraint name. + AddConstraint(NamespacedIdentifier), + /// Add a sequence. Payload is the full namespaced sequence name. + AddSequence(NamespacedIdentifier), + /// Add a schedule annotation to a table. Payload is the full namespaced TABLE name. + AddSchedule(NamespacedIdentifier), + /// Add a view and corresponding view table. Payload is the full namespaced view name. + AddView(NamespacedIdentifier), + /// Add a row-level security query. Payload is the SQL text. + AddRowLevelSecurity(Box), + + /// Change the access of a table or view. Payload is the full namespaced table/view name. + ChangeAccess(NamespacedIdentifier), /// Change the primary key of a table. /// - /// This updates the `table_primary_key` field in `st_table` - /// to match the new module definition. - /// Without this step, a stale primary key in the stored schema - /// causes `check_compatible` to fail on the next publish. - /// See: - ChangePrimaryKey(::Key<'def>), + /// This updates the `table_primary_key` field in `st_table` to match the new module definition. + /// Without this step, a stale primary key in the stored schema causes `check_compatible` to + /// fail on the next publish. See: + /// Payload is the full namespaced table name. + ChangePrimaryKey(NamespacedIdentifier), - /// Recompute a view, update its backing table, and push updates to clients - UpdateView(::Key<'def>), + /// Recompute a view, update its backing table, and push updates to clients. + /// Payload is the full namespaced view name. + UpdateView(NamespacedIdentifier), /// Disconnect all users connected to the module. DisconnectAllUsers, @@ -486,26 +480,26 @@ pub fn ponder_auto_migrate<'def>(old: &'def ModuleDef, new: &'def ModuleDef) -> let views_ok = auto_migrate_views(&mut plan); let tables_ok = auto_migrate_tables(&mut plan); - // Filter out sub-objects of added/removed tables — they're handled by `AddTable`/`RemoveTable`. - let (new_tables, removed_tables): (HashSet<&Identifier>, HashSet<&Identifier>) = - diff(plan.old, plan.new, ModuleDef::tables).fold( - (HashSet::new(), HashSet::new()), - |(mut added, mut removed), diff| { - match diff { - Diff::Add { new } => { - added.insert(&new.name); - } - Diff::Remove { old } => { - removed.insert(&old.name); - } - Diff::MaybeChange { .. } => {} - } - (added, removed) - }, - ); - let indexes_ok = auto_migrate_indexes(&mut plan, &new_tables, &removed_tables); - let sequences_ok = auto_migrate_sequences(&mut plan, &new_tables, &removed_tables); - let constraints_ok = auto_migrate_constraints(&mut plan, &new_tables, &removed_tables); + // Compute full-name sets for added/removed tables (across all submodules). + // Sub-objects of added/removed tables are handled by AddTable/RemoveTable, not individually. + let old_module = plan.old; + let new_module = plan.new; + let old_table_names: HashSet = old_module + .all_tables_with_prefix() + .into_iter() + .map(|(prefix, _, t)| format!("{}{}", prefix, &*t.accessor_name)) + .collect(); + let new_table_names: HashSet = new_module + .all_tables_with_prefix() + .into_iter() + .map(|(prefix, _, t)| format!("{}{}", prefix, &*t.accessor_name)) + .collect(); + let added_tables: HashSet = new_table_names.difference(&old_table_names).cloned().collect(); + let removed_tables: HashSet = old_table_names.difference(&new_table_names).cloned().collect(); + + let indexes_ok = auto_migrate_indexes(&mut plan, &added_tables, &removed_tables); + let sequences_ok = auto_migrate_sequences(&mut plan, &added_tables, &removed_tables); + let constraints_ok = auto_migrate_constraints(&mut plan, &added_tables, &removed_tables); // IMPORTANT: RLS auto-migrate steps must come last, // since they assume that any schema changes, like adding or dropping tables, // have already been reflected in the database state. @@ -520,170 +514,249 @@ pub fn ponder_auto_migrate<'def>(old: &'def ModuleDef, new: &'def ModuleDef) -> Ok(plan) } -/// A diff between two items. -/// `Add` means the item is present in the new `ModuleDef` but not the old. -/// `Remove` means the item is present in the old `ModuleDef` but not the new. -/// `MaybeChange` indicates the item is present in both. -#[derive(Debug)] -enum Diff<'def, T> { - Add { new: &'def T }, - Remove { old: &'def T }, - MaybeChange { old: &'def T, new: &'def T }, -} - -/// Diff a collection of items, looking them up in both the old and new `ModuleDef` by their `ModuleDefLookup::Key`. -/// Keys are required to be stable across migrations, which makes this possible. -fn diff<'def, T: ModuleDefLookup, I: Iterator>( - old: &'def ModuleDef, - new: &'def ModuleDef, - iter: impl Fn(&'def ModuleDef) -> I, -) -> impl Iterator> { - iter(old) - .map(move |old_item| match T::lookup(new, old_item.key()) { - Some(new_item) => Diff::MaybeChange { - old: old_item, - new: new_item, - }, - None => Diff::Remove { old: old_item }, - }) - .chain(iter(new).filter_map(move |new_item| { - if T::lookup(old, new_item.key()).is_none() { - Some(Diff::Add { new: new_item }) - } else { - None - } - })) -} - fn auto_migrate_views(plan: &mut AutoMigratePlan<'_>) -> Result<()> { - diff(plan.old, plan.new, ModuleDef::views) - .map(|table_diff| -> Result<()> { - match table_diff { - Diff::Add { new } => { - plan.steps.push(AutoMigrateStep::AddView(new.key())); - Ok(()) - } - // From the user's perspective, views do not have persistent state. - // Hence removal does not require a manual migration - just disconnecting clients. - Diff::Remove { old } => { - plan.steps.push(AutoMigrateStep::RemoveView(old.key())); - plan.ensure_disconnect_all_users(); - Ok(()) - } - Diff::MaybeChange { old, new } => auto_migrate_view(plan, old, new), + let old_module = plan.old; + let new_module = plan.new; + + // Build full-name maps for views across all submodules. + let old_views: HashMap = old_module + .all_views_with_prefix() + .into_iter() + .map(|(prefix, owning, view)| (format!("{}{}", prefix, &*view.name), (owning, view))) + .collect(); + let new_views: HashMap = new_module + .all_views_with_prefix() + .into_iter() + .map(|(prefix, owning, view)| (format!("{}{}", prefix, &*view.name), (owning, view))) + .collect(); + + type ViewEntry<'a> = (String, (&'a ModuleDef, &'a ViewDef), (&'a ModuleDef, &'a ViewDef)); + let mut maybe_change: Vec> = vec![]; + for (full_name, old_entry) in &old_views { + match new_views.get(full_name.as_str()) { + Some(new_entry) => maybe_change.push((full_name.clone(), *old_entry, *new_entry)), + None => { + plan.steps + .push(AutoMigrateStep::RemoveView(NamespacedIdentifier::new_assume_valid( + full_name.clone(), + ))); + plan.ensure_disconnect_all_users(); } + } + } + for (full_name, _) in &new_views { + if !old_views.contains_key(full_name.as_str()) { + plan.steps + .push(AutoMigrateStep::AddView(NamespacedIdentifier::new_assume_valid( + full_name.clone(), + ))); + } + } + + let results: Vec> = maybe_change + .into_iter() + .map(|(full_name, (old_owning, old_view), (new_owning, new_view))| { + auto_migrate_view( + plan, + NamespacedIdentifier::new_assume_valid(full_name), + old_owning, + old_view, + new_owning, + new_view, + ) }) - .collect_all_errors() + .collect(); + results.into_iter().collect_all_errors::>().map(|_| ()) } -fn auto_migrate_view<'def>(plan: &mut AutoMigratePlan<'def>, old: &'def ViewDef, new: &'def ViewDef) -> Result<()> { - let key = old.key(); - - if old.is_public != new.is_public { - plan.steps.push(AutoMigrateStep::ChangeAccess(key)); - } - +fn auto_migrate_view( + plan: &mut AutoMigratePlan, + full_name: NamespacedIdentifier, + old_owning: &ModuleDef, + old: &ViewDef, + new_owning: &ModuleDef, + new: &ViewDef, +) -> Result<()> { // We can always auto-migrate a view because we can always re-compute it. // However certain things require us to disconnect clients: // 1. If we add or remove a column or parameter // 2. If we change the order of the columns or parameters // 3. If we change the types of the columns or parameters // 4. If we change the context parameter - let Any(incompatible_return_type) = diff(plan.old, plan.new, |def| { - def.lookup_expect::(key).return_columns.iter() - }) - .map(|col_diff| { - match col_diff { - // We must disconnect clients if we add or remove a parameter or column - Diff::Add { .. } | Diff::Remove { .. } => Any(true), - Diff::MaybeChange { old, new } => { - if old.col_id != new.col_id { + let old_return_cols: HashMap<&Identifier, &ViewColumnDef> = + old.return_columns.iter().map(|c| (&c.name, c)).collect(); + let new_return_cols: HashMap<&Identifier, &ViewColumnDef> = + new.return_columns.iter().map(|c| (&c.name, c)).collect(); + + let Any(incompatible_return_type) = old + .return_columns + .iter() + .map(|old_col| match new_return_cols.get(&old_col.name) { + None => Any(true), + Some(new_col) => { + if old_col.col_id != new_col.col_id { return Any(true); - }; - + } ensure_old_ty_upgradable_to_new( false, - &|| old.view_name.clone(), - &|| old.name.clone(), - &WithTypespace::new(plan.old.typespace(), &old.ty) + &|| old_col.view_name.clone(), + &|| old_col.name.clone(), + &WithTypespace::new(old_owning.typespace(), &old_col.ty) .resolve_refs() .expect("valid ViewDefs must have valid type refs"), - &WithTypespace::new(plan.new.typespace(), &new.ty) + &WithTypespace::new(new_owning.typespace(), &new_col.ty) .resolve_refs() .expect("valid ViewDefs must have valid type refs"), ) .unwrap_or(Any(true)) } - } - }) - .collect(); - - let Any(incompatible_param_types) = diff(plan.old, plan.new, |def| { - def.lookup_expect::(key).param_columns.iter() - }) - .map(|col_diff| { - match col_diff { - // We must disconnect clients if we add or remove a parameter or column - Diff::Add { .. } | Diff::Remove { .. } => Any(true), - Diff::MaybeChange { old, new } => { - if old.col_id != new.col_id { + }) + .chain(new.return_columns.iter().map(|new_col| { + if old_return_cols.contains_key(&new_col.name) { + Any(false) + } else { + Any(true) // added column → incompatible + } + })) + .collect(); + + let old_param_cols: HashMap<&Identifier, &ViewParamDef> = old.param_columns.iter().map(|c| (&c.name, c)).collect(); + let new_param_cols: HashMap<&Identifier, &ViewParamDef> = new.param_columns.iter().map(|c| (&c.name, c)).collect(); + + let Any(incompatible_param_types) = old + .param_columns + .iter() + .map(|old_col| match new_param_cols.get(&old_col.name) { + None => Any(true), + Some(new_col) => { + if old_col.col_id != new_col.col_id { return Any(true); - }; - + } ensure_old_ty_upgradable_to_new( false, - &|| old.view_name.clone(), - &|| old.name.clone(), - &WithTypespace::new(plan.old.typespace(), &old.ty) + &|| old_col.view_name.clone(), + &|| old_col.name.clone(), + &WithTypespace::new(old_owning.typespace(), &old_col.ty) .resolve_refs() .expect("valid ViewDefs must have valid type refs"), - &WithTypespace::new(plan.new.typespace(), &new.ty) + &WithTypespace::new(new_owning.typespace(), &new_col.ty) .resolve_refs() .expect("valid ViewDefs must have valid type refs"), ) .unwrap_or(Any(true)) } - } - }) - .collect(); + }) + .chain(new.param_columns.iter().map(|new_col| { + if old_param_cols.contains_key(&new_col.name) { + Any(false) + } else { + Any(true) + } + })) + .collect(); if old.is_anonymous != new.is_anonymous + || old.is_public != new.is_public || old.primary_key != new.primary_key || incompatible_return_type || incompatible_param_types { - plan.steps.push(AutoMigrateStep::AddView(new.key())); - plan.steps.push(AutoMigrateStep::RemoveView(old.key())); - + plan.steps.push(AutoMigrateStep::AddView(full_name.clone())); + plan.steps.push(AutoMigrateStep::RemoveView(full_name)); plan.ensure_disconnect_all_users(); } else { - plan.steps.push(AutoMigrateStep::UpdateView(new.key())); + plan.steps.push(AutoMigrateStep::UpdateView(full_name)); } Ok(()) } fn auto_migrate_tables(plan: &mut AutoMigratePlan<'_>) -> Result<()> { - diff(plan.old, plan.new, ModuleDef::tables) - .map(|table_diff| -> Result<()> { - match table_diff { - Diff::Add { new } => { - plan.steps.push(AutoMigrateStep::AddTable(new.key())); - Ok(()) - } - Diff::Remove { old } => { - plan.steps.push(AutoMigrateStep::RemoveTable(old.key())); - plan.ensure_disconnect_all_users(); - Ok(()) - } - Diff::MaybeChange { old, new } => auto_migrate_table(plan, old, new), - } + let old_module = plan.old; + let new_module = plan.new; + + // Map canonical name (table_def.name, after case conversion) → (accessor full name, owning, table). + // Matching by canonical name means tables like `Events` (accessor) / `events` (canonical) in + // the old module and `events` (accessor + canonical) in the new module are treated as the same + // logical table, preventing spurious Remove+Add steps when only the accessor casing changed. + // Step payloads use accessor names (prefix + accessor_name) since that's what the DB stores. + let old_tables: HashMap = old_module + .all_tables_with_prefix() + .into_iter() + .map(|(prefix, owning, table)| { + let canonical = format!("{}{}", prefix, &*table.name); + let accessor = format!("{}{}", prefix, &*table.accessor_name); + (canonical, (accessor, owning, table)) + }) + .collect(); + let new_tables: HashMap = new_module + .all_tables_with_prefix() + .into_iter() + .map(|(prefix, owning, table)| { + let canonical = format!("{}{}", prefix, &*table.name); + let accessor = format!("{}{}", prefix, &*table.accessor_name); + (canonical, (accessor, owning, table)) + }) + .collect(); + + for (canonical, (accessor, _, _)) in &old_tables { + if !new_tables.contains_key(canonical.as_str()) { + plan.steps + .push(AutoMigrateStep::RemoveTable(NamespacedIdentifier::new_assume_valid( + accessor.clone(), + ))); + plan.ensure_disconnect_all_users(); + } + } + for (canonical, (accessor, _, _)) in &new_tables { + if !old_tables.contains_key(canonical.as_str()) { + plan.steps + .push(AutoMigrateStep::AddTable(NamespacedIdentifier::new_assume_valid( + accessor.clone(), + ))); + } + } + + type TableEntry<'a> = (String, (&'a ModuleDef, &'a TableDef), (&'a ModuleDef, &'a TableDef)); + let maybe_change: Vec> = old_tables + .iter() + .filter_map(|(canonical, (_, old_owning, old_table))| { + new_tables + .get(canonical.as_str()) + .map(|(new_accessor, new_owning, new_table)| { + ( + new_accessor.clone(), + (*old_owning, *old_table), + (*new_owning, *new_table), + ) + }) + }) + .collect(); + + let results: Vec> = maybe_change + .into_iter() + .map(|(full_name, (old_owning, old_table), (new_owning, new_table))| { + auto_migrate_table( + plan, + NamespacedIdentifier::new_assume_valid(full_name), + old_owning, + old_table, + new_owning, + new_table, + ) }) - .collect_all_errors() + .collect(); + results.into_iter().collect_all_errors::>().map(|_| ()) } -fn auto_migrate_table<'def>(plan: &mut AutoMigratePlan<'def>, old: &'def TableDef, new: &'def TableDef) -> Result<()> { - let key = old.key(); +fn auto_migrate_table( + plan: &mut AutoMigratePlan, + full_name: NamespacedIdentifier, + old_owning: &ModuleDef, + old: &TableDef, + new_owning: &ModuleDef, + new: &TableDef, +) -> Result<()> { let type_ok: Result<()> = if old.table_type == new.table_type { Ok(()) } else { @@ -707,120 +780,111 @@ fn auto_migrate_table<'def>(plan: &mut AutoMigratePlan<'def>, old: &'def TableDe let is_event = old.is_event; if old.table_access != new.table_access { - plan.steps.push(AutoMigrateStep::ChangeAccess(key)); + plan.steps.push(AutoMigrateStep::ChangeAccess(full_name.clone())); } if old.primary_key != new.primary_key { - plan.steps.push(AutoMigrateStep::ChangePrimaryKey(key)); + plan.steps.push(AutoMigrateStep::ChangePrimaryKey(full_name.clone())); } if old.schedule != new.schedule { - // Note: this handles the case where there's an altered ScheduleDef for some reason. - if let Some(old_schedule) = old.schedule.as_ref() { - plan.steps.push(AutoMigrateStep::RemoveSchedule(old_schedule.key())); + // Schedule steps are keyed by full TABLE name (schedules are 1:1 with tables). + if old.schedule.is_some() { + plan.steps.push(AutoMigrateStep::RemoveSchedule(full_name.clone())); } - if let Some(new_schedule) = new.schedule.as_ref() { - plan.steps.push(AutoMigrateStep::AddSchedule(new_schedule.key())); + if new.schedule.is_some() { + plan.steps.push(AutoMigrateStep::AddSchedule(full_name.clone())); } } - let columns_ok = diff(plan.old, plan.new, |def| { - def.lookup_expect::(key).columns.iter() - }) - .map(|col_diff| -> Result<_> { - match col_diff { - Diff::Add { new } => { - if is_event { - // Event tables never have any resident rows. - // As such, this is not a data migration; the table doesn't have any data in it to migrate. - // However, changing the schema of an event table will break clients. - - // `row_type_changed`, `columns_added`, `event_schema_changed` - Ok(ArrayMonoid([Any(false), Any(false), Any(true)])) - } else if new.default_value.is_some() { - // `row_type_changed`, `columns_added`, `event_schema_changed` - Ok(ArrayMonoid([Any(false), Any(true), Any(false)])) - } else { - Err(AutoMigrateError::AddColumn { - table: new.table_name.clone(), - column: new.name.clone(), - } - .into()) - } - } - Diff::Remove { old } => { - if is_event { - // Event tables never have any resident rows. - // As such, this is not a data migration; the table doesn't have any data in it to migrate. - // However, changing the schema of an event table will break clients. - - // `row_type_changed`, `columns_added`, `event_schema_changed` - Ok(ArrayMonoid([Any(false), Any(false), Any(true)])) - } else { - Err(AutoMigrateError::RemoveColumn { - table: old.table_name.clone(), - column: old.name.clone(), - } - .into()) - } - } - Diff::MaybeChange { old, new } => { - // Check column type upgradability. - let old_ty = WithTypespace::new(plan.old.typespace(), &old.ty) - .resolve_refs() - .expect("valid TableDef must have valid type refs"); - let new_ty = WithTypespace::new(plan.new.typespace(), &new.ty) - .resolve_refs() - .expect("valid TableDef must have valid type refs"); - let types_ok = ensure_old_ty_upgradable_to_new( - false, - &|| old.table_name.clone(), - &|| old.name.clone(), - &old_ty, - &new_ty, - ) - .or_else(|err| { + // Diff columns directly using the table defs (avoids root-only ModuleDefLookup). + let new_col_by_name: HashMap<&Identifier, &ColumnDef> = new.columns.iter().map(|c| (&c.name, c)).collect(); + let old_col_by_name: HashMap<&Identifier, &ColumnDef> = old.columns.iter().map(|c| (&c.name, c)).collect(); + + let columns_ok = old + .columns + .iter() + .map(|old_col| -> Result> { + match new_col_by_name.get(&old_col.name) { + None => { if is_event { - // If this is an event table, it's fine to layout-incompatibly non-upgradably change the layout, - // 'cause there can't be any rows to break. - Ok(Any(true)) + // Event tables never have any resident rows, so removing a column is not a + // data migration. However, changing the schema will break clients. + // `row_type_changed`, `columns_added`, `event_schema_changed` + Ok(ArrayMonoid([Any(false), Any(false), Any(true)])) } else { - Err(err) - } - }); - - // Note that the diff algorithm relies on `ModuleDefLookup` for `ColumnDef`, - // which looks up columns by NAME, NOT position: precisely to allow this step to work! - - // Note: We reject changes to positions. This means that, if a column was present in the old version of the table, - // it must be in the same place in the new version of the table. - // This guarantees that any added columns live at the end of the table. - let positions_ok = if old.col_id == new.col_id { - Ok(Any(false)) - } else if is_event { - Ok(Any(true)) - } else { - Err(AutoMigrateError::ReorderTable { - table: old.table_name.clone(), + Err(AutoMigrateError::RemoveColumn { + table: old_col.table_name.clone(), + column: old_col.name.clone(), + } + .into()) } - .into()) - }; - - (types_ok, positions_ok) - .combine_errors() - // `row_type_changed`, `column_added`, `event_schema_changed` - .map(|(types_changed, positions_changed)| { + } + Some(new_col) => { + let old_ty = WithTypespace::new(old_owning.typespace(), &old_col.ty) + .resolve_refs() + .expect("valid TableDef must have valid type refs"); + let new_ty = WithTypespace::new(new_owning.typespace(), &new_col.ty) + .resolve_refs() + .expect("valid TableDef must have valid type refs"); + let types_ok = ensure_old_ty_upgradable_to_new( + false, + &|| old_col.table_name.clone(), + &|| old_col.name.clone(), + &old_ty, + &new_ty, + ) + .or_else(|err| { if is_event { - // Event tables get a different auto-migrate step when their schema changes, - // as they don't have any rows to rewrite. So we track a different element in the array of change types. - ArrayMonoid([Any(false), Any(false), types_changed | positions_changed]) + // Event tables have no rows, so layout-incompatible type changes are fine. + Ok(Any(true)) } else { - assert!(!positions_changed.0); - ArrayMonoid([types_changed, Any(false), Any(false)]) + Err(err) } - }) + }); + // Reject reordering of existing columns (unless it's an event table). + let positions_ok = if old_col.col_id == new_col.col_id { + Ok(Any(false)) + } else if is_event { + Ok(Any(true)) + } else { + Err(AutoMigrateError::ReorderTable { + table: old_col.table_name.clone(), + } + .into()) + }; + (types_ok, positions_ok) + .combine_errors() + // `row_type_changed`, `columns_added`, `event_schema_changed` + .map(|(types_changed, positions_changed)| { + if is_event { + ArrayMonoid([Any(false), Any(false), types_changed | positions_changed]) + } else { + assert!(!positions_changed.0); + ArrayMonoid([types_changed, Any(false), Any(false)]) + } + }) + } } - } - }) - .collect_all_errors::>(); + }) + .chain(new.columns.iter().map(|new_col| -> Result> { + if old_col_by_name.contains_key(&new_col.name) { + Ok(ArrayMonoid([Any(false), Any(false), Any(false)])) + } else if is_event { + // Event tables never have any resident rows, so adding a column is not a data + // migration. However, changing the schema will break clients. + // `row_type_changed`, `columns_added`, `event_schema_changed` + Ok(ArrayMonoid([Any(false), Any(false), Any(true)])) + } else if new_col.default_value.is_some() { + // `row_type_changed`, `columns_added`, `event_schema_changed` + Ok(ArrayMonoid([Any(false), Any(true), Any(false)])) + } else { + Err(AutoMigrateError::AddColumn { + table: new_col.table_name.clone(), + column: new_col.name.clone(), + } + .into()) + } + })) + .collect_all_errors::>(); let ((), (), ArrayMonoid([Any(row_type_changed), Any(columns_added), Any(event_schema_changed)])) = (type_ok, event_ok, columns_ok).combine_errors()?; @@ -829,14 +893,14 @@ fn auto_migrate_table<'def>(plan: &mut AutoMigratePlan<'def>, old: &'def TableDe // If we're rewriting an event table, there's no data migration to do. // But incompatibly changing the schema can break clients. plan.ensure_disconnect_all_users(); - plan.steps.push(AutoMigrateStep::ReschemaEventTable(key)); + plan.steps.push(AutoMigrateStep::ReschemaEventTable(full_name)); } else if columns_added { // If we're adding a column, we'll rewrite the whole table. // That makes any `ChangeColumns` moot, so we can skip it. plan.ensure_disconnect_all_users(); - plan.steps.push(AutoMigrateStep::AddColumns(key)); + plan.steps.push(AutoMigrateStep::AddColumns(full_name)); } else if row_type_changed { - plan.steps.push(AutoMigrateStep::ChangeColumns(key)); + plan.steps.push(AutoMigrateStep::ChangeColumns(full_name)); } Ok(()) @@ -1031,139 +1095,257 @@ fn ensure_old_ty_upgradable_to_new( fn auto_migrate_indexes( plan: &mut AutoMigratePlan<'_>, - new_tables: &HashSet<&Identifier>, - removed_tables: &HashSet<&Identifier>, + new_tables: &HashSet, + removed_tables: &HashSet, ) -> Result<()> { - diff(plan.old, plan.new, ModuleDef::indexes) - .map(|index_diff| -> Result<()> { - match index_diff { - Diff::Add { new } => { - if !new_tables.contains(&plan.new.stored_in_table_def(&new.name).unwrap().name) { - plan.steps.push(AutoMigrateStep::AddIndex(new.key())); - } - Ok(()) - } - Diff::Remove { old } => { - if !removed_tables.contains(&plan.old.stored_in_table_def(&old.name).unwrap().name) { - plan.steps.push(AutoMigrateStep::RemoveIndex(old.key())); - } - Ok(()) + let old_module = plan.old; + let new_module = plan.new; + + // key = full index name (e.g. "lib.library_table_id_idx_btree") + // value = (full_table_name, &IndexDef) + let old_indexes: HashMap = { + let mut map = HashMap::new(); + for (prefix, _, table) in old_module.all_tables_with_prefix() { + let table_full = format!("{}{}", prefix, &*table.accessor_name); + for idx in table.indexes.values() { + map.insert(format!("{}{}", prefix, &*idx.name), (table_full.clone(), idx)); + } + } + map + }; + let new_indexes: HashMap = { + let mut map = HashMap::new(); + for (prefix, _, table) in new_module.all_tables_with_prefix() { + let table_full = format!("{}{}", prefix, &*table.accessor_name); + for idx in table.indexes.values() { + map.insert(format!("{}{}", prefix, &*idx.name), (table_full.clone(), idx)); + } + } + map + }; + + // Removed indexes: in old but not in new, and not part of a removed table. + for (full_idx_name, (table_full_name, _)) in &old_indexes { + if !new_indexes.contains_key(full_idx_name.as_str()) && !removed_tables.contains(table_full_name) { + plan.steps + .push(AutoMigrateStep::RemoveIndex(NamespacedIdentifier::new_assume_valid( + full_idx_name.clone(), + ))); + } + } + + // Added indexes: in new but not in old, and not part of a newly added table. + for (full_idx_name, (table_full_name, _)) in &new_indexes { + if !old_indexes.contains_key(full_idx_name.as_str()) && !new_tables.contains(table_full_name) { + plan.steps + .push(AutoMigrateStep::AddIndex(NamespacedIdentifier::new_assume_valid( + full_idx_name.clone(), + ))); + } + } + + // Changed indexes: same name in both. + let change_results: Vec> = old_indexes + .iter() + .filter_map(|(full_idx_name, (_, old_idx))| { + new_indexes + .get(full_idx_name.as_str()) + .map(|(_, new_idx)| (full_idx_name, old_idx, new_idx)) + }) + .map(|(full_idx_name, old_idx, new_idx)| { + if old_idx.accessor_name != new_idx.accessor_name { + Err(AutoMigrateError::ChangeIndexAccessor { + index: old_idx.name.clone(), + old_accessor: old_idx.accessor_name.clone(), + new_accessor: new_idx.accessor_name.clone(), } - Diff::MaybeChange { old, new } => { - if old.accessor_name != new.accessor_name { - Err(AutoMigrateError::ChangeIndexAccessor { - index: old.name.clone(), - old_accessor: old.accessor_name.clone(), - new_accessor: new.accessor_name.clone(), - } - .into()) - } else { - if old.algorithm != new.algorithm { - plan.steps.push(AutoMigrateStep::RemoveIndex(old.key())); - plan.steps.push(AutoMigrateStep::AddIndex(old.key())); - } - Ok(()) - } + .into()) + } else { + if old_idx.algorithm != new_idx.algorithm { + plan.steps + .push(AutoMigrateStep::RemoveIndex(NamespacedIdentifier::new_assume_valid( + full_idx_name.clone(), + ))); + plan.steps + .push(AutoMigrateStep::AddIndex(NamespacedIdentifier::new_assume_valid( + full_idx_name.clone(), + ))); } + Ok(()) } }) - .collect_all_errors() + .collect(); + change_results.into_iter().collect_all_errors::>().map(|_| ()) } fn auto_migrate_sequences( plan: &mut AutoMigratePlan, - new_tables: &HashSet<&Identifier>, - removed_tables: &HashSet<&Identifier>, + new_tables: &HashSet, + removed_tables: &HashSet, ) -> Result<()> { - diff(plan.old, plan.new, ModuleDef::sequences) - .map(|sequence_diff| -> Result<()> { - match sequence_diff { - Diff::Add { new } => { - if !new_tables.contains(&plan.new.stored_in_table_def(&new.name).unwrap().name) { - plan.prechecks - .push(AutoMigratePrecheck::CheckAddSequenceRangeValid(new.key())); - plan.steps.push(AutoMigrateStep::AddSequence(new.key())); - } - Ok(()) - } - Diff::Remove { old } => { - if !removed_tables.contains(&plan.old.stored_in_table_def(&old.name).unwrap().name) { - plan.steps.push(AutoMigrateStep::RemoveSequence(old.key())); - } - Ok(()) - } - Diff::MaybeChange { old, new } => { - // we do not need to check column ids, since in an automigrate, column ids are not changed. - if old != new { - plan.prechecks - .push(AutoMigratePrecheck::CheckAddSequenceRangeValid(new.key())); - plan.steps.push(AutoMigrateStep::RemoveSequence(old.key())); - plan.steps.push(AutoMigrateStep::AddSequence(new.key())); - } - Ok(()) - } + let old_module = plan.old; + let new_module = plan.new; + + // key = full sequence name (e.g. "lib.Bananas_id_seq") + // value = (full_table_name, &SequenceDef) + let old_seqs: HashMap = { + let mut map = HashMap::new(); + for (prefix, _, table) in old_module.all_tables_with_prefix() { + let table_full = format!("{}{}", prefix, &*table.accessor_name); + for seq in table.sequences.values() { + map.insert(format!("{}{}", prefix, &*seq.name), (table_full.clone(), seq)); } - }) - .collect_all_errors() + } + map + }; + let new_seqs: HashMap = { + let mut map = HashMap::new(); + for (prefix, _, table) in new_module.all_tables_with_prefix() { + let table_full = format!("{}{}", prefix, &*table.accessor_name); + for seq in table.sequences.values() { + map.insert(format!("{}{}", prefix, &*seq.name), (table_full.clone(), seq)); + } + } + map + }; + + // Removed sequences: in old but not in new, and not part of a removed table. + for (full_seq_name, (table_full_name, _)) in &old_seqs { + if !new_seqs.contains_key(full_seq_name.as_str()) && !removed_tables.contains(table_full_name) { + plan.steps + .push(AutoMigrateStep::RemoveSequence(NamespacedIdentifier::new_assume_valid( + full_seq_name.clone(), + ))); + } + } + + // Added or changed sequences. + for (full_seq_name, (table_full_name, new_seq)) in &new_seqs { + if let Some((_, old_seq)) = old_seqs.get(full_seq_name.as_str()) { + // we do not need to check column ids, since in an automigrate, column ids are not changed. + if *old_seq != *new_seq { + plan.prechecks.push(AutoMigratePrecheck::CheckAddSequenceRangeValid( + NamespacedIdentifier::new_assume_valid(full_seq_name.clone()), + )); + plan.steps + .push(AutoMigrateStep::RemoveSequence(NamespacedIdentifier::new_assume_valid( + full_seq_name.clone(), + ))); + plan.steps + .push(AutoMigrateStep::AddSequence(NamespacedIdentifier::new_assume_valid( + full_seq_name.clone(), + ))); + } + } else if !new_tables.contains(table_full_name) { + plan.prechecks.push(AutoMigratePrecheck::CheckAddSequenceRangeValid( + NamespacedIdentifier::new_assume_valid(full_seq_name.clone()), + )); + plan.steps + .push(AutoMigrateStep::AddSequence(NamespacedIdentifier::new_assume_valid( + full_seq_name.clone(), + ))); + } + } + + Ok(()) } fn auto_migrate_constraints( plan: &mut AutoMigratePlan, - new_tables: &HashSet<&Identifier>, - removed_tables: &HashSet<&Identifier>, + new_tables: &HashSet, + removed_tables: &HashSet, ) -> Result<()> { - diff(plan.old, plan.new, ModuleDef::constraints) - .map(|constraint_diff| -> Result<()> { - match constraint_diff { - Diff::Add { new } => { - if new_tables.contains(&plan.new.stored_in_table_def(&new.name).unwrap().name) { - // it's okay to add a constraint in a new table. - Ok(()) - } else { - // existing table — duplicate detection happens inside create_constraint - plan.steps.push(AutoMigrateStep::AddConstraint(new.key())); - Ok(()) - } - } - Diff::Remove { old } => { - if !removed_tables.contains(&plan.old.stored_in_table_def(&old.name).unwrap().name) { - plan.steps.push(AutoMigrateStep::RemoveConstraint(old.key())); - } - Ok(()) - } - Diff::MaybeChange { old, new } => { - if old == new { - Ok(()) - } else { - Err(AutoMigrateError::ChangeUniqueConstraint { - constraint: old.name.clone(), - } - .into()) - } - } + let old_module = plan.old; + let new_module = plan.new; + + // key = full constraint name (e.g. "lib.Apples_id_key") + // value = (full_table_name, &ConstraintDef) + let old_constraints: HashMap = { + let mut map = HashMap::new(); + for (prefix, _, table) in old_module.all_tables_with_prefix() { + let table_full = format!("{}{}", prefix, &*table.accessor_name); + for constraint in table.constraints.values() { + map.insert( + format!("{}{}", prefix, &*constraint.name), + (table_full.clone(), constraint), + ); } - }) - .collect_all_errors() + } + map + }; + let new_constraints: HashMap = { + let mut map = HashMap::new(); + for (prefix, _, table) in new_module.all_tables_with_prefix() { + let table_full = format!("{}{}", prefix, &*table.accessor_name); + for constraint in table.constraints.values() { + map.insert( + format!("{}{}", prefix, &*constraint.name), + (table_full.clone(), constraint), + ); + } + } + map + }; + + let mut results: Vec> = vec![]; + + // Added constraints. + for (full_constraint_name, (table_full_name, _new_constraint)) in &new_constraints { + if !old_constraints.contains_key(full_constraint_name.as_str()) && !new_tables.contains(table_full_name) { + // existing table — duplicate detection happens inside create_constraint + plan.steps + .push(AutoMigrateStep::AddConstraint(NamespacedIdentifier::new_assume_valid( + full_constraint_name.clone(), + ))); + // it's okay to add a constraint in a new table — AddTable covers it. + } + } + + // Removed constraints: not part of a removed table. + for (full_constraint_name, (table_full_name, _)) in &old_constraints { + if !new_constraints.contains_key(full_constraint_name.as_str()) && !removed_tables.contains(table_full_name) { + plan.steps.push(AutoMigrateStep::RemoveConstraint( + NamespacedIdentifier::new_assume_valid(full_constraint_name.clone()), + )); + } + } + + // Changed constraints. + for (full_constraint_name, (_, new_constraint)) in &new_constraints { + if let Some((_, old_constraint)) = old_constraints.get(full_constraint_name.as_str()) + && *old_constraint != *new_constraint + { + results.push(Err(AutoMigrateError::ChangeUniqueConstraint { + constraint: old_constraint.name.clone(), + } + .into())); + } + } + + results.into_iter().collect_all_errors::>().map(|_| ()) } // Because we can refer to many tables and fields on the row level-security query, we need to remove all of them, // then add the new ones, instead of trying to track the graph of dependencies. fn auto_migrate_row_level_security(plan: &mut AutoMigratePlan) -> Result<()> { - // Track if any RLS rules were changed. - let mut old_rls = HashSet::new(); - let mut new_rls = HashSet::new(); + let old_sqls: Vec> = plan.old.row_level_security().map(|rls| rls.sql.clone()).collect(); + let new_sqls: Vec> = plan.new.row_level_security().map(|rls| rls.sql.clone()).collect(); - for rls in plan.old.row_level_security() { - old_rls.insert(rls.key()); - plan.steps.push(AutoMigrateStep::RemoveRowLevelSecurity(rls.key())); + let changed = { + let old_set: HashSet<&str> = old_sqls.iter().map(|s| &**s).collect(); + let new_set: HashSet<&str> = new_sqls.iter().map(|s| &**s).collect(); + old_set != new_set + }; + + for sql in old_sqls { + plan.steps.push(AutoMigrateStep::RemoveRowLevelSecurity(sql)); } - for rls in plan.new.row_level_security() { - new_rls.insert(rls.key()); - plan.steps.push(AutoMigrateStep::AddRowLevelSecurity(rls.key())); + for sql in new_sqls { + plan.steps.push(AutoMigrateStep::AddRowLevelSecurity(sql)); } // We can force flush the cache by force disconnecting all clients if an RLS rule has been added, removed, or updated. - if old_rls != new_rls { + if changed { plan.ensure_disconnect_all_users(); } @@ -1179,7 +1361,7 @@ mod tests { AlgebraicType, AlgebraicValue, ProductType, ScheduleAt, }; use spacetimedb_primitives::ColId; - use v10::{ExplicitNames, RawModuleDefV10Builder}; + use v10::{ExplicitNames, RawModuleDefV10Builder, RawModuleDefV10Section, RawSubmoduleV10}; use v9::{RawModuleDefV9Builder, TableAccess}; use validate::tests::expect_identifier; @@ -1410,95 +1592,130 @@ mod tests { let new_def = updated_module_def(); let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); - let apples = expect_identifier("Apples"); - let bananas = expect_identifier("Bananas"); - let deliveries = expect_identifier("Deliveries"); - let oranges = expect_identifier("Oranges"); - let my_view = expect_identifier("my_view"); - - let bananas_sequence: RawIdentifier = "Bananas_id_seq".into(); - let apples_unique_constraint: RawIdentifier = "Apples_id_key".into(); - let apples_sequence: RawIdentifier = "Apples_id_seq".into(); - let apples_id_name_index: RawIdentifier = "Apples_id_name_idx_btree".into(); - let apples_id_count_index: RawIdentifier = "Apples_id_count_idx_btree".into(); - let deliveries_schedule = expect_identifier("Deliveries_sched"); - let inspections_schedule = expect_identifier("Inspections_sched"); - assert!(plan.prechecks.is_sorted()); assert_eq!(plan.prechecks.len(), 1); assert_eq!( plan.prechecks[0], - AutoMigratePrecheck::CheckAddSequenceRangeValid(&bananas_sequence) + AutoMigratePrecheck::CheckAddSequenceRangeValid(NamespacedIdentifier::new_assume_valid("Bananas_id_seq")) ); - let sql_old = RawRowLevelSecurityDefV9 { - sql: "SELECT * FROM Apples".into(), - }; - - let sql_new = RawRowLevelSecurityDefV9 { - sql: "SELECT * FROM Bananas".into(), - }; let steps = &plan.steps[..]; assert!(steps.is_sorted()); assert!( - steps.contains(&AutoMigrateStep::RemoveSequence(&apples_sequence)), + steps.contains(&AutoMigrateStep::RemoveSequence( + NamespacedIdentifier::new_assume_valid("Apples_id_seq") + )), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::RemoveConstraint(&apples_unique_constraint)), + steps.contains(&AutoMigrateStep::RemoveConstraint( + NamespacedIdentifier::new_assume_valid("Apples_id_key") + )), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::RemoveIndex(&apples_id_name_index)), + steps.contains(&AutoMigrateStep::RemoveIndex(NamespacedIdentifier::new_assume_valid( + "Apples_id_name_idx_btree" + ))), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::AddIndex(&apples_id_count_index)), + steps.contains(&AutoMigrateStep::AddIndex(NamespacedIdentifier::new_assume_valid( + "Apples_id_count_idx_btree" + ))), "{steps:?}" ); - assert!(steps.contains(&AutoMigrateStep::ChangeAccess(&bananas)), "{steps:?}"); assert!( - steps.contains(&AutoMigrateStep::AddSequence(&bananas_sequence)), + steps.contains(&AutoMigrateStep::ChangeAccess(NamespacedIdentifier::new_assume_valid( + "Bananas" + ))), + "{steps:?}" + ); + assert!( + steps.contains(&AutoMigrateStep::AddSequence(NamespacedIdentifier::new_assume_valid( + "Bananas_id_seq" + ))), "{steps:?}" ); - assert!(steps.contains(&AutoMigrateStep::AddTable(&oranges)), "{steps:?}"); + assert!( + steps.contains(&AutoMigrateStep::AddTable(NamespacedIdentifier::new_assume_valid( + "Oranges" + ))), + "{steps:?}" + ); + // Schedule steps are keyed by TABLE name (schedules are 1:1 with tables). assert!( - steps.contains(&AutoMigrateStep::RemoveSchedule(&deliveries_schedule)), + steps.contains(&AutoMigrateStep::RemoveSchedule( + NamespacedIdentifier::new_assume_valid("Deliveries") + )), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::AddSchedule(&inspections_schedule)), + steps.contains(&AutoMigrateStep::AddSchedule(NamespacedIdentifier::new_assume_valid( + "Inspections" + ))), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::RemoveRowLevelSecurity(&sql_old.sql)), + steps.contains(&AutoMigrateStep::RemoveRowLevelSecurity(Box::from( + "SELECT * FROM Apples" + ))), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::AddRowLevelSecurity(&sql_new.sql)), + steps.contains(&AutoMigrateStep::AddRowLevelSecurity(Box::from( + "SELECT * FROM Bananas" + ))), "{steps:?}" ); - assert!(steps.contains(&AutoMigrateStep::ChangeColumns(&apples)), "{steps:?}"); assert!( - steps.contains(&AutoMigrateStep::ChangeColumns(&deliveries)), + steps.contains(&AutoMigrateStep::ChangeColumns(NamespacedIdentifier::new_assume_valid( + "Apples" + ))), + "{steps:?}" + ); + assert!( + steps.contains(&AutoMigrateStep::ChangeColumns(NamespacedIdentifier::new_assume_valid( + "Deliveries" + ))), "{steps:?}" ); assert!(steps.contains(&AutoMigrateStep::DisconnectAllUsers), "{steps:?}"); - assert!(steps.contains(&AutoMigrateStep::AddColumns(&bananas)), "{steps:?}"); + assert!( + steps.contains(&AutoMigrateStep::AddColumns(NamespacedIdentifier::new_assume_valid( + "Bananas" + ))), + "{steps:?}" + ); // Column is changed but it will not reflect in steps due to `AutoMigrateStep::AddColumns` - assert!(!steps.contains(&AutoMigrateStep::ChangeColumns(&bananas)), "{steps:?}"); + assert!( + !steps.contains(&AutoMigrateStep::ChangeColumns(NamespacedIdentifier::new_assume_valid( + "Bananas" + ))), + "{steps:?}" + ); - assert!(steps.contains(&AutoMigrateStep::RemoveView(&my_view)), "{steps:?}"); - assert!(steps.contains(&AutoMigrateStep::AddView(&my_view)), "{steps:?}"); + assert!( + steps.contains(&AutoMigrateStep::RemoveView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), + "{steps:?}" + ); + assert!( + steps.contains(&AutoMigrateStep::AddView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), + "{steps:?}" + ); } #[test] @@ -1827,9 +2044,6 @@ mod tests { && type1.0 == prod1_ty && type2.0 == new_prod1_ty ); - // Note: `AddUniqueConstraint` is no longer an error — adding unique constraints - // to existing tables is now allowed; duplicate detection happens inside create_constraint. - expect_error_matching!( result, AutoMigrateError::ChangeTableType { table, type1, type2 } => table == &apples && type1 == &TableType::User && type2 == &TableType::System @@ -1920,14 +2134,22 @@ mod tests { ); }); - let my_view = expect_identifier("my_view"); - let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); let steps = &plan.steps[..]; assert!(!plan.disconnects_all_users(), "{plan:#?}"); - assert!(steps.contains(&AutoMigrateStep::AddView(&my_view)), "{steps:?}"); - assert!(!steps.contains(&AutoMigrateStep::RemoveView(&my_view)), "{steps:?}"); + assert!( + steps.contains(&AutoMigrateStep::AddView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), + "{steps:?}" + ); + assert!( + !steps.contains(&AutoMigrateStep::RemoveView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), + "{steps:?}" + ); } #[test] @@ -1950,14 +2172,22 @@ mod tests { }); let new_def = create_module_def(|_| {}); - let my_view = expect_identifier("my_view"); - let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); let steps = &plan.steps[..]; assert!(plan.disconnects_all_users(), "{plan:#?}"); - assert!(steps.contains(&AutoMigrateStep::RemoveView(&my_view)), "{steps:?}"); - assert!(!steps.contains(&AutoMigrateStep::AddView(&my_view)), "{steps:?}"); + assert!( + steps.contains(&AutoMigrateStep::RemoveView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), + "{steps:?}" + ); + assert!( + !steps.contains(&AutoMigrateStep::AddView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), + "{steps:?}" + ); } #[test] @@ -2044,23 +2274,27 @@ mod tests { }), }, ] { - let my_view = expect_identifier("my_view"); - let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); let steps = &plan.steps[..]; assert!(!plan.disconnects_all_users(), "{name}, plan: {plan:#?}"); assert!( - steps.contains(&AutoMigrateStep::UpdateView(&my_view)), + steps.contains(&AutoMigrateStep::UpdateView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), "{name}, steps: {steps:?}" ); assert!( - !steps.contains(&AutoMigrateStep::AddView(&my_view)), + !steps.contains(&AutoMigrateStep::AddView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), "{name}, steps: {steps:?}" ); assert!( - !steps.contains(&AutoMigrateStep::RemoveView(&my_view)), + !steps.contains(&AutoMigrateStep::RemoveView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), "{name}, steps: {steps:?}" ); } @@ -2093,22 +2327,26 @@ mod tests { let old_def = module_def(); let new_def = module_def(); - let level_2_person = expect_identifier("Level2Person"); - let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); let steps = &plan.steps[..]; assert!(!plan.disconnects_all_users(), "{plan:#?}"); assert!( - steps.contains(&AutoMigrateStep::UpdateView(&level_2_person)), + steps.contains(&AutoMigrateStep::UpdateView(NamespacedIdentifier::new_assume_valid( + "Level2Person" + ))), "steps: {steps:?}" ); assert!( - !steps.contains(&AutoMigrateStep::AddView(&level_2_person)), + !steps.contains(&AutoMigrateStep::AddView(NamespacedIdentifier::new_assume_valid( + "Level2Person" + ))), "steps: {steps:?}" ); assert!( - !steps.contains(&AutoMigrateStep::RemoveView(&level_2_person)), + !steps.contains(&AutoMigrateStep::RemoveView(NamespacedIdentifier::new_assume_valid( + "Level2Person" + ))), "steps: {steps:?}" ); } @@ -2442,23 +2680,27 @@ mod tests { }), }, ] { - let my_view = expect_identifier("my_view"); - let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); let steps = &plan.steps[..]; assert!(plan.disconnects_all_users(), "{name}, plan: {plan:?}"); assert!( - steps.contains(&AutoMigrateStep::AddView(&my_view)), + steps.contains(&AutoMigrateStep::AddView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), "{name}, steps: {steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::RemoveView(&my_view)), + steps.contains(&AutoMigrateStep::RemoveView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), "{name}, steps: {steps:?}" ); assert!( - !steps.contains(&AutoMigrateStep::UpdateView(&my_view)), + !steps.contains(&AutoMigrateStep::UpdateView(NamespacedIdentifier::new_assume_valid( + "my_view" + ))), "{name}, steps: {steps:?}" ); } @@ -2588,12 +2830,11 @@ mod tests { .finish(); }); - let drop_table = expect_identifier("Drop"); let plan = ponder_auto_migrate(&old, &new).expect("removing a table should produce a valid plan"); assert_eq!( plan.steps, &[ - AutoMigrateStep::RemoveTable(&drop_table), + AutoMigrateStep::RemoveTable(NamespacedIdentifier::new_assume_valid("Drop")), AutoMigrateStep::DisconnectAllUsers, ], ); @@ -2611,15 +2852,430 @@ mod tests { }); let new = create_module_def(|_builder| {}); - let drop_table = expect_identifier("Drop"); let plan = ponder_auto_migrate(&old, &new).expect("removing a table should produce a valid plan"); assert_eq!( plan.steps, &[ - AutoMigrateStep::RemoveTable(&drop_table), + AutoMigrateStep::RemoveTable(NamespacedIdentifier::new_assume_valid("Drop")), AutoMigrateStep::DisconnectAllUsers, ], "plan should only contain RemoveTable + DisconnectAllUsers, no orphan sub-object steps" ); } + + fn make_submodule(namespace: &str, build: impl Fn(&mut RawModuleDefV10Builder)) -> RawSubmoduleV10 { + let mut builder = RawModuleDefV10Builder::new(); + build(&mut builder); + RawSubmoduleV10 { + namespace: namespace.to_string(), + module: builder.finish(), + } + } + + fn create_module_def_with_submodules( + build_root: impl Fn(&mut RawModuleDefV10Builder), + submodules: Vec, + ) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + build_root(&mut builder); + let mut raw = builder.finish(); + if !submodules.is_empty() { + raw.sections.push(RawModuleDefV10Section::Submodules(submodules)); + } + raw.try_into().expect("should be a valid module definition") + } + + #[test] + fn submodule_table_unchanged() { + let submodule = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + }) + }; + let old = create_module_def_with_submodules(|_| {}, vec![submodule()]); + let new = create_module_def_with_submodules(|_| {}, vec![submodule()]); + + let plan = ponder_auto_migrate(&old, &new).expect("no-op migration should succeed"); + let namespaced: Vec<_> = plan + .steps + .iter() + .filter(|s| format!("{s:?}").contains("lib.sessions")) + .collect(); + assert!( + namespaced.is_empty(), + "unchanged submodule should produce no steps for lib.sessions: {plan:#?}" + ); + } + + #[test] + fn submodule_add_table() { + let old = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + let new = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + b.build_table_with_new_type("tokens", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a submodule table should succeed"); + let steps = &plan.steps[..]; + + assert!(!plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::AddTable(NamespacedIdentifier::new_assume_valid( + "lib.tokens" + ))), + "{steps:?}" + ); + } + + #[test] + fn submodule_remove_table() { + let old = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + b.build_table_with_new_type("tokens", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + let new = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + + let plan = ponder_auto_migrate(&old, &new).expect("removing a submodule table should succeed"); + let steps = &plan.steps[..]; + + assert!(plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::RemoveTable(NamespacedIdentifier::new_assume_valid( + "lib.tokens" + ))), + "{steps:?}" + ); + } + + #[test] + fn submodule_add_index() { + let sessions_without_index = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + }) + }; + let sessions_with_index = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .with_index(btree(0), "sessions_id_idx", "sessions_id_idx") + .finish(); + }) + }; + + let old = create_module_def_with_submodules(|_| {}, vec![sessions_without_index()]); + let new = create_module_def_with_submodules(|_| {}, vec![sessions_with_index()]); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a submodule index should succeed"); + let steps = &plan.steps[..]; + + assert!( + steps.contains(&AutoMigrateStep::AddIndex(NamespacedIdentifier::new_assume_valid( + "lib.sessions_id_idx_btree" + ))), + "{steps:?}" + ); + } + + #[test] + fn submodule_remove_index() { + let sessions_without_index = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + }) + }; + let sessions_with_index = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .with_index(btree(0), "sessions_id_idx", "sessions_id_idx") + .finish(); + }) + }; + + let old = create_module_def_with_submodules(|_| {}, vec![sessions_with_index()]); + let new = create_module_def_with_submodules(|_| {}, vec![sessions_without_index()]); + + let plan = ponder_auto_migrate(&old, &new).expect("removing a submodule index should succeed"); + let steps = &plan.steps[..]; + + assert!( + steps.contains(&AutoMigrateStep::RemoveIndex(NamespacedIdentifier::new_assume_valid( + "lib.sessions_id_idx_btree" + ))), + "{steps:?}" + ); + } + + #[test] + fn submodule_add_sequence() { + let without_seq = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + }) + }; + let with_seq = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .with_column_sequence(0) + .finish(); + }) + }; + + let old = create_module_def_with_submodules(|_| {}, vec![without_seq()]); + let new = create_module_def_with_submodules(|_| {}, vec![with_seq()]); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a submodule sequence should succeed"); + let steps = &plan.steps[..]; + + assert!( + steps.iter().any(|s| matches!( + s, + AutoMigrateStep::AddSequence(n) if n.starts_with("lib.sessions") + )), + "expected AddSequence for lib.sessions_*: {steps:?}" + ); + assert!( + plan.prechecks.iter().any(|p| matches!( + p, + AutoMigratePrecheck::CheckAddSequenceRangeValid(n) if n.starts_with("lib.sessions") + )), + "expected CheckAddSequenceRangeValid precheck: {:?}", + plan.prechecks + ); + } + + #[test] + fn submodule_add_view() { + let without_view = || make_submodule("lib", |_| {}); + let with_view = || { + make_submodule("lib", |b| { + let ret_ref = b.add_algebraic_type( + [], + "lib_view_return", + AlgebraicType::product([("a", AlgebraicType::U64)]), + true, + ); + b.add_view( + "lib_view", + 0, + true, + true, + ProductType::from([("x", AlgebraicType::U32)]), + AlgebraicType::array(AlgebraicType::Ref(ret_ref)), + ); + }) + }; + + let old = create_module_def_with_submodules(|_| {}, vec![without_view()]); + let new = create_module_def_with_submodules(|_| {}, vec![with_view()]); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a submodule view should succeed"); + let steps = &plan.steps[..]; + + assert!(!plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::AddView(NamespacedIdentifier::new_assume_valid( + "lib.lib_view" + ))), + "{steps:?}" + ); + } + + #[test] + fn submodule_remove_view() { + let without_view = || make_submodule("lib", |_| {}); + let with_view = || { + make_submodule("lib", |b| { + let ret_ref = b.add_algebraic_type( + [], + "lib_view_return", + AlgebraicType::product([("a", AlgebraicType::U64)]), + true, + ); + b.add_view( + "lib_view", + 0, + true, + true, + ProductType::from([("x", AlgebraicType::U32)]), + AlgebraicType::array(AlgebraicType::Ref(ret_ref)), + ); + }) + }; + + let old = create_module_def_with_submodules(|_| {}, vec![with_view()]); + let new = create_module_def_with_submodules(|_| {}, vec![without_view()]); + + let plan = ponder_auto_migrate(&old, &new).expect("removing a submodule view should succeed"); + let steps = &plan.steps[..]; + + assert!(plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::RemoveView(NamespacedIdentifier::new_assume_valid( + "lib.lib_view" + ))), + "{steps:?}" + ); + } + + #[test] + fn add_whole_submodule() { + let old = create_module_def_with_submodules(|_| {}, vec![]); + let new = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + b.build_table_with_new_type("tokens", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a whole submodule should succeed"); + let steps = &plan.steps[..]; + + assert!(!plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::AddTable(NamespacedIdentifier::new_assume_valid( + "lib.sessions" + ))), + "{steps:?}" + ); + assert!( + steps.contains(&AutoMigrateStep::AddTable(NamespacedIdentifier::new_assume_valid( + "lib.tokens" + ))), + "{steps:?}" + ); + } + + #[test] + fn remove_whole_submodule() { + let old = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + b.build_table_with_new_type("tokens", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + let new = create_module_def_with_submodules(|_| {}, vec![]); + + let plan = ponder_auto_migrate(&old, &new).expect("removing a whole submodule should succeed"); + let steps = &plan.steps[..]; + + assert!(plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::RemoveTable(NamespacedIdentifier::new_assume_valid( + "lib.sessions" + ))), + "{steps:?}" + ); + assert!( + steps.contains(&AutoMigrateStep::RemoveTable(NamespacedIdentifier::new_assume_valid( + "lib.tokens" + ))), + "{steps:?}" + ); + } + + #[test] + fn nested_submodule_add_table() { + let make_nested_def = |add_baz_items: bool| { + let baz_submodule = make_submodule("baz", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + if add_baz_items { + b.build_table_with_new_type("baz_items", ProductType::from([("id", AlgebraicType::U32)]), true) + .finish(); + } + }); + + let mut auth_builder = RawModuleDefV10Builder::new(); + auth_builder + .build_table_with_new_type("auth_users", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + let mut auth_raw = auth_builder.finish(); + auth_raw + .sections + .push(RawModuleDefV10Section::Submodules(vec![baz_submodule])); + + let auth_submodule = RawSubmoduleV10 { + namespace: "auth".to_string(), + module: auth_raw, + }; + + let root_builder = RawModuleDefV10Builder::new(); + let mut root_raw = root_builder.finish(); + root_raw + .sections + .push(RawModuleDefV10Section::Submodules(vec![auth_submodule])); + root_raw.try_into().expect("should be a valid module definition") + }; + + let old: ModuleDef = make_nested_def(false); + let new: ModuleDef = make_nested_def(true); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a deeply nested table should succeed"); + let steps = &plan.steps[..]; + + assert!(!plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::AddTable(NamespacedIdentifier::new_assume_valid( + "auth.baz.baz_items" + ))), + "{steps:?}" + ); + } + + #[test] + fn submodule_remove_table_no_orphan_sub_objects() { + let old = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .with_primary_key(0) + .with_unique_constraint(0) + .with_index(btree(0), "sessions_id_idx", "sessions_id_idx") + .finish(); + })], + ); + let new = create_module_def_with_submodules(|_| {}, vec![make_submodule("lib", |_| {})]); + + let plan = ponder_auto_migrate(&old, &new).expect("removing a submodule table with sub-objects should succeed"); + assert_eq!( + plan.steps, + &[ + AutoMigrateStep::RemoveTable(NamespacedIdentifier::new_assume_valid("lib.sessions")), + AutoMigrateStep::DisconnectAllUsers, + ], + "should only contain RemoveTable + DisconnectAllUsers, no orphan sub-object steps" + ); + } } diff --git a/crates/schema/src/auto_migrate/formatter.rs b/crates/schema/src/auto_migrate/formatter.rs index cd6c1042077..b17175f8ea7 100644 --- a/crates/schema/src/auto_migrate/formatter.rs +++ b/crates/schema/src/auto_migrate/formatter.rs @@ -2,14 +2,14 @@ use std::io; -use super::{AutoMigratePlan, ModuleDefLookup, TableDef}; +use super::AutoMigratePlan; use crate::{ auto_migrate::AutoMigrateStep, - def::{ConstraintData, FunctionKind, ModuleDef, ScheduleDef, ViewDef}, - identifier::Identifier, + def::{ConstraintData, FunctionKind, ModuleDef}, + identifier::{Identifier, NamespacedIdentifier}, }; use itertools::Itertools; -use spacetimedb_lib::db::raw_def::v9::{RawRowLevelSecurityDefV9, TableAccess, TableType}; +use spacetimedb_lib::db::raw_def::v9::{TableAccess, TableType}; use spacetimedb_primitives::ColId; use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::{AlgebraicType, AlgebraicValue, WithTypespace}; @@ -32,87 +32,94 @@ fn format_step( ) -> Result<(), FormattingErrors> { match step { AutoMigrateStep::AddView(view) => { - let view_info = extract_view_info(*view, plan.new)?; + let view_info = extract_view_info(view, plan.new)?; f.format_view(&view_info, Action::Created) } AutoMigrateStep::RemoveView(view) => { - let view_info = extract_view_info(*view, plan.old)?; + let view_info = extract_view_info(view, plan.old)?; f.format_view(&view_info, Action::Removed) } // This means the body of the view may have been updated. // So we must recompute it and send any updates to clients. // No need to include this step in the formatted plan. AutoMigrateStep::UpdateView(_) => Ok(()), - AutoMigrateStep::RemoveTable(t) => { - let table_def: &TableDef = plan.old.expect_lookup(*t); - f.format_remove_table(&table_def.name) - } + AutoMigrateStep::RemoveTable(t) => f.format_remove_table(t), AutoMigrateStep::AddTable(t) => { - let table_info = extract_table_info(*t, plan)?; + let table_info = extract_table_info(t, plan)?; f.format_add_table(&table_info) } AutoMigrateStep::AddIndex(index) => { - let index_info = extract_index_info(*index, plan.new)?; + let index_info = extract_index_info(index, plan.new)?; f.format_index(&index_info, Action::Created) } AutoMigrateStep::RemoveIndex(index) => { - let index_info = extract_index_info(*index, plan.old)?; + let index_info = extract_index_info(index, plan.old)?; f.format_index(&index_info, Action::Removed) } AutoMigrateStep::RemoveConstraint(constraint) => { - let constraint_info = extract_constraint_info(*constraint, plan.old)?; + let constraint_info = extract_constraint_info(constraint, plan.old)?; f.format_constraint(&constraint_info, Action::Removed) } AutoMigrateStep::AddConstraint(constraint) => { - let constraint_info = extract_constraint_info(*constraint, plan.new)?; + let constraint_info = extract_constraint_info(constraint, plan.new)?; f.format_constraint(&constraint_info, Action::Created) } AutoMigrateStep::AddSequence(sequence) => { - let sequence_info = extract_sequence_info(*sequence, plan.new)?; + let sequence_info = extract_sequence_info(sequence, plan.new)?; f.format_sequence(&sequence_info, Action::Created) } AutoMigrateStep::RemoveSequence(sequence) => { - let sequence_info = extract_sequence_info(*sequence, plan.old)?; + let sequence_info = extract_sequence_info(sequence, plan.old)?; f.format_sequence(&sequence_info, Action::Removed) } AutoMigrateStep::ChangeAccess(table) => { - let access_info = extract_access_change_info(*table, plan)?; + let access_info = extract_access_change_info(table, plan)?; f.format_change_access(&access_info) } AutoMigrateStep::ChangePrimaryKey(table) => { - let old_table = plan.old.lookup_expect::(*table); - let new_table = plan.new.lookup_expect::(*table); + let (_, _, old_table) = + plan.old + .find_table_by_full_name(table) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: (&**table).into(), + })?; + let (_, _, new_table) = + plan.new + .find_table_by_full_name(table) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: (&**table).into(), + })?; f.format_change_primary_key(table, old_table.primary_key, new_table.primary_key) } AutoMigrateStep::AddSchedule(schedule) => { - let schedule_info = extract_schedule_info(*schedule, plan.new)?; + let schedule_info = extract_schedule_info(schedule, plan.new)?; f.format_schedule(&schedule_info, Action::Created) } AutoMigrateStep::RemoveSchedule(schedule) => { - let schedule_info = extract_schedule_info(*schedule, plan.old)?; + let schedule_info = extract_schedule_info(schedule, plan.old)?; f.format_schedule(&schedule_info, Action::Removed) } AutoMigrateStep::AddRowLevelSecurity(rls) => { - if let Some(rls_info) = extract_rls_info(*rls, plan)? { + if let Some(rls_info) = extract_rls_info(rls.as_ref(), plan)? { f.format_rls(&rls_info, Action::Created)?; } Ok(()) } AutoMigrateStep::RemoveRowLevelSecurity(rls) => { - if let Some(rls_info) = extract_rls_info(*rls, plan)? { + if let Some(rls_info) = extract_rls_info(rls.as_ref(), plan)? { f.format_rls(&rls_info, Action::Removed)?; } Ok(()) } AutoMigrateStep::ChangeColumns(table) => { - let column_changes = extract_column_changes(*table, plan)?; + let column_changes = extract_column_changes(table, plan)?; f.format_change_columns(&column_changes) } AutoMigrateStep::AddColumns(table) => { // FIXME: It looks (pgoldman 2026-06-10) like `super::auto_migrate_table` will emit only `AddColumns` // in the case where a table has both new columns and changed columns. // As such, we probably need to call `extract_column_changes` here too. - let new_columns = extract_new_columns(*table, plan)?; + let new_columns = extract_new_columns(table, plan)?; f.format_add_columns(&new_columns) } AutoMigrateStep::DisconnectAllUsers => f.format_disconnect_warning(), @@ -161,7 +168,7 @@ pub enum Action { pub trait MigrationFormatter { fn format_header(&mut self) -> io::Result<()>; fn format_add_table(&mut self, table_info: &TableInfo) -> io::Result<()>; - fn format_remove_table(&mut self, table_name: &Identifier) -> io::Result<()>; + fn format_remove_table(&mut self, table_name: &str) -> io::Result<()>; fn format_view(&mut self, view_info: &ViewInfo, action: Action) -> io::Result<()>; fn format_index(&mut self, index_info: &IndexInfo, action: Action) -> io::Result<()>; fn format_constraint(&mut self, constraint_info: &ConstraintInfo, action: Action) -> io::Result<()>; @@ -180,7 +187,7 @@ pub trait MigrationFormatter { fn format_disconnect_warning(&mut self) -> io::Result<()>; // TODO(format-event-table-reschema): I (pgoldman 2026-06-10) didn't have time to meaningfully format event table reschemas, // so for now we're just printing the table name. - fn format_event_table_reschema(&mut self, table_name: &Identifier) -> io::Result<()>; + fn format_event_table_reschema(&mut self, table_name: &NamespacedIdentifier) -> io::Result<()>; } #[derive(Debug, Clone, PartialEq)] @@ -226,7 +233,7 @@ pub struct ColumnInfo { pub struct ConstraintInfo { pub name: RawIdentifier, pub columns: Vec, - pub table_name: Identifier, + pub table_name: RawIdentifier, } #[derive(Debug, Clone, PartialEq)] @@ -240,18 +247,18 @@ pub struct IndexInfo { pub struct SequenceInfo { pub name: RawIdentifier, pub column_name: Identifier, - pub table_name: Identifier, + pub table_name: RawIdentifier, } #[derive(Debug, Clone, PartialEq)] pub struct AccessChangeInfo { - pub table_name: Identifier, + pub table_name: RawIdentifier, pub new_access: TableAccess, } #[derive(Debug, Clone, PartialEq)] pub struct ScheduleInfo { - pub table_name: Identifier, + pub table_name: RawIdentifier, pub function_name: Identifier, pub function_kind: FunctionKind, } @@ -263,7 +270,7 @@ pub struct RlsInfo { #[derive(Debug, Clone, PartialEq)] pub struct ColumnChanges { - pub table_name: Identifier, + pub table_name: RawIdentifier, pub changes: Vec, } @@ -282,24 +289,23 @@ pub enum ColumnChange { #[derive(Debug, Clone, PartialEq)] pub struct NewColumns { - pub table_name: Identifier, + pub table_name: RawIdentifier, pub columns: Vec, } -// Data extraction functions (these replace the original print functions' data gathering logic) -fn extract_table_info( - table: ::Key<'_>, - plan: &super::AutoMigratePlan, -) -> Result { - let table_def = plan.new.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; +fn extract_table_info(full_name: &str, plan: &super::AutoMigratePlan) -> Result { + let (_, owning_def, table_def) = + plan.new + .find_table_by_full_name(full_name) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: full_name.into(), + })?; let columns = table_def .columns .iter() .map(|column| { - let type_name = WithTypespace::new(plan.new.typespace(), &column.ty) + let type_name = WithTypespace::new(owning_def.typespace(), &column.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; Ok(ColumnInfo { @@ -326,7 +332,7 @@ fn extract_table_info( Ok(column.name.clone()) }) .collect::, FormattingErrors>>()?, - table_name: table_def.name.clone(), + table_name: RawIdentifier::new(full_name), }) }) .collect::, FormattingErrors>>()?; @@ -349,7 +355,7 @@ fn extract_table_info( Ok(IndexInfo { name: index.name.clone(), columns, - table_name: table_def.name.clone().into(), + table_name: RawIdentifier::new(full_name), }) }) .collect::, FormattingErrors>>()?; @@ -365,19 +371,19 @@ fn extract_table_info( Ok(SequenceInfo { name: sequence.name.clone(), column_name: column.name.clone(), - table_name: table_def.name.clone(), + table_name: RawIdentifier::new(full_name), }) }) .collect::, FormattingErrors>>()?; let schedule = table_def.schedule.as_ref().map(|schedule| ScheduleInfo { - table_name: table_def.name.clone(), + table_name: RawIdentifier::new(full_name), function_name: schedule.function_name.clone(), function_kind: schedule.function_kind, }); Ok(TableInfo { - name: table_def.name.clone().into(), + name: RawIdentifier::new(full_name), is_system: table_def.table_type == TableType::System, access: table_def.table_access, columns, @@ -388,22 +394,19 @@ fn extract_table_info( }) } -fn extract_view_info( - view: ::Key<'_>, - module_def: &ModuleDef, -) -> Result { - let view_def = module_def.view(view).ok_or_else(|| FormattingErrors::ViewNotFound { - view: view.to_string().into(), - })?; +fn extract_view_info(full_name: &str, module_def: &ModuleDef) -> Result { + let (_, owning_def, view_def) = module_def + .find_view_by_full_name(full_name) + .ok_or_else(|| FormattingErrors::ViewNotFound { view: full_name.into() })?; - let name = view_def.name.clone().into(); + let name = RawIdentifier::new(full_name); let is_anonymous = view_def.is_anonymous; let params = view_def .param_columns .iter() .map(|column| { - let type_name = WithTypespace::new(module_def.typespace(), &column.ty) + let type_name = WithTypespace::new(owning_def.typespace(), &column.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; Ok(ViewParamInfo { @@ -417,7 +420,7 @@ fn extract_view_info( .return_columns .iter() .map(|column| { - let type_name = WithTypespace::new(module_def.typespace(), &column.ty) + let type_name = WithTypespace::new(owning_def.typespace(), &column.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; Ok(ViewColumnInfo { @@ -435,14 +438,10 @@ fn extract_view_info( }) } -fn extract_index_info( - index: ::Key<'_>, - module_def: &ModuleDef, -) -> Result { - let table_def = module_def - .stored_in_table_def(index) +fn extract_index_info(full_name: &str, module_def: &ModuleDef) -> Result { + let (prefix, _, table_def, index_def) = module_def + .find_index_by_full_name(full_name) .ok_or(FormattingErrors::IndexNotFound)?; - let index_def = table_def.indexes.get(index).ok_or(FormattingErrors::IndexNotFound)?; let columns = index_def .algorithm @@ -455,22 +454,15 @@ fn extract_index_info( .collect::, FormattingErrors>>()?; Ok(IndexInfo { - name: index_def.name.clone(), + name: RawIdentifier::new(full_name), columns, - table_name: table_def.name.clone().into(), + table_name: RawIdentifier::new(format!("{}{}", prefix, &*table_def.accessor_name)), }) } -fn extract_constraint_info( - constraint: ::Key<'_>, - module_def: &ModuleDef, -) -> Result { - let table_def = module_def - .stored_in_table_def(constraint) - .ok_or(FormattingErrors::ConstraintNotFound)?; - let constraint_def = table_def - .constraints - .get(constraint) +fn extract_constraint_info(full_name: &str, module_def: &ModuleDef) -> Result { + let (prefix, _, table_def, constraint_def) = module_def + .find_constraint_by_full_name(full_name) .ok_or(FormattingErrors::ConstraintNotFound)?; let ConstraintData::Unique(unique_constraint_data) = &constraint_def.data; @@ -484,22 +476,15 @@ fn extract_constraint_info( .collect::, FormattingErrors>>()?; Ok(ConstraintInfo { - name: constraint_def.name.clone(), + name: RawIdentifier::new(full_name), columns, - table_name: table_def.name.clone(), + table_name: RawIdentifier::new(format!("{}{}", prefix, &*table_def.accessor_name)), }) } -fn extract_sequence_info( - sequence: ::Key<'_>, - module_def: &ModuleDef, -) -> Result { - let table_def = module_def - .stored_in_table_def(sequence) - .ok_or(FormattingErrors::SequenceNotFound)?; - let sequence_def = table_def - .sequences - .get(sequence) +fn extract_sequence_info(full_name: &str, module_def: &ModuleDef) -> Result { + let (prefix, _, table_def, sequence_def) = module_def + .find_sequence_by_full_name(full_name) .ok_or(FormattingErrors::SequenceNotFound)?; let column = table_def @@ -507,47 +492,47 @@ fn extract_sequence_info( .ok_or(FormattingErrors::ColumnNotFound)?; Ok(SequenceInfo { - name: sequence_def.name.clone(), + name: RawIdentifier::new(full_name), column_name: column.name.clone(), - table_name: table_def.name.clone(), + table_name: RawIdentifier::new(format!("{}{}", prefix, &*table_def.accessor_name)), }) } fn extract_access_change_info( - table: ::Key<'_>, + full_name: &str, plan: &super::AutoMigratePlan, ) -> Result { - let table_def = plan.new.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; + let (_, _, table_def) = + plan.new + .find_table_by_full_name(full_name) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: full_name.into(), + })?; Ok(AccessChangeInfo { - table_name: table_def.name.clone(), + table_name: RawIdentifier::new(full_name), new_access: table_def.table_access, }) } -fn extract_schedule_info( - schedule_table: ::Key<'_>, - module_def: &ModuleDef, -) -> Result { - let schedule_def: &ScheduleDef = module_def - .lookup(schedule_table) +fn extract_schedule_info(table_full_name: &str, module_def: &ModuleDef) -> Result { + let (_, _, table_def) = module_def + .find_table_by_full_name(table_full_name) .ok_or(FormattingErrors::ScheduleNotFound)?; + let schedule_def = table_def.schedule.as_ref().ok_or(FormattingErrors::ScheduleNotFound)?; Ok(ScheduleInfo { - table_name: schedule_def.name.clone(), + table_name: RawIdentifier::new(table_full_name), function_name: schedule_def.function_name.clone(), function_kind: schedule_def.function_kind, }) } -fn extract_rls_info( - rls: ::Key<'_>, - plan: &super::AutoMigratePlan, -) -> Result, FormattingErrors> { - // Skip if policy unchanged (implementation detail workaround) - if plan.old.lookup::(rls) == plan.new.lookup::(rls) { +fn extract_rls_info(rls: &str, plan: &super::AutoMigratePlan) -> Result, FormattingErrors> { + let in_old = plan.old.row_level_security().any(|r| &*r.sql == rls); + let in_new = plan.new.row_level_security().any(|r| &*r.sql == rls); + // Skip if policy is unchanged (present in both old and new). + if in_old == in_new { return Ok(None); } @@ -556,16 +541,19 @@ fn extract_rls_info( })) } -fn extract_column_changes( - table: ::Key<'_>, - plan: &super::AutoMigratePlan, -) -> Result { - let old_table = plan.old.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; - let new_table = plan.new.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; +fn extract_column_changes(full_name: &str, plan: &super::AutoMigratePlan) -> Result { + let (_, old_owning, old_table) = + plan.old + .find_table_by_full_name(full_name) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: full_name.into(), + })?; + let (_, new_owning, new_table) = + plan.new + .find_table_by_full_name(full_name) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: full_name.into(), + })?; let mut changes = Vec::new(); @@ -579,10 +567,10 @@ fn extract_column_changes( }); } if old_col.ty != new_col.ty { - let old_type = WithTypespace::new(plan.old.typespace(), &old_col.ty) + let old_type = WithTypespace::new(old_owning.typespace(), &old_col.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; - let new_type = WithTypespace::new(plan.new.typespace(), &new_col.ty) + let new_type = WithTypespace::new(new_owning.typespace(), &new_col.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; changes.push(ColumnChange::TypeChanged { @@ -595,26 +583,29 @@ fn extract_column_changes( } Ok(ColumnChanges { - table_name: new_table.name.clone(), + table_name: RawIdentifier::new(full_name), changes, }) } -fn extract_new_columns( - table: ::Key<'_>, - plan: &super::AutoMigratePlan, -) -> Result { - let table_def = plan.new.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; - let old_table_def = plan.old.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; +fn extract_new_columns(full_name: &str, plan: &super::AutoMigratePlan) -> Result { + let (_, new_owning, table_def) = + plan.new + .find_table_by_full_name(full_name) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: full_name.into(), + })?; + let (_, _, old_table_def) = + plan.old + .find_table_by_full_name(full_name) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: full_name.into(), + })?; let mut new_columns = Vec::new(); for column in &table_def.columns { if !old_table_def.columns.iter().any(|c| c.col_id == column.col_id) { - let type_name = WithTypespace::new(plan.new.typespace(), &column.ty) + let type_name = WithTypespace::new(new_owning.typespace(), &column.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; new_columns.push(ColumnInfo { @@ -626,7 +617,7 @@ fn extract_new_columns( } Ok(NewColumns { - table_name: table_def.name.clone(), + table_name: RawIdentifier::new(full_name), columns: new_columns, }) } diff --git a/crates/schema/src/auto_migrate/termcolor_formatter.rs b/crates/schema/src/auto_migrate/termcolor_formatter.rs index 3f9764cfabf..5f031ac8afc 100644 --- a/crates/schema/src/auto_migrate/termcolor_formatter.rs +++ b/crates/schema/src/auto_migrate/termcolor_formatter.rs @@ -7,12 +7,12 @@ use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type; use termcolor::{Buffer, Color, ColorChoice, ColorSpec, WriteColor}; use crate::auto_migrate::formatter::ViewInfo; -use crate::identifier::Identifier; use super::formatter::{ AccessChangeInfo, Action, ColumnChange, ColumnChanges, ConstraintInfo, IndexInfo, MigrationFormatter, NewColumns, RlsInfo, ScheduleInfo, SequenceInfo, TableInfo, }; +use crate::identifier::NamespacedIdentifier; /// Color scheme for consistent formatting #[derive(Debug, Clone)] @@ -231,7 +231,7 @@ impl MigrationFormatter for TermColorFormatter { self.write_line("") } - fn format_remove_table(&mut self, table_name: &Identifier) -> io::Result<()> { + fn format_remove_table(&mut self, table_name: &str) -> io::Result<()> { self.write_action_prefix(&Action::Removed)?; self.buffer.write_all(b" table: ")?; self.write_colored(table_name, Some(self.colors.table_name), true)?; @@ -427,7 +427,7 @@ impl MigrationFormatter for TermColorFormatter { self.buffer.write_all(b"\n") } - fn format_event_table_reschema(&mut self, table_name: &Identifier) -> io::Result<()> { + fn format_event_table_reschema(&mut self, table_name: &NamespacedIdentifier) -> io::Result<()> { // TODO(format-event-table-reschema): I (pgoldman 2026-06-10) didn't have time to meaningfully format event table reschemas, // so for now we're just printing the table name. diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 9ad07dce72f..f4faf62cb21 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -34,8 +34,8 @@ use spacetimedb_lib::db::raw_def; use spacetimedb_lib::db::raw_def::v10::{ ExplicitNames, MethodOrAny, RawConstraintDefV10, RawHttpHandlerDefV10, RawHttpRouteDefV10, RawIndexDefV10, RawLifeCycleReducerDefV10, RawModuleDefV10, RawModuleDefV10Section, RawProcedureDefV10, RawReducerDefV10, - RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, RawSequenceDefV10, RawTableDefV10, - RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10, + RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, RawSequenceDefV10, RawSubmoduleV10, + RawTableDefV10, RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10, }; use spacetimedb_lib::db::raw_def::v9::{ Lifecycle, RawColumnDefaultValueV9, RawConstraintDataV9, RawConstraintDefV9, RawIndexAlgorithm, RawIndexDefV9, @@ -164,6 +164,9 @@ pub struct ModuleDef { /// was authored under. #[allow(unused)] raw_module_def_version: RawModuleDefVersion, + + /// Submodules, keyed by the namespace they are registered under. + submodules: IndexMap, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -180,6 +183,11 @@ impl ModuleDef { self.raw_module_def_version } + /// The submodules of the module definition. + pub fn submodules(&self) -> &IndexMap { + &self.submodules + } + /// The tables of the module definition. pub fn tables(&self) -> impl Iterator { self.tables.values() @@ -205,14 +213,176 @@ impl ModuleDef { self.tables().filter_map(|table| table.schedule.as_ref()) } + /// All tables across this module and all submodules, in depth-first order. + /// + /// Each item is `(namespace, owning_def, table_def)` where `namespace` is the dot-terminated + /// namespace string (e.g., `"alias."`) to be prepended to the table's name for database storage. + /// The consumer module's own tables yield namespace `""`. + pub fn all_tables_with_prefix(&self) -> Vec<(String, &ModuleDef, &TableDef)> { + let mut out = Vec::new(); + self.collect_tables_with_prefix("", &mut out); + out + } + + fn collect_tables_with_prefix<'a>(&'a self, prefix: &str, out: &mut Vec<(String, &'a ModuleDef, &'a TableDef)>) { + for table in self.tables.values() { + out.push((prefix.to_string(), self, table)); + } + for (ns, submodule) in &self.submodules { + submodule.collect_tables_with_prefix(&format!("{prefix}{ns}."), out); + } + } + + /// All views across this module and all submodules, in depth-first order. + /// + /// Each item is `(namespace, owning_def, view_def)` where `namespace` is the dot-terminated + /// namespace string (e.g., `"alias."`) to be prepended to the view's name. + /// The consumer module's own views yield namespace `""`. + pub fn all_views_with_prefix(&self) -> Vec<(String, &ModuleDef, &ViewDef)> { + let mut out = Vec::new(); + self.collect_views_with_prefix("", &mut out); + out + } + + fn collect_views_with_prefix<'a>(&'a self, prefix: &str, out: &mut Vec<(String, &'a ModuleDef, &'a ViewDef)>) { + for view in self.views.values() { + out.push((prefix.to_string(), self, view)); + } + for (ns, submodule) in &self.submodules { + submodule.collect_views_with_prefix(&format!("{prefix}{ns}."), out); + } + } + + /// Look up a table by its full namespaced name (e.g., `"lib.library_table"` or `"user"`). + pub fn find_table_by_full_name(&self, full_name: &str) -> Option<(String, &ModuleDef, &TableDef)> { + self.all_tables_with_prefix() + .into_iter() + .find(|(prefix, _, table_def)| format!("{}{}", prefix, &*table_def.accessor_name) == full_name) + } + + /// Look up a view by its full namespaced name (e.g., `"lib.library_view"` or `"my_view"`). + pub fn find_view_by_full_name(&self, full_name: &str) -> Option<(String, &ModuleDef, &ViewDef)> { + self.all_views_with_prefix() + .into_iter() + .find(|(prefix, _, view_def)| format!("{}{}", prefix, &*view_def.name) == full_name) + } + + /// Look up an index by its full namespaced name (e.g., `"lib.library_table_id_idx_btree"`). + pub fn find_index_by_full_name(&self, full_name: &str) -> Option<(String, &ModuleDef, &TableDef, &IndexDef)> { + for (prefix, owning, table) in self.all_tables_with_prefix() { + for idx in table.indexes.values() { + if format!("{}{}", prefix, &*idx.name) == full_name { + return Some((prefix, owning, table, idx)); + } + } + } + None + } + + /// Look up a sequence by its full namespaced name (e.g., `"lib.library_table_id_seq"`). + pub fn find_sequence_by_full_name(&self, full_name: &str) -> Option<(String, &ModuleDef, &TableDef, &SequenceDef)> { + for (prefix, owning, table) in self.all_tables_with_prefix() { + for seq in table.sequences.values() { + if format!("{}{}", prefix, &*seq.name) == full_name { + return Some((prefix, owning, table, seq)); + } + } + } + None + } + + /// Look up a constraint by its full namespaced name (e.g., `"lib.library_table_id_unique"`). + pub fn find_constraint_by_full_name( + &self, + full_name: &str, + ) -> Option<(String, &ModuleDef, &TableDef, &ConstraintDef)> { + for (prefix, owning, table) in self.all_tables_with_prefix() { + for constraint in table.constraints.values() { + if format!("{}{}", prefix, &*constraint.name) == full_name { + return Some((prefix, owning, table, constraint)); + } + } + } + None + } + /// The reducers of the module definition. pub fn reducers(&self) -> impl Iterator { self.reducers.values() } - /// Returns an iterator over all reducer ids and definitions. - pub fn reducer_ids_and_defs(&self) -> impl ExactSizeIterator { - self.reducers.values().enumerate().map(|(idx, def)| (idx.into(), def)) + /// Returns all reducer ids and definitions in depth-first submodule order. + /// + /// IDs are assigned as follows: consumer's own reducers first (0..N), then each + /// submodule's reducers in the order they appear in `submodules`, recursively. + pub fn reducer_ids_and_defs(&self) -> Vec<(ReducerId, &ReducerDef)> { + let mut out = Vec::with_capacity(self.reducer_count()); + self.collect_reducers(0, &mut out); + out + } + + /// Total reducer count including all submodules (depth-first sum). + pub fn reducer_count(&self) -> usize { + self.reducers.len() + self.submodules.values().map(|m| m.reducer_count()).sum::() + } + + fn collect_reducers<'a>(&'a self, offset: usize, out: &mut Vec<(ReducerId, &'a ReducerDef)>) { + for (i, def) in self.reducers.values().enumerate() { + out.push(((offset + i).into(), def)); + } + let mut child_offset = offset + self.reducers.len(); + for submodule in self.submodules.values() { + submodule.collect_reducers(child_offset, out); + child_offset += submodule.reducer_count(); + } + } + + /// All reducers across this module and all submodules, in depth-first order. + /// + /// Each item is `(prefix, owning_def, reducer_def)` where `prefix` is the dot-terminated + /// namespace string (e.g., `"lib."`) to be prepended to the reducer's name as its wire name. + /// The consumer module's own reducers yield prefix `""`. + pub fn all_reducers_with_prefix(&self) -> Vec<(String, &ModuleDef, &ReducerDef)> { + let mut out = Vec::new(); + self.collect_reducers_with_prefix("", &mut out); + out + } + + fn collect_reducers_with_prefix<'a>( + &'a self, + prefix: &str, + out: &mut Vec<(String, &'a ModuleDef, &'a ReducerDef)>, + ) { + for reducer in self.reducers.values() { + out.push((prefix.to_string(), self, reducer)); + } + for (ns, submodule) in &self.submodules { + submodule.collect_reducers_with_prefix(&format!("{prefix}{ns}."), out); + } + } + + /// All procedures across this module and all submodules, in depth-first order. + /// + /// Each item is `(prefix, owning_def, procedure_def)` where `prefix` is the dot-terminated + /// namespace string (e.g., `"lib."`) to be prepended to the procedure's name as its wire name. + /// The consumer module's own procedures yield prefix `""`. + pub fn all_procedures_with_prefix(&self) -> Vec<(String, &ModuleDef, &ProcedureDef)> { + let mut out = Vec::new(); + self.collect_procedures_with_prefix("", &mut out); + out + } + + fn collect_procedures_with_prefix<'a>( + &'a self, + prefix: &str, + out: &mut Vec<(String, &'a ModuleDef, &'a ProcedureDef)>, + ) { + for procedure in self.procedures.values() { + out.push((prefix.to_string(), self, procedure)); + } + for (ns, submodule) in &self.submodules { + submodule.collect_procedures_with_prefix(&format!("{prefix}{ns}."), out); + } } /// The procedures of the module definition. @@ -230,6 +400,26 @@ impl ModuleDef { &self.http_routes } + /// Returns warnings about this module's definition that will not prevent publishing. + /// + /// - Warns when a submodule registers HTTP routes via a top-level router export + /// because those routes are ignored by the host. + pub fn collect_warnings(&self) -> Vec { + let mut warnings = Vec::new(); + for (namespace, submodule_def) in self.submodules() { + if !submodule_def.http_routes().is_empty() { + warnings.push(format!( + "The submodule under namespace '{namespace}' registers HTTP routes via a router. \ + Route registrations in submodules are ignored. Only the root module's routes \ + are served. Define routes in the root module and call the submodule's HTTP handler \ + functions via `ctx.as.{namespace}`." + )); + } + warnings.extend(submodule_def.collect_warnings()); + } + warnings + } + /// Returns an iterator over all HTTP handler ids and definitions. pub fn http_handler_ids_and_defs(&self) -> impl ExactSizeIterator { self.http_handlers @@ -364,14 +554,59 @@ impl ModuleDef { self.reducers.get_full(name).map(|(idx, _, def)| (idx.into(), def)) } - /// Look up a reducer by its id. + /// Look up a reducer by its wire name, resolving qualified names like `"myauth.verify_token"`. + /// + /// A plain name searches the consumer's own reducers. A dot-qualified name routes to + /// the matching submodule and recurses. Nesting is supported: `"auth.baz.cleanup"`. + /// Returns the depth-first `ReducerId` and the `ReducerDef`. + pub fn reducer_by_name(&self, name: &str) -> Option<(ReducerId, &ReducerDef)> { + self.reducer_by_name_with_module(name).map(|(id, def, _)| (id, def)) + } + + /// Like `reducer_by_name` but also returns the `ModuleDef` that owns the reducer. + /// Use the returned `ModuleDef` (not `self`) when calling `arg_seed_for`, so that + /// type-index references in the `ReducerDef` are resolved against the correct typespace. + pub fn reducer_by_name_with_module<'a>(&'a self, name: &str) -> Option<(ReducerId, &'a ReducerDef, &'a ModuleDef)> { + match name.split_once('.') { + None => self + .reducers + .get_full(name) + .map(|(idx, _, def)| (idx.into(), def, self)), + Some((namespace, rest)) => { + let mut offset = self.reducers.len(); + for (ns, submodule) in &self.submodules { + if ns == namespace { + let (inner_id, def, owning) = submodule.reducer_by_name_with_module(rest)?; + return Some(((offset + inner_id.idx()).into(), def, owning)); + } + offset += submodule.reducer_count(); + } + None + } + } + } + + /// Look up a reducer by its depth-first id. pub fn reducer_by_id(&self, id: ReducerId) -> &ReducerDef { - &self.reducers[id.idx()] + self.get_reducer_by_id(id) + .unwrap_or_else(|| panic!("reducer id {id:?} out of range")) } - /// Look up a reducer by its id. + /// Look up a reducer by its depth-first id, returning `None` if it doesn't exist. pub fn get_reducer_by_id(&self, id: ReducerId) -> Option<&ReducerDef> { - self.reducers.get_index(id.idx()).map(|(_, def)| def) + let idx = id.idx(); + if idx < self.reducers.len() { + return self.reducers.get_index(idx).map(|(_, def)| def); + } + let mut offset = self.reducers.len(); + for submodule in self.submodules.values() { + let count = submodule.reducer_count(); + if idx < offset + count { + return submodule.get_reducer_by_id(ReducerId::from(idx - offset)); + } + offset += count; + } + None } /// Look up a view by its id, and whether it is anonymous. @@ -382,6 +617,139 @@ impl ModuleDef { .map(|(_, def)| def) } + /// Look up a view by its globally-unique fn_ptr (the offset-adjusted id used by the WASM dispatch layer). + /// Returns the `ViewDef` and the owning `ModuleDef`. + pub fn get_view_by_global_id_with_module( + &self, + global_id: ViewFnPtr, + is_anonymous: bool, + ) -> Option<(&ViewDef, &ModuleDef)> { + self.get_view_by_global_id_inner(global_id.0, is_anonymous, 0, 0) + } + + fn get_view_by_global_id_inner( + &self, + global_id: u32, + is_anonymous: bool, + anon_offset: u32, + non_anon_offset: u32, + ) -> Option<(&ViewDef, &ModuleDef)> { + let local_count = if is_anonymous { + self.anon_view_count() as u32 + } else { + self.non_anon_view_count() as u32 + }; + let offset = if is_anonymous { anon_offset } else { non_anon_offset }; + if global_id < offset + local_count { + return self + .views + .values() + .find(|def| def.fn_ptr.0 + offset == global_id && def.is_anonymous == is_anonymous) + .map(|def| (def, self)); + } + let mut anon_off = anon_offset + self.anon_view_count() as u32; + let mut non_anon_off = non_anon_offset + self.non_anon_view_count() as u32; + for submodule in self.submodules.values() { + let submodule_anon = submodule.total_anon_view_count() as u32; + let submodule_non_anon = submodule.total_non_anon_view_count() as u32; + let submodule_count = if is_anonymous { + submodule_anon + } else { + submodule_non_anon + }; + let submodule_off = if is_anonymous { anon_off } else { non_anon_off }; + if global_id < submodule_off + submodule_count { + return submodule.get_view_by_global_id_inner(global_id, is_anonymous, anon_off, non_anon_off); + } + anon_off += submodule_anon; + non_anon_off += submodule_non_anon; + } + None + } + + /// Look up a view by its wire name, resolving dot-qualified names like `"lib.library_view"`. + /// + /// A plain name searches this module's own views. A dot-qualified name routes to + /// the matching submodule and recurses. Returns the `ViewDef` and the owning `ModuleDef`. + pub fn view_by_name_with_module<'a>(&'a self, name: &str) -> Option<(&'a ViewDef, &'a ModuleDef)> { + match name.split_once('.') { + None => self.views.get(name).map(|def| (def, self)), + Some((namespace, rest)) => { + let submodule = self.submodules.get(namespace)?; + submodule.view_by_name_with_module(rest) + } + } + } + + /// Like [`view_by_name_with_module`] but also returns the globally-unique `ViewFnPtr` + /// that the WASM dispatch layer expects (offset by all anon/non-anon views that precede + /// this one in depth-first module order). + pub fn view_by_name_with_global_fn_ptr<'a>( + &'a self, + name: &str, + ) -> Option<(ViewFnPtr, &'a ViewDef, &'a ModuleDef)> { + let anon_offset = 0u32; + let non_anon_offset = 0u32; + self.view_by_name_with_global_fn_ptr_inner(name, anon_offset, non_anon_offset) + } + + fn view_by_name_with_global_fn_ptr_inner<'a>( + &'a self, + name: &str, + anon_offset: u32, + non_anon_offset: u32, + ) -> Option<(ViewFnPtr, &'a ViewDef, &'a ModuleDef)> { + match name.split_once('.') { + None => { + let def = self.views.get(name)?; + let offset = if def.is_anonymous { anon_offset } else { non_anon_offset }; + Some((ViewFnPtr(def.fn_ptr.0 + offset), def, self)) + } + Some((namespace, rest)) => { + let mut anon_off = anon_offset + self.anon_view_count() as u32; + let mut non_anon_off = non_anon_offset + self.non_anon_view_count() as u32; + for (ns, submodule) in &self.submodules { + if ns == namespace { + return submodule.view_by_name_with_global_fn_ptr_inner(rest, anon_off, non_anon_off); + } + anon_off += submodule.total_anon_view_count() as u32; + non_anon_off += submodule.total_non_anon_view_count() as u32; + } + None + } + } + } + + /// Count of anonymous views in this module (not including submodules). + pub fn anon_view_count(&self) -> usize { + self.views.values().filter(|v| v.is_anonymous).count() + } + + /// Count of non-anonymous views in this module (not including submodules). + pub fn non_anon_view_count(&self) -> usize { + self.views.values().filter(|v| !v.is_anonymous).count() + } + + /// Total anonymous view count including all submodules (depth-first sum). + pub fn total_anon_view_count(&self) -> usize { + self.anon_view_count() + + self + .submodules + .values() + .map(|m| m.total_anon_view_count()) + .sum::() + } + + /// Total non-anonymous view count including all submodules (depth-first sum). + pub fn total_non_anon_view_count(&self) -> usize { + self.non_anon_view_count() + + self + .submodules + .values() + .map(|m| m.total_non_anon_view_count()) + .sum::() + } + /// Convenience method to look up a procedure, possibly by a string. pub fn procedure>(&self, name: &K) -> Option<&ProcedureDef> { // If the string IS a valid identifier, we can just look it up. @@ -399,12 +767,63 @@ impl ModuleDef { /// Look up a procuedure by its id, panicking if it doesn't exist. pub fn procedure_by_id(&self, id: ProcedureId) -> &ProcedureDef { - &self.procedures[id.idx()] + self.get_procedure_by_id(id) + .unwrap_or_else(|| panic!("procedure id {id:?} out of range")) } /// Look up a procuedure by its id, returning `None` if it doesn't exist. pub fn get_procedure_by_id(&self, id: ProcedureId) -> Option<&ProcedureDef> { - self.procedures.get_index(id.idx()).map(|(_, def)| def) + let idx = id.idx(); + if idx < self.procedures.len() { + return self.procedures.get_index(idx).map(|(_, def)| def); + } + let mut offset = self.procedures.len(); + for submodule in self.submodules.values() { + let count = submodule.procedure_count(); + if idx < offset + count { + return submodule.get_procedure_by_id(ProcedureId::from(idx - offset)); + } + offset += count; + } + None + } + + /// Total procedure count including all submodules (depth-first sum). + pub fn procedure_count(&self) -> usize { + self.procedures.len() + self.submodules.values().map(|m| m.procedure_count()).sum::() + } + + /// Look up a procedure by its wire name, resolving qualified names like `"mylib.proc_name"`. + /// + /// A plain name searches the module's own procedures. A dot-qualified name routes to + /// the matching submodule and recurses. Returns the depth-first `ProcedureId` and the `ProcedureDef`. + pub fn procedure_by_name(&self, name: &str) -> Option<(ProcedureId, &ProcedureDef)> { + self.procedure_by_name_with_module(name).map(|(id, def, _)| (id, def)) + } + + /// Like `procedure_by_name` but also returns the `ModuleDef` that owns the procedure. + /// Use the returned `ModuleDef` (not `self`) when calling `arg_seed_for`. + pub fn procedure_by_name_with_module<'a>( + &'a self, + name: &str, + ) -> Option<(ProcedureId, &'a ProcedureDef, &'a ModuleDef)> { + match name.split_once('.') { + None => self + .procedures + .get_full(name) + .map(|(idx, _, def)| (idx.into(), def, self)), + Some((namespace, rest)) => { + let mut offset = self.procedures.len(); + for (ns, submodule) in &self.submodules { + if ns == namespace { + let (inner_id, def, owning) = submodule.procedure_by_name_with_module(rest)?; + return Some(((offset + inner_id.idx()).into(), def, owning)); + } + offset += submodule.procedure_count(); + } + None + } + } } /// Looks up a lifecycle reducer defined in the module. @@ -412,6 +831,11 @@ impl ModuleDef { self.lifecycle_reducers[lifecycle].map(|i| (i, &self.reducers[i.idx()])) } + /// All lifecycle reducer assignments for this module (does not include submodules). + pub fn lifecycle_reducers_map(&self) -> &EnumMap> { + &self.lifecycle_reducers + } + /// Returns a `DeserializeSeed` that can pull data from a `Deserializer` for `def`. pub fn arg_seed_for<'a, T>(&'a self, def: &'a T) -> ArgsSeed<'a, T> { ArgsSeed(self.typespace.with_type(def)) @@ -507,6 +931,7 @@ impl From for RawModuleDefV9 { http_handlers: _, http_routes: _, raw_module_def_version: _, + submodules: _, } = val; // Extract column defaults from tables before consuming tables @@ -565,6 +990,7 @@ impl From for RawModuleDefV10 { http_handlers, http_routes, raw_module_def_version: _, + submodules, } = val; let mut sections = Vec::new(); @@ -723,6 +1149,17 @@ impl From for RawModuleDefV10 { // Always emit ExplicitNames so canonical names survive the round-trip. sections.push(RawModuleDefV10Section::ExplicitNames(explicit_names)); + let submodules: Vec<_> = submodules + .into_iter() + .map(|(namespace, module)| RawSubmoduleV10 { + namespace, + module: module.into(), + }) + .collect(); + if !submodules.is_empty() { + sections.push(RawModuleDefV10Section::Submodules(submodules)); + } + RawModuleDefV10 { sections } } } @@ -2174,4 +2611,82 @@ mod tests { .count() == 2)) } + + #[test] + fn submodule_reducer_ids_are_depth_first() { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10Builder, RawModuleDefV10Section, RawSubmoduleV10}; + + // baz library: 1 reducer + let mut baz_builder = RawModuleDefV10Builder::new(); + baz_builder.add_reducer("baz_reduce", ProductType::unit()); + + // auth library: 1 own reducer, uses baz as a submodule + let mut auth_builder = RawModuleDefV10Builder::new(); + auth_builder.add_reducer("auth_verify", ProductType::unit()); + let mut auth_raw = auth_builder.finish(); + auth_raw + .sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "baz".to_string(), + module: baz_builder.finish(), + }])); + + // consumer: 2 own reducers, uses auth as a submodule + let mut consumer_builder = RawModuleDefV10Builder::new(); + consumer_builder.add_reducer("consumer_a", ProductType::unit()); + consumer_builder.add_reducer("consumer_b", ProductType::unit()); + let mut consumer_raw = consumer_builder.finish(); + consumer_raw + .sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "auth".to_string(), + module: auth_raw, + }])); + + let def: ModuleDef = consumer_raw.try_into().expect("valid module"); + + // Total count: 2 consumer + 1 auth + 1 baz + assert_eq!(def.reducer_count(), 4); + + // Depth-first order: consumer_a=0, consumer_b=1, auth_verify=2, baz_reduce=3 + let ids_and_defs = def.reducer_ids_and_defs(); + assert_eq!(ids_and_defs.len(), 4); + assert_eq!(ids_and_defs[0].0, ReducerId(0)); + assert_eq!(&*ids_and_defs[0].1.name, "consumer_a"); + assert_eq!(ids_and_defs[1].0, ReducerId(1)); + assert_eq!(&*ids_and_defs[1].1.name, "consumer_b"); + assert_eq!(ids_and_defs[2].0, ReducerId(2)); + assert_eq!(&*ids_and_defs[2].1.name, "auth_verify"); + assert_eq!(ids_and_defs[3].0, ReducerId(3)); + assert_eq!(&*ids_and_defs[3].1.name, "baz_reduce"); + + // get_reducer_by_id resolves submodule reducer IDs correctly + assert_eq!(&*def.reducer_by_id(ReducerId(2)).name, "auth_verify"); + assert_eq!(&*def.reducer_by_id(ReducerId(3)).name, "baz_reduce"); + assert!(def.get_reducer_by_id(ReducerId(4)).is_none()); + + // reducer_by_name routes plain names to own reducers + let (id, rdef) = def.reducer_by_name("consumer_a").expect("plain name resolves"); + assert_eq!(id, ReducerId(0)); + assert_eq!(&*rdef.name, "consumer_a"); + + // reducer_by_name routes qualified names to submodule reducers + let (id, rdef) = def + .reducer_by_name("auth.auth_verify") + .expect("qualified name resolves"); + assert_eq!(id, ReducerId(2)); + assert_eq!(&*rdef.name, "auth_verify"); + + // reducer_by_name routes deeply nested qualified names + let (id, rdef) = def + .reducer_by_name("auth.baz.baz_reduce") + .expect("nested qualified name resolves"); + assert_eq!(id, ReducerId(3)); + assert_eq!(&*rdef.name, "baz_reduce"); + + // Non-existent names return None + assert!(def.reducer_by_name("auth.nonexistent").is_none()); + assert!(def.reducer_by_name("nonexistent").is_none()); + assert!(def.reducer_by_name("nonamespace.auth_verify").is_none()); + } } diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index aa398743f97..e7d26b63e76 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -2,9 +2,11 @@ use spacetimedb_data_structures::error_stream::ErrorStream; use spacetimedb_data_structures::map::HashMap; use spacetimedb_lib::bsatn::Deserializer; use spacetimedb_lib::db::raw_def::v10::*; +use spacetimedb_lib::db::raw_def::v9::Lifecycle; use spacetimedb_lib::db::view::{extract_view_return_product_type_ref, ViewKind}; use spacetimedb_lib::de::DeserializeSeed as _; use spacetimedb_lib::http::character_is_acceptable_for_route_path; +use spacetimedb_primitives::ReducerId; use spacetimedb_sats::{Typespace, WithTypespace}; use crate::def::validate::v9::{ @@ -13,6 +15,7 @@ use crate::def::validate::v9::{ }; use crate::def::*; use crate::error::ValidationError; +use crate::identifier::validate_identifier; use crate::type_for_generate::ProductTypeDef; use crate::{def::validate::Result, error::TypeLocation}; // Utitility struct to look up canonical names for tables, functions, and indexes based on the @@ -71,7 +74,7 @@ impl From for ValidationCase { } } } -/// Validate a `RawModuleDefV9` and convert it into a `ModuleDef`, +/// Validate a `RawModuleDefV10` and convert it into a `ModuleDef`, /// or return a stream of errors if the definition is invalid. pub fn validate(def: RawModuleDefV10) -> Result { let mut typespace = def.typespace().cloned().unwrap_or_else(|| Typespace::EMPTY.clone()); @@ -83,6 +86,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { .map(ExplicitNamesLookup::new) .unwrap_or_default(); let view_primary_keys = def.view_primary_keys().cloned().unwrap_or_default(); + let submodules = validate_submodules(def.submodules().into_iter().flat_map(|s| s.iter().cloned()).collect()); // Original `typespace` needs to be preserved to be assign `accesor_name`s to columns. let typespace_with_accessor_names = typespace.clone(); @@ -292,13 +296,9 @@ pub fn validate(def: RawModuleDefV10) -> Result { .map(|rls| (rls.sql.clone(), rls.to_owned())) .collect(); - let (tables, types, reducers, procedures, views, http_handlers, http_routes) = - tables_types_reducers_procedures_views - .map( - |(tables, types, reducers, procedures, views, (http_handlers, http_routes))| { - (tables, types, reducers, procedures, views, http_handlers, http_routes) - }, - ) + let ((tables, types, reducers, procedures, views, (http_handlers, http_routes)), submodules) = + (tables_types_reducers_procedures_views, submodules) + .combine_errors() .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; let typespace_for_generate = typespace_for_generate.finish(); @@ -318,9 +318,55 @@ pub fn validate(def: RawModuleDefV10) -> Result { http_handlers, http_routes, raw_module_def_version: RawModuleDefVersion::V10, + submodules, }) } +/// Validate that each submodule's namespace is a valid identifier of at most 63 characters, +/// that no two submodules share the same namespace, and that no submodule declares lifecycle +/// reducers (lifecycle reducers are only permitted in the root module). +/// This function will inspect each sub-submodule and recursively collect errors. +fn validate_submodules(submodules: Vec) -> Result> { + let mut errors = vec![]; + let mut map = IndexMap::with_capacity(submodules.len()); + + for submodule in submodules { + if let Err(e) = validate_identifier(&submodule.namespace) { + errors.push(ValidationError::IdentifierError { error: e }); + } + + if submodule.namespace.len() > 63 { + errors.push(ValidationError::NamespaceTooLong { + namespace: submodule.namespace.clone().into(), + len: submodule.namespace.len(), + }); + } + + if map.contains_key(&submodule.namespace) { + errors.push(ValidationError::DuplicateName { + name: submodule.namespace.into(), + }); + } else { + match validate(submodule.module) { + Ok(def) => { + for (lifecycle, opt_id) in def.lifecycle_reducers_map() { + if opt_id.is_some() { + errors.push(ValidationError::LifecycleInSubmodule { + lifecycle, + namespace: submodule.namespace.clone(), + }); + } + } + map.insert(submodule.namespace, def); + } + Err(e) => errors.extend(e.into_iter()), + } + } + } + + ValidationErrors::add_extra_errors(Ok(map), errors) +} + /// Change the visibility of scheduled functions and lifecycle reducers to Internal. /// fn change_scheduled_functions_and_lifetimes_visibility( @@ -1094,12 +1140,16 @@ mod tests { use itertools::Itertools; use spacetimedb_data_structures::expect_error_matching; - use spacetimedb_lib::db::raw_def::v10::{CaseConversionPolicy, MethodOrAny, RawModuleDefV10Builder}; + use spacetimedb_lib::db::raw_def::v10::{ + CaseConversionPolicy, MethodOrAny, RawModuleDefV10, RawModuleDefV10Builder, RawModuleDefV10Section, + RawSubmoduleV10, + }; use spacetimedb_lib::db::raw_def::v9::{btree, direct, hash}; use spacetimedb_lib::db::raw_def::*; use spacetimedb_lib::http::Method as HttpMethod; use spacetimedb_lib::ScheduleAt; use spacetimedb_primitives::{ColId, ColList, ColSet}; + use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, AlgebraicValue, ProductType, SumValue}; use v9::{Lifecycle, TableAccess, TableType}; @@ -1474,6 +1524,66 @@ mod tests { }); } + #[test] + fn validates_submodules_recursively() { + let mut submodule_builder = RawModuleDefV10Builder::new(); + submodule_builder + .build_table_with_new_type("Sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "authlib".to_string(), + module: submodule_builder.finish(), + }])], + }; + + let def: ModuleDef = raw.try_into().expect("submodule should validate"); + let submodules = def.submodules(); + + assert_eq!(submodules.len(), 1); + let submodule = submodules.get("authlib").expect("authlib submodule should exist"); + assert!(submodule.table(&expect_identifier("sessions")).is_some()); + } + + #[test] + fn invalid_submodule_namespace() { + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "".to_string(), + module: RawModuleDefV10::default(), + }])], + }; + + let result: Result = raw.try_into(); + + expect_error_matching!(result, ValidationError::IdentifierError { error } => { + error == &IdentifierError::Empty {} + }); + } + + #[test] + fn duplicate_submodule_namespace() { + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![ + RawSubmoduleV10 { + namespace: "authlib".to_string(), + module: RawModuleDefV10::default(), + }, + RawSubmoduleV10 { + namespace: "authlib".to_string(), + module: RawModuleDefV10::default(), + }, + ])], + }; + + let result: Result = raw.try_into(); + + expect_error_matching!(result, ValidationError::DuplicateName { name } => { + name == &RawIdentifier::from("authlib") + }); + } + #[test] fn invalid_unique_constraint_column_ref() { let mut builder = RawModuleDefV10Builder::new(); @@ -2516,4 +2626,94 @@ mod tests { assert_eq!(view.return_columns[0].view_name, id("Level2Person")); assert_eq!(view.param_columns[0].view_name, id("Level2Person")); } + + #[test] + fn namespace_exactly_63_chars_is_ok() { + let namespace = "a".repeat(63); + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace, + module: RawModuleDefV10::default(), + }])], + }; + let result: Result = raw.try_into(); + assert!(result.is_ok(), "63-char namespace should be valid"); + } + + #[test] + fn namespace_64_chars_is_rejected() { + let namespace = "a".repeat(64); + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: namespace.clone(), + module: RawModuleDefV10::default(), + }])], + }; + let expected_ns = RawIdentifier::from(namespace.clone()); + let result: Result = raw.try_into(); + expect_error_matching!(result, ValidationError::NamespaceTooLong { namespace: ns, len } => { + ns == &expected_ns && len == &64usize + }); + } + + fn make_module_with_lifecycle(lifecycle: Lifecycle) -> RawModuleDefV10 { + let mut b = RawModuleDefV10Builder::new(); + b.add_lifecycle_reducer(lifecycle, "lifecycle_fn", ProductType::unit()); + b.finish() + } + + #[test] + fn lifecycle_in_submodule_is_rejected() { + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "auth".to_string(), + module: make_module_with_lifecycle(Lifecycle::Init), + }])], + }; + + let result: Result = raw.try_into(); + expect_error_matching!(result, ValidationError::LifecycleInSubmodule { lifecycle, namespace } => { + lifecycle == &Lifecycle::Init && namespace == "auth" + }); + } + + #[test] + fn lifecycle_in_root_with_submodule_is_ok() { + // Root declares Init; the submodule has no lifecycle — this is valid. + let consumer_raw = make_module_with_lifecycle(Lifecycle::Init); + let mut sections = consumer_raw.sections; + sections.push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "auth".to_string(), + module: RawModuleDefV10::default(), + }])); + + let result: Result = RawModuleDefV10 { sections }.try_into(); + assert!( + result.is_ok(), + "lifecycle in root with a lifecycle-free submodule should be valid" + ); + } + + #[test] + fn lifecycle_in_nested_submodule_is_rejected() { + // Root uses auth as submodule; auth uses baz as submodule; baz declares a lifecycle. Should be rejected. + let auth = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "baz".to_string(), + module: make_module_with_lifecycle(Lifecycle::Init), + }])], + }; + + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "auth".to_string(), + module: auth, + }])], + }; + + let result: Result = raw.try_into(); + expect_error_matching!(result, ValidationError::LifecycleInSubmodule { lifecycle, namespace } => { + lifecycle == &Lifecycle::Init && namespace == "baz" + }); + } } diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 618f8e3c9c4..401a6e67e75 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -168,6 +168,7 @@ pub fn validate(def: RawModuleDefV9) -> Result { http_handlers: IndexMap::new(), http_routes: Vec::new(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, + submodules: IndexMap::new(), }) } diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 0bab5445d92..3b5f19c4840 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -176,6 +176,13 @@ pub enum ValidationError { ok_type: PrettyAlgebraicType, err_type: PrettyAlgebraicType, }, + #[error( + "lifecycle event {lifecycle:?} is not permitted in submodule under namespace `{namespace}`; \ + lifecycle reducers may only be declared in the root module" + )] + LifecycleInSubmodule { lifecycle: Lifecycle, namespace: String }, + #[error("submodule namespace `{namespace}` is {len} characters, which exceeds the 63-character limit")] + NamespaceTooLong { namespace: RawIdentifier, len: usize }, } /// A wrapper around an `AlgebraicType` that implements `fmt::Display`. diff --git a/crates/schema/src/identifier.rs b/crates/schema/src/identifier.rs index 06a396ec012..25a1fb8d5c4 100644 --- a/crates/schema/src/identifier.rs +++ b/crates/schema/src/identifier.rs @@ -35,6 +35,47 @@ impl_st!([] Identifier, ts => RawIdentifier::make_type(ts)); impl_serialize!([] Identifier, (self, ser) => ser.serialize_str(&self.id)); impl_deserialize!([] Identifier, de => RawIdentifier::deserialize(de).map(Self::new_assume_valid)); +/// Validates that `name` is a valid identifier string +pub fn validate_identifier(name: &str) -> Result<(), IdentifierError> { + if name.is_empty() { + return Err(IdentifierError::Empty {}); + } + + // Convert to Unicode Normalization Form C (canonical decomposition followed by composition). + if name.nfc().zip(name.chars()).any(|(a, b)| a != b) { + return Err(IdentifierError::NotCanonicalized { + name: RawIdentifier::new(name), + }); + } + + let mut chars = name.chars(); + + let start = chars.next().ok_or(IdentifierError::Empty {})?; + if !is_xid_start(start) && start != '_' { + return Err(IdentifierError::InvalidStart { + name: RawIdentifier::new(name), + invalid_start: start, + }); + } + + for char_ in chars { + if !is_xid_continue(char_) { + return Err(IdentifierError::InvalidContinue { + name: RawIdentifier::new(name), + invalid_continue: char_, + }); + } + } + + if Identifier::is_reserved(name) { + return Err(IdentifierError::Reserved { + name: RawIdentifier::new(name), + }); + } + + Ok(()) +} + impl Identifier { /// Returns a new identifier without validating the input. pub fn new_assume_valid(name: RawIdentifier) -> Self { @@ -46,38 +87,7 @@ impl Identifier { /// Currently, this rejects non-canonicalized identifiers. /// Eventually, it will be changed to canonicalize the input string. pub fn new(name: RawIdentifier) -> Result { - if name.is_empty() { - return Err(IdentifierError::Empty {}); - } - - // Convert to Unicode Normalization Form C (canonical decomposition followed by composition). - if name.nfc().zip(name.chars()).any(|(a, b)| a != b) { - return Err(IdentifierError::NotCanonicalized { name }); - } - - let mut chars = name.chars(); - - let start = chars.next().ok_or(IdentifierError::Empty {})?; - if !is_xid_start(start) && start != '_' { - return Err(IdentifierError::InvalidStart { - name, - invalid_start: start, - }); - } - - for char_ in chars { - if !is_xid_continue(char_) { - return Err(IdentifierError::InvalidContinue { - name, - invalid_continue: char_, - }); - } - } - - if Identifier::is_reserved(&name) { - return Err(IdentifierError::Reserved { name }); - } - + validate_identifier(&name)?; Ok(Identifier { id: name }) } @@ -128,6 +138,78 @@ impl From for RawIdentifier { } } +/// An identifier that allows dot or slash separators between otherwise-normal identifier segments. +/// +/// Used for fully-qualified names of mounted module items, e.g.: +/// - `"lib.library_table"` (table name) +/// - `"lib.library_table_id_idx_btree"` (index name) +/// - `"lib.library_reducer"` (reducer name) +/// +/// Root-level items use their plain name with no separator (e.g., `"user"`). +/// Construction from known-valid components should use `new_assume_valid`. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct NamespacedIdentifier(Box); + +impl NamespacedIdentifier { + /// Construct without validation. Use when building from known-valid `Identifier` segments + /// (e.g., `format!("{}{}", prefix, &*identifier)`). + pub fn new_assume_valid(s: impl Into>) -> Self { + Self(s.into()) + } + + /// Validated construction: each segment (split on `.`) must satisfy XID rules. + pub fn new(s: impl Into>) -> Result { + let s = s.into(); + for segment in s.split(['.']) { + validate_identifier(segment)?; + } + Ok(Self(s)) + } +} + +impl std::fmt::Debug for NamespacedIdentifier { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", &*self.0) + } +} + +impl std::fmt::Display for NamespacedIdentifier { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::ops::Deref for NamespacedIdentifier { + type Target = str; + fn deref(&self) -> &str { + &self.0 + } +} + +impl AsRef for NamespacedIdentifier { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl From for Box { + fn from(id: NamespacedIdentifier) -> Self { + id.0 + } +} + +impl From<&str> for NamespacedIdentifier { + fn from(s: &str) -> Self { + Self::new_assume_valid(s) + } +} + +impl From for NamespacedIdentifier { + fn from(s: String) -> Self { + Self::new_assume_valid(s) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/schema/src/schema.rs b/crates/schema/src/schema.rs index 4079d98a70a..6551ed0c68b 100644 --- a/crates/schema/src/schema.rs +++ b/crates/schema/src/schema.rs @@ -983,7 +983,10 @@ impl Schema for TableSchema { } fn check_compatible(&self, module_def: &ModuleDef, def: &Self::Def) -> Result<(), anyhow::Error> { - ensure_eq!(&self.table_name[..], &def.name[..], "Table name mismatch"); + // Submodule tables are stored in the DB with a namespace prefix (e.g. "lib.library_table"), + // but the def's name is just the local name ("library_table"). Strip any prefix before comparing. + let self_local_name = self.table_name.rsplit('.').next().unwrap_or(&self.table_name[..]); + ensure_eq!(self_local_name, &def.name[..], "Table name mismatch"); ensure_eq!(self.primary_key, def.primary_key, "Primary key mismatch"); let def_table_access: StAccess = (def.table_access).into(); ensure_eq!(self.table_access, def_table_access, "Table access mismatch"); @@ -999,10 +1002,15 @@ impl Schema for TableSchema { } ensure_eq!(self.columns.len(), def.columns.len(), "Column count mismatch"); + // Index names in the DB are prefixed for submodule tables (e.g. "lib.library_table_id_idx_btree"), + // but def.indexes is keyed by the bare name. Derive the namespace prefix length from the + // difference between the full DB table name and the def's canonical (local) table name. + let ns_prefix_len = self.table_name.len() - def.name.len(); for index in &self.indexes { + let bare_name = &index.index_name[ns_prefix_len..]; let index_def = def .indexes - .get(&index.index_name) + .get(bare_name) .ok_or_else(|| anyhow::anyhow!("Index {} not found in definition", index.index_id.0))?; index.check_compatible(module_def, index_def)?; } @@ -1352,8 +1360,11 @@ impl Schema for ScheduleSchema { fn check_compatible(&self, _module_def: &ModuleDef, def: &Self::Def) -> Result<(), anyhow::Error> { ensure_eq!(&self.schedule_name[..], &def.name[..], "Schedule name mismatch"); + // For submodule tables, schedule function names in the DB are namespace-prefixed using '.' + // (e.g. "lib.library_scheduled_procedure") while def.function_name is the bare name. + let ns_len = self.function_name.len().saturating_sub(def.function_name.len()); ensure_eq!( - &self.function_name[..], + &self.function_name[ns_len..], &def.function_name[..], "Schedule function name mismatch" ); @@ -1421,7 +1432,11 @@ impl Schema for IndexSchema { } fn check_compatible(&self, _module_def: &ModuleDef, def: &Self::Def) -> Result<(), anyhow::Error> { - ensure_eq!(&self.index_name[..], &def.name[..], "Index name mismatch"); + // For submodule tables, the DB stores index names with a namespace prefix + // (e.g. "lib.library_table_id_idx_btree") while def.name is the bare name. + // Strip any prefix by comparing only the suffix of length def.name.len(). + let ns_len = self.index_name.len().saturating_sub(def.name.len()); + ensure_eq!(&self.index_name[ns_len..], &def.name[..], "Index name mismatch"); ensure_eq!(&self.index_algorithm, &def.algorithm, "Index algorithm mismatch"); Ok(()) } diff --git a/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print no color.snap b/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print no color.snap index bb6cbc98ef0..dea71a64749 100644 --- a/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print no color.snap +++ b/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print no color.snap @@ -9,7 +9,7 @@ Database Migration Plan ▸ Removed index Apples_id_name_idx_btree on [id, name] of table Apples ▸ Removed unique constraint Apples_id_key on [id] of table Apples ▸ Removed auto-increment constraint Apples_id_seq on column id of table Apples -▸ Removed schedule for table Deliveries_sched calling reducer check_deliveries +▸ Removed schedule for table Deliveries calling reducer check_deliveries ▸ ▸ Removed anonymous view: my_view Parameters: • x: U32 @@ -38,7 +38,7 @@ Database Migration Plan ▸ Created index Apples_id_count_idx_btree on [id, count] of table Apples ▸ Created auto-increment constraint Bananas_id_seq on column id of table Bananas -▸ Created schedule for table Inspections_sched calling reducer perform_inspection +▸ Created schedule for table Inspections calling reducer perform_inspection ▸ ▸ Created anonymous view: my_view Parameters: • x: U32 diff --git a/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print.snap b/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print.snap index b9a9b7827bd..6d643270705 100644 --- a/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print.snap +++ b/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print.snap @@ -9,7 +9,7 @@ expression: "plan.pretty_print(PrettyPrintStyle::AnsiColor).expect(\"should pret ▸ Removed index Apples_id_name_idx_btree on [id, name] of table Apples ▸ Removed unique constraint Apples_id_key on [id] of table Apples ▸ Removed auto-increment constraint Apples_id_seq on column id of table Apples -▸ Removed schedule for table Deliveries_sched calling reducer check_deliveries +▸ Removed schedule for table Deliveries calling reducer check_deliveries ▸ ▸ Removed anonymous view: my_view Parameters: • x: U32 @@ -38,7 +38,7 @@ expression: "plan.pretty_print(PrettyPrintStyle::AnsiColor).expect(\"should pret ▸ Created index Apples_id_count_idx_btree on [id, count] of table Apples ▸ Created auto-increment constraint Bananas_id_seq on column id of table Bananas -▸ Created schedule for table Inspections_sched calling reducer perform_inspection +▸ Created schedule for table Inspections calling reducer perform_inspection ▸ ▸ Created anonymous view: my_view Parameters: • x: U32 diff --git a/crates/schema/src/table_name.rs b/crates/schema/src/table_name.rs index 3fe32ed70da..c535d8c0472 100644 --- a/crates/schema/src/table_name.rs +++ b/crates/schema/src/table_name.rs @@ -5,20 +5,26 @@ use spacetimedb_sats::{impl_deserialize, impl_serialize, impl_st, raw_identifier /// The name of a table. #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct TableName(Identifier); +pub struct TableName(RawIdentifier); -impl_st!([] TableName, ts => Identifier::make_type(ts)); +impl_st!([] TableName, ts => RawIdentifier::make_type(ts)); impl_serialize!([] TableName, (self, ser) => self.0.serialize(ser)); -impl_deserialize!([] TableName, de => Identifier::deserialize(de).map(Self)); +impl_deserialize!([] TableName, de => RawIdentifier::deserialize(de).map(Self)); impl TableName { + /// Construct from a validated identifier (all user-defined tables). pub fn new(id: Identifier) -> Self { - Self(id) + Self(id.into()) + } + + /// Construct from an arbitrary raw string (e.g. submodule tables whose names contain `.`). + pub fn new_raw(name: RawIdentifier) -> Self { + Self(name) } #[cfg(any(test, feature = "test"))] pub fn for_test(name: &str) -> Self { - Self(Identifier::for_test(name)) + Self(RawIdentifier::new(name)) } } @@ -38,13 +44,13 @@ impl AsRef for TableName { impl From for Identifier { fn from(id: TableName) -> Self { - id.0 + Identifier::new(id.0).expect("TableName contains '.' or other non-identifier chars; use RawIdentifier instead") } } impl From for RawIdentifier { fn from(id: TableName) -> Self { - Identifier::from(id).into() + id.0 } } diff --git a/crates/schema/tests/ensure_same_schema.rs b/crates/schema/tests/ensure_same_schema.rs index 7e5cb28f626..edbc547f83e 100644 --- a/crates/schema/tests/ensure_same_schema.rs +++ b/crates/schema/tests/ensure_same_schema.rs @@ -81,11 +81,32 @@ macro_rules! declare_tests { declare_tests! { benchmarks => "benchmarks", - module_test => "module-test", + // module_test is excluded here and handled explicitly below. sdk_test_connect_disconnect => "sdk-test-connect-disconnect", sdk_test => "sdk-test", } +// These are written out explicitly rather than via the macro so we can +// temporarily disable just the TypeScript variant. +mod ensure_same_schema_module_test { + use super::*; + + #[test] + #[serial] + fn csharp() { + super::assert_identical_modules("module-test", "C#", "cs"); + } + + // TODO: Re-enable once Rust supports namespaced tables (sub-module mounts). + // TypeScript's module-test now includes a sub-module that emits namespaced tables + // (e.g. "lib.lib_data"), which Rust does not yet produce, causing a spurious diff. + // #[test] + // #[serial] + // fn typescript() { + // super::assert_identical_modules("module-test", "typescript", "ts"); + // } +} + #[test] #[serial] fn ensure_same_schema_rust_csharp_benchmarks() { diff --git a/crates/sql-parser/src/parser/mod.rs b/crates/sql-parser/src/parser/mod.rs index b6fbe15c0dd..fb930abde6f 100644 --- a/crates/sql-parser/src/parser/mod.rs +++ b/crates/sql-parser/src/parser/mod.rs @@ -5,6 +5,8 @@ use sqlparser::ast::{ WildcardAdditionalOptions, }; +use spacetimedb_lib::sats::raw_identifier::RawIdentifier; + use crate::ast::{ BinOp, LogOp, Parameter, Project, ProjectElem, ProjectExpr, SqlExpr, SqlFrom, SqlIdent, SqlJoin, SqlLiteral, }; @@ -348,5 +350,8 @@ pub(crate) fn parse_parts(mut parts: Vec) -> SqlParseResult { if parts.len() == 1 { return Ok(parts.swap_remove(0).into()); } - Err(SqlUnsupported::MultiPartName(ObjectName(parts)).into()) + // Join multi-part names (e.g. `lib.library_table`) with dots to match + // namespace-prefixed table names stored in the catalog. + let joined = parts.iter().map(|p| p.value.as_str()).collect::>().join("."); + Ok(SqlIdent(RawIdentifier::new(joined))) } diff --git a/crates/sql-parser/src/parser/sql.rs b/crates/sql-parser/src/parser/sql.rs index e689817cc4c..27cf43337b5 100644 --- a/crates/sql-parser/src/parser/sql.rs +++ b/crates/sql-parser/src/parser/sql.rs @@ -403,8 +403,6 @@ mod tests { for sql in [ // FROM is required "select 1", - // Multi-part table names - "select a from s.t", // Bit-string literals "select * from t where a = B'1010'", // Wildcard with non-wildcard projections @@ -430,6 +428,8 @@ mod tests { fn supported() { for sql in [ "select a from t", + // Multi-part names are joined with dots for namespace-qualified tables + "select a from s.t", "select a from t where x = :sender", "select count(*) as n from t", "select count(*) as n from t join s on t.id = s.id where s.x = 1", diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index b03b87df9b5..a6db21350d5 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -9,11 +9,14 @@ use std::time::Instant; use bytes::{Bytes, BytesMut}; use futures::{FutureExt as _, TryStreamExt as _}; use spacetimedb::config::CertificateAuthority; +use spacetimedb::host::ModuleHost; use spacetimedb::messages::control_db::HostType; use spacetimedb::util::jobs::JobCores; use spacetimedb::Identity; use spacetimedb_client_api::auth::SpacetimeAuth; use spacetimedb_client_api::routes::subscribe::{generate_random_connection_id, WebSocketOptions}; +use spacetimedb_lib::http as st_http; +use spacetimedb_lib::AlgebraicValue; use spacetimedb_paths::{RootDir, SpacetimePaths}; use spacetimedb_schema::auto_migrate::MigrationPolicy; use spacetimedb_schema::def::ModuleDef; @@ -166,6 +169,56 @@ impl ModuleHandle { .expect("failed to collect log stream"); String::from_utf8(bytes.into()).unwrap() } + + async fn module_host(&self) -> ModuleHost { + let database = self + .env + .get_database_by_identity(&self.db_identity) + .await + .unwrap() + .unwrap(); + let host = self.env.leader(database.id).await.expect("host should be running"); + host.module().await.expect("module should be running") + } + + /// Call a procedure by name with JSON-encoded args, returning the raw `AlgebraicValue` on success. + pub async fn call_procedure_with_args(&self, procedure: &str, args_json: &str) -> anyhow::Result { + let module = self.module_host().await; + let ret = module + .call_procedure( + Identity::ZERO, + None, + None, + procedure, + FunctionArgs::Json(args_json.into()), + ) + .await; + ret.result + .map(|r| r.return_val) + .map_err(|e| anyhow::anyhow!("procedure {procedure} failed: {e:#}")) + } + + /// Dispatch a GET request to a module HTTP route by path, returning the response body on success. + pub async fn call_http_route_get(&self, path: &str) -> anyhow::Result { + let module = self.module_host().await; + let (handler_id, _, _) = module + .info() + .module_def + .match_http_route(&st_http::Method::Get, path) + .ok_or_else(|| anyhow::anyhow!("no GET route registered for {path}"))?; + let request = st_http::Request { + method: st_http::Method::Get, + headers: std::iter::empty::<(Option>, Box<[u8]>)>().collect(), + timeout: None, + uri: format!("http://localhost{path}"), + version: st_http::Version::Http11, + }; + let (_response, body) = module + .call_http_handler(handler_id, request, Bytes::new()) + .await + .map_err(|e| anyhow::anyhow!("HTTP handler error: {e}"))?; + Ok(body) + } } pub struct CompiledModule { diff --git a/crates/testing/tests/standalone_integration_test.rs b/crates/testing/tests/standalone_integration_test.rs index 8de5cfe5bc4..bd5bf090f01 100644 --- a/crates/testing/tests/standalone_integration_test.rs +++ b/crates/testing/tests/standalone_integration_test.rs @@ -459,3 +459,49 @@ fn test_calling_bench_db_ia_loop_typescript() { fn test_calling_bench_db_ia_loop_cpp() { test_calling_bench_db_ia_loop::(); } + +fn test_submodule_in_module(module_name: &'static str) { + init(); + + CompiledModule::compile(module_name, CompilationMode::Debug).with_module_async( + DEFAULT_CONFIG, + |mut module| async move { + // ── 1. Cross-namespace reducer call ────────────────────────────────── + // use_submodule calls lib_insert in the lib submodule. + let json = + r#"{"CallReducer": {"reducer": "use_submodule", "args": "[\"hello_submodule\"]", "request_id": 0, "flags": 0}}"# + .to_string(); + module.send_reducer_and_recv_update(json, 0).await.unwrap(); + + let logs = read_logs(&module).await; + let relevant: Vec<_> = logs + .into_iter() + .filter(|l| !is_scheduled_test_log(l)) + .collect(); + assert_eq!(relevant, ["lib_insert: hello_submodule"].map(String::from)); + + // ── 2. Cross-namespace procedure call ───────────────────────────────── + // use_submodule_procedure calls lib_count in the lib submodule. + // We inserted one row above, so the count should be 1. + let return_val = module + .call_procedure_with_args("use_submodule_procedure", "[]") + .await + .expect("use_submodule_procedure should succeed"); + assert_eq!(return_val, AlgebraicValue::U64(1), "lib_count should return 1 after one insert"); + + // ── 3. Cross-namespace HTTP handler ─────────────────────────────────── + // The root module's /lib-hello route delegates to lib_submodule's lib_hello handler. + let body = module + .call_http_route_get("/lib-hello") + .await + .expect("GET /lib-hello should succeed"); + assert_eq!(body.as_ref(), b"Hello from lib submodule!"); + }, + ); +} + +#[test] +#[serial] +fn test_submodule_typescript() { + test_submodule_in_module("module-test-ts"); +} diff --git a/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md b/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md new file mode 100644 index 00000000000..5399e4a2b23 --- /dev/null +++ b/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md @@ -0,0 +1,346 @@ +--- +title: Submodules +slug: /submodules +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +A **submodule** is a SpacetimeDB module that can be included in another module's database. The submodule's tables and functions register under a **namespace** you choose, keeping them separate from the consumer's own tables and from other submodules. + +Submodules let you package reusable database logic as a library that any consumer can integrate without coordinating table names. + +:::note +Submodules are currently supported in TypeScript only. Support for Rust, C#, and C++ is coming soon. +::: + +## Writing a Submodule + +A submodule is a regular SpacetimeDB module. Nothing special marks a module as a submodule. Export the schema as the default export and export every function (reducers, procedures, views, HTTP handlers) the consumer needs to register. + + + + +```typescript +// auth_lib/src/index.ts +import { schema, table, t, SyncResponse, Router } from 'spacetimedb/server'; + +const users = table( + { name: 'users', public: true }, + { identity: t.identity().primaryKey(), username: t.string() } +); + +const sessions = table( + { name: 'sessions' }, + { + id: t.u64().primaryKey().autoInc(), + user_identity: t.identity(), + token: t.string(), + } +); + +const spacetimedb = schema({ users, sessions }); +export default spacetimedb; + +export const verify_token = spacetimedb.reducer( + { token: t.string() }, + (ctx, { token }) => { /* ... */ } +); + +export const session_count = spacetimedb.procedure( + t.u64(), + (ctx) => ctx.withTx(tx => tx.db.sessions.count()) +); + +export const active_sessions = spacetimedb.anonymousView( + { name: 'active_sessions', public: true }, + t.array(sessions.rowType), + (ctx) => [...ctx.db.sessions.iter()] +); + +export const health = spacetimedb.httpHandler( + (_ctx, _req) => new SyncResponse('ok') +); + +export const router = spacetimedb.httpRouter( + new Router().get('/health', health) +); +``` + +The same file can be published as a standalone database or used as a submodule by another module. The module does not need to declare which role it plays. + + + + +## Using a Submodule + +The consumer controls the namespace name. Pass the submodule's module-namespace object under the alias you choose. + + + + +```typescript +// my-database/src/index.ts +import { schema } from 'spacetimedb/server'; +import * as authLib from 'auth_lib'; + +const players = table({ name: 'players', public: true }, { /* ... */ }); + +const spacetimedb = schema({ + players, + myauth: authLib, // register auth_lib under the namespace "myauth" +}); +export default spacetimedb; +``` + +`export default spacetimedb` is the only JS export required from the consumer. Registering the submodule adds all of its reducers, procedures, views, scheduled tables, and HTTP handlers automatically. + +:::warning Use `import * as`, not a default import +```typescript +import authLib from 'auth_lib'; // ❌ misses all named exports +import * as authLib from 'auth_lib'; // ✅ correct +``` +A default-only import exposes only the submodule's schema, not its named exports (reducers, procedures, views, handlers). The submodule walker requires the full module-namespace object. A clear error is raised if you use the wrong form. +::: + + + + +## Accessing Submodule Tables and Views + +Submodule tables appear under a namespace field on `ctx.db`. The field matches the alias you chose. Views exported by a submodule behave like tables from the client's perspective: they are accessible as `.` in subscriptions and SQL queries. + + + + +```typescript +spacetimedb.reducer('example', {}, (ctx) => { + // Consumer's own tables (no namespace) + for (const player of ctx.db.players.iter()) { /* ... */ } + + // Submodule tables, registered as "myauth" + const user = ctx.db.myauth.users.identity.find(ctx.sender); + for (const session of ctx.db.myauth.sessions.iter()) { /* ... */ } +}); +``` + + + + +## Calling Submodule Functions + +A submodule can expose reducers, procedures, views, and HTTP handlers that the consumer calls from its own functions. Because the submodule's context type and the consumer's context type are distinct, use `ctx.as.` to narrow the context before passing it to a submodule function. + +### From a Reducer + + + + +Call a submodule reducer or a plain helper function typed against the submodule's schema using `ctx.as.`: + +```typescript +// auth_lib: plain helper function typed against the submodule's own schema +export function sessionCountHelper(ctx: ReducerContext): number { + return ctx.db.sessions.count(); +} + +// my-database: call submodule reducer and helper from a consumer reducer +spacetimedb.reducer('on_login', { token: t.string() }, (ctx, { token }) => { + // call a submodule reducer + authLib.verify_token(ctx.as.myauth, { token }); + + // call a submodule helper function + const count = authLib.sessionCountHelper(ctx.as.myauth); + console.log(`Active sessions: ${count}`); +}); +``` + +`ctx.as.myauth` is a `ReducerContext` scoped to the `myauth` namespace. It shares the same sender, timestamp, and connectionId as the parent context, but its `ctx.db` points at `ctx.db.myauth`. + +For reducers registered through the submodule's own schema (via `schema.reducer(...)`), the host passes a scoped context automatically when invoked directly. `ctx.as` is only needed when the consumer calls a submodule function explicitly. + + + + +### From a Procedure + + + + +Use `ctx.as.` to pass a submodule-scoped `ProcedureContext` to a submodule procedure. To call a submodule reducer from inside a procedure, open a transaction first with `ctx.withTx` and then narrow with `tx.as.`: + +```typescript +// call a submodule procedure +export const stats = spacetimedb.procedure( + t.u64(), + (ctx) => authLib.session_count(ctx.as.myauth) +); + +// call a submodule reducer inside a withTx block +export const transact_and_count = spacetimedb.procedure( + { token: t.string() }, + t.u64(), + (ctx, { token }) => { + ctx.withTx(tx => { + // tx is a root ReducerContext; narrow to the submodule namespace + authLib.verify_token(tx.as.myauth, { token }); + }); + return authLib.session_count(ctx.as.myauth); + } +); +``` + + + + +### From an HTTP Handler + + + + +Delegate to a submodule's HTTP handler by passing `ctx.as.` and the request to the submodule handler function, then register it on the consumer's router: + +```typescript +import { Router } from 'spacetimedb/server'; + +// delegate the /health route to the submodule's handler +export const health_check = spacetimedb.httpHandler((ctx, req) => { + return authLib.health(ctx.as.myauth, req); +}); + +export const router = spacetimedb.httpRouter( + new Router().get('/health', health_check) +); +``` + + + + +## Multiple and Nested Submodules + +Multiple submodules compose freely. Each gets its own namespace, so name collisions between submodules are impossible. + + + + +```typescript +import * as authLib from 'auth_lib'; +import * as paymentLib from 'payment_lib'; + +const spacetimedb = schema({ + players, + myauth: authLib, + payments: paymentLib, +}); +export default spacetimedb; +``` + +A submodule can itself include other submodules using the same syntax. The nested submodule's tables appear under a two-level path in the top-level consumer: + +```typescript +// auth_lib includes session_lib as "sessions" +const authSchema = schema({ users, sessions: sessionLib }); +export default authSchema; + +// consumer includes auth_lib as "myauth" +// session_lib's tables are at ctx.db.myauth.sessions.
+``` + +**Lifecycle reducers** (`init`, `clientConnected`, `clientDisconnected`) are an exception: these are only allowed for the root module. Modules containing submodules which define lifecycle reducers will fail to publish. + + + + +## Client Subscriptions + +Client subscriptions use the same namespace structure as server-side access. Submodule tables and views are queried as `.`. + + + + +```typescript +conn.subscriptionBuilder() + .addQuery(q => q.from.players.build()) // public.players + .addQuery(q => q.from.myauth.users.build()) // myauth.users + .addQuery(q => q.from.myauth.activeSessions.build()) // myauth.active_sessions view + .subscribe(); +``` + + + + +## Calling Submodule Reducers and Procedures from the Client + +Submodule reducers and procedures are identified by their fully-qualified name, using `/` as the separator between namespace and function name. + +### Client SDK + +In generated bindings, submodule tables, views, reducers, and procedures appear as nested objects under the namespace alias. The `tables`, `reducers`, and `procedures` exports all reflect the same nesting. + + + + +React hooks: + +```typescript +import { tables, reducers, procedures } from './module_bindings'; +import { useTable, useReducer, useProcedure } from 'spacetimedb/react'; + +// subscribe to a submodule table and view +const [users] = useTable(tables.myauth.users); +const [activeSessions] = useTable(tables.myauth.activeSessions); + +// call a submodule reducer +const verifyToken = useReducer(reducers.myauth.verifyToken); +verifyToken({ token: 'abc123' }); + +// call a submodule procedure +const sessionCount = useProcedure(procedures.myauth.sessionCount); +sessionCount().then(count => console.log(`Sessions: ${count}`)); +``` + +Vanilla (non-React): + +```typescript +import { DbConnection, tables, reducers, procedures } from './module_bindings'; + +const conn = DbConnection.builder() + .withUri(SPACETIMEDB_URI) + .withDatabaseName('my-database') + .onConnect(ctx => { + ctx.subscriptionBuilder() + .subscribe([tables.myauth.users, tables.myauth.activeSessions]); + }) + .build(); + +conn.reducers.myauth.verifyToken({ token: 'abc123' }); +``` + + + + +### HTTP API + +``` +POST /v1/database/my-database/call/myauth/verify_token +``` + +### CLI + +```bash +spacetime call my-database "myauth/verify_token" '{"token": "abc123"}' +``` + +The namespace prefix is the alias you chose, and the function name after `/` is the snake_case export name from the submodule. + +## Namespace Name Rules + +The alias you choose becomes the SQL-level namespace name. It must be a valid SpacetimeDB identifier: starts with a letter or underscore, continues with letters, digits, or underscores, maximum 63 characters, case-insensitive for resolution. A submodule can be registered under at most one alias per consumer module. + +The reserved namespaces `public`, `st`, `spacetimedb`, and `pg_*` cannot be used as submodule aliases. + +## Limitations + +### Submodule routers are not applied automatically + +A submodule can define its own `httpRouter`, but when used as a submodule that router is ignored. Only the consumer's root router is used. To expose a submodule's HTTP handlers, register them explicitly on the consumer's router using `ctx.as.` as shown in the [HTTP handler section](#from-an-http-handler). diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 164f7b7c1f1..8269eb9e8a9 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -3,15 +3,15 @@ // ───────────────────────────────────────────────────────────────────────────── import { ScheduleAt } from 'spacetimedb'; import { - Router, schema, - SyncResponse, table, t, type Infer, type InferTypeOfRow, errors, + Router, } from 'spacetimedb/server'; +import * as libSubmodule from './lib_submodule'; // ───────────────────────────────────────────────────────────────────────────── // TYPE ALIASES @@ -246,6 +246,7 @@ const spacetimedb = schema({ playerLikeRow ), tableToRemove: table({ name: 'table_to_remove' }, { id: t.u32() }), + lib: libSubmodule, }); export default spacetimedb; @@ -523,10 +524,25 @@ export const getMySchemaViaHttp = spacetimedb.procedure(t.string(), ctx => { } }); -export const getSimple = spacetimedb.httpHandler( - (_ctx, _req) => new SyncResponse('ok') +// use_submodule: calls the lib submodule's lib_insert reducer cross-namespace. +export const use_submodule = spacetimedb.reducer( + { value: t.string() }, + (ctx, { value }) => { + libSubmodule.lib_insert(ctx.as.lib, { value }); + } ); +// use_submodule_procedure: calls the lib submodule's lib_count procedure and returns the result. +export const use_submodule_procedure = spacetimedb.procedure( + t.u64(), + (ctx) => libSubmodule.lib_count(ctx.as.lib, {}) +); + +// Delegates to the lib submodule's HTTP handler, demonstrating cross-namespace HTTP dispatch. +export const lib_hello = spacetimedb.httpHandler((ctx, req) => { + return libSubmodule.lib_hello(ctx.as.lib, req); +}); + export const router = spacetimedb.httpRouter( - new Router().get('/get', getSimple) + new Router().get('/lib-hello', lib_hello) ); diff --git a/modules/module-test-ts/src/lib_submodule.ts b/modules/module-test-ts/src/lib_submodule.ts new file mode 100644 index 00000000000..c91d73d357c --- /dev/null +++ b/modules/module-test-ts/src/lib_submodule.ts @@ -0,0 +1,29 @@ +import { schema, table, t, SyncResponse, Router } from 'spacetimedb/server'; + +const lib_data = table( + { name: 'lib_data', public: true }, + { + id: t.u64().primaryKey().autoInc(), + value: t.string(), + } +); + +const libSubmoduleSchema = schema({ lib_data }); +export default libSubmoduleSchema; + +export const lib_insert = libSubmoduleSchema.reducer( + { value: t.string() }, + (ctx, { value }) => { + console.info(`lib_insert: ${value}`); + ctx.db.lib_data.insert({ id: 0n, value }); + } +); + +export const lib_count = libSubmoduleSchema.procedure( + t.u64(), + (ctx) => ctx.withTx(tx => tx.db.lib_data.count()) +); + +export const lib_hello = libSubmoduleSchema.httpHandler((_ctx, _req) => { + return new SyncResponse('Hello from lib submodule!'); +});