diff --git a/projection.go b/projection.go index 47339e2..cb09cbb 100644 --- a/projection.go +++ b/projection.go @@ -2,6 +2,7 @@ package dalgo2sql import ( "fmt" + "strings" "github.com/dal-go/dalgo/dal" ) @@ -12,6 +13,31 @@ type wildcardProjectionPlan struct { explicitCount int } +// expandSQLiteWildcard replaces the wildcard with the source's visible column +// names. SQLite schema metadata supplies names without fetching any row +// values. Duplicate names cannot be addressed unambiguously in a +// SELECT list, so those retain the existing result-filtering path. +func (p wildcardProjectionPlan) expandSQLiteWildcard(q dal.StructuredQuery, names []string) (dal.StructuredQuery, bool) { + seen := make(map[string]bool, len(names)) + columns := make([]dal.Column, 0, len(names)+p.explicitCount) + for _, name := range names { + if name == "" || seen[strings.ToLower(name)] { + return q, false + } + seen[strings.ToLower(name)] = true + if !p.projection.Excludes(name) { + columns = append(columns, dal.Column{Expression: dal.Field(name)}) + } + } + columns = append(columns, q.Columns()[1:]...) + if len(columns) == 0 { + // SQL has no zero-column SELECT. Preserve the current empty-result + // behavior until an explicit zero-column reader representation exists. + return q, false + } + return dal.WithColumns(q, columns), true +} + func planWildcardProjection(q dal.StructuredQuery) (*wildcardProjectionPlan, error) { columns := q.Columns() var projection *dal.WildcardProjection diff --git a/projection_test.go b/projection_test.go index daaba62..4779442 100644 --- a/projection_test.go +++ b/projection_test.go @@ -2,7 +2,9 @@ package dalgo2sql import ( "context" + "database/sql" "reflect" + "strings" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -58,9 +60,10 @@ func TestWildcardExclusionRecordsReader(t *testing.T) { q := dal.From(dal.NewRootCollectionRef("customers", "c")).NewQuery(). SelectColumns(dal.AllColumnsExceptFrom("c", "email", "password_hash", "email")) - mock.ExpectQuery("SELECT `c`.* FROM `customers` AS `c`"). - WillReturnRows(sqlmock.NewRows([]string{"id", "name", "email", "created_at"}). - AddRow("c1", "Ada", "ada@example.test", "2026-09-20")) + expectSQLiteSourceColumns(mock, "customers", "id", "name", "email", "created_at") + mock.ExpectQuery("SELECT `id`, `name`, `created_at` FROM `customers` AS `c`"). + WillReturnRows(sqlmock.NewRows([]string{"id", "name", "created_at"}). + AddRow("c1", "Ada", "2026-09-20")) r, err := getRecordsReaderWithOptions(context.Background(), q, db.QueryContext, DbOptions{ StructuredQueryDialect: "sqlite", @@ -97,9 +100,10 @@ func TestWildcardExclusionMasksRecordsReader(t *testing.T) { q := dal.From(dal.NewRootCollectionRef("customers", "c")).NewQuery(). SelectColumns(dal.AllColumnsExceptFrom("c", "Billing*", "Password*", "missing*")) - mock.ExpectQuery("SELECT `c`.* FROM `customers` AS `c`"). - WillReturnRows(sqlmock.NewRows([]string{"id", "name", "billing_address", "PASSWORD_HASH", "created_at"}). - AddRow("c1", "Ada", "1 Main St", "secret", "2026-09-20")) + expectSQLiteSourceColumns(mock, "customers", "id", "name", "billing_address", "PASSWORD_HASH", "created_at") + mock.ExpectQuery("SELECT `id`, `name`, `created_at` FROM `customers` AS `c`"). + WillReturnRows(sqlmock.NewRows([]string{"id", "name", "created_at"}). + AddRow("c1", "Ada", "2026-09-20")) r, err := getRecordsReaderWithOptions(context.Background(), q, db.QueryContext, DbOptions{ StructuredQueryDialect: "sqlite", @@ -136,9 +140,10 @@ func TestWildcardExclusionRecordsetReaderPreservesOrder(t *testing.T) { q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery(). SelectColumns(dal.AllColumnsExcept("email", "missing")) - mock.ExpectQuery("SELECT * FROM `customers`"). - WillReturnRows(sqlmock.NewRows([]string{"id", "name", "email", "created_at"}). - AddRow([]byte("c1"), []byte("Ada"), []byte("ada@example.test"), []byte("2026-09-20"))) + expectSQLiteSourceColumns(mock, "customers", "id", "name", "email", "created_at") + mock.ExpectQuery("SELECT `id`, `name`, `created_at` FROM `customers`"). + WillReturnRows(sqlmock.NewRows([]string{"id", "name", "created_at"}). + AddRow([]byte("c1"), []byte("Ada"), []byte("2026-09-20"))) r, err := getRecordsetReaderWithDialect(context.Background(), q, db.QueryContext, "sqlite") if err != nil { @@ -175,9 +180,10 @@ func TestWildcardExclusionMasksRecordsetReaderPreservesOrder(t *testing.T) { q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery(). SelectColumns(dal.AllColumnsExcept("Billing*", "Password*", "missing*")) - mock.ExpectQuery("SELECT * FROM `customers`"). - WillReturnRows(sqlmock.NewRows([]string{"id", "name", "BillingEmail", "password_hash", "created_at"}). - AddRow([]byte("c1"), []byte("Ada"), []byte("ada@example.test"), []byte("secret"), []byte("2026-09-20"))) + expectSQLiteSourceColumns(mock, "customers", "id", "name", "BillingEmail", "password_hash", "created_at") + mock.ExpectQuery("SELECT `id`, `name`, `created_at` FROM `customers`"). + WillReturnRows(sqlmock.NewRows([]string{"id", "name", "created_at"}). + AddRow([]byte("c1"), []byte("Ada"), []byte("2026-09-20"))) r, err := getRecordsetReaderWithDialect(context.Background(), q, db.QueryContext, "sqlite") if err != nil { @@ -212,6 +218,15 @@ func valueString(value any) string { return value.(string) } +func expectSQLiteSourceColumns(mock sqlmock.Sqlmock, table string, names ...string) { + rows := sqlmock.NewRows([]string{"name", "hidden"}) + for _, name := range names { + rows.AddRow(name, 0) + } + mock.ExpectQuery("SELECT name, hidden FROM pragma_table_xinfo(?) ORDER BY cid"). + WithArgs(table).WillReturnRows(rows) +} + func TestWildcardProjectionValidation(t *testing.T) { base := dal.From(dal.NewRootCollectionRef("customers", "c")).NewQuery() tests := []struct { @@ -281,6 +296,130 @@ func TestWildcardProjectionMaskKeepsExplicitTail(t *testing.T) { } } +func TestSQLiteWildcardMaskPushdownRealDatabase(t *testing.T) { + db := openTestSQLiteDB(t, "CREATE TABLE customers (id TEXT, name TEXT, BillingSecret BLOB, PasswordHash TEXT)") + if _, err := db.Exec("INSERT INTO customers VALUES (?, ?, ?, ?)", "c1", "Ada", []byte("large secret"), "hash"); err != nil { + t.Fatal(err) + } + q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery().SelectColumns( + dal.AllColumnsExcept("Billing*", "Password*"), + dal.Column{Expression: dal.Field("name"), Alias: "BillingDisplay"}, + ) + queries := make([]string, 0, 2) + execute := func(ctx context.Context, text string, args ...any) (*sql.Rows, error) { + queries = append(queries, text) + return db.QueryContext(ctx, text, args...) + } + r, err := getRecordsetReaderWithDialect(context.Background(), q, execute, "sqlite") + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + if len(queries) != 2 || queries[0] != "SELECT name, hidden FROM pragma_table_xinfo(?) ORDER BY cid" || + queries[1] != "SELECT `id`, `name`, `name` AS `BillingDisplay` FROM `customers`" { + t.Fatalf("queries = %#v", queries) + } + if strings.Contains(queries[1], "BillingSecret") || strings.Contains(queries[1], "PasswordHash") { + t.Fatalf("excluded columns remain in data query: %s", queries[1]) + } + if got := r.Recordset().Columns(); len(got) != 3 || got[2].Name() != "BillingDisplay" { + t.Fatalf("visible columns = %#v", got) + } + if _, _, err := r.Next(); err != nil { + t.Fatal(err) + } +} + +func TestSQLiteWildcardPushdownUsesSchemaNamesWithFullColumnLabels(t *testing.T) { + db := openTestSQLiteDB(t, "CREATE TABLE customers (id TEXT, BillingSecret TEXT, computed TEXT GENERATED ALWAYS AS (id || '-computed') VIRTUAL)") + db.SetMaxOpenConns(1) + if _, err := db.Exec("PRAGMA short_column_names=OFF; PRAGMA full_column_names=ON"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("INSERT INTO customers (id, BillingSecret) VALUES ('c1', 'secret')"); err != nil { + t.Fatal(err) + } + q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery().SelectColumns(dal.AllColumnsExcept("Billing*")) + var dataQuery string + execute := func(ctx context.Context, text string, args ...any) (*sql.Rows, error) { + if !strings.Contains(text, "pragma_table_xinfo") { + dataQuery = text + } + return db.QueryContext(ctx, text, args...) + } + r, err := getReaderBaseWithDialect(context.Background(), q, execute, "sqlite") + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.rows.Close() }() + if dataQuery != "SELECT `id`, `computed` FROM `customers`" { + t.Fatalf("data query = %q", dataQuery) + } + if !r.rows.Next() { + t.Fatal("expected one row") + } + values, err := r.scanValues() + if err != nil || len(values) != 2 || valueString(values[0]) != "c1" || valueString(values[1]) != "c1-computed" { + t.Fatalf("values = %#v, err = %v", values, err) + } +} + +func TestSQLiteSourceColumnsOmitOnlyVirtualTableHiddenColumns(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Fatal(err) + } + defer closeDatabase(t, db) + q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery().SelectColumns(dal.AllColumnsExcept("Billing*")) + mock.ExpectQuery("SELECT name, hidden FROM pragma_table_xinfo(?) ORDER BY cid").WithArgs("customers"). + WillReturnRows(sqlmock.NewRows([]string{"name", "hidden"}). + AddRow("id", 0).AddRow("internal", 1).AddRow("virtual_generated", 2).AddRow("stored_generated", 3)) + names, err := sqliteSourceColumns(context.Background(), q, db.QueryContext) + if err != nil || !reflect.DeepEqual(names, []string{"id", "virtual_generated", "stored_generated"}) { + t.Fatalf("source columns = %#v, err = %v", names, err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestSQLiteWildcardAllExcludedRetainsEmptyResultFallback(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + if err != nil { + t.Fatal(err) + } + defer closeDatabase(t, db) + q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery().SelectColumns(dal.AllColumnsExcept("*")) + expectSQLiteSourceColumns(mock, "customers", "id", "secret") + mock.ExpectQuery("SELECT * FROM `customers`").WillReturnRows(sqlmock.NewRows([]string{"id", "secret"}).AddRow("c1", "value")) + r, err := getReaderBaseWithDialect(context.Background(), q, db.QueryContext, "sqlite") + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.rows.Close() }() + if len(r.colNames) != 0 || !r.rows.Next() { + t.Fatalf("expected one row with zero visible columns, got %#v", r.colNames) + } + values, err := r.scanValues() + if err != nil || len(values) != 0 { + t.Fatalf("values = %#v, err = %v", values, err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func TestSQLiteWildcardDuplicateMetadataNamesRetainResultFiltering(t *testing.T) { + q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery().SelectColumns(dal.AllColumnsExcept("secret")) + plan, err := planWildcardProjection(q) + if err != nil { + t.Fatal(err) + } + if _, ok := plan.expandSQLiteWildcard(q, []string{"id", "ID", "secret"}); ok { + t.Fatal("ambiguous source column names must not be expanded") + } +} + func TestWildcardExclusionIdentityHelperDoesNotHideSameNamedSourceColumn(t *testing.T) { db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) if err != nil { @@ -290,9 +429,10 @@ func TestWildcardExclusionIdentityHelperDoesNotHideSameNamedSourceColumn(t *test q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery(). SelectColumns(dal.AllColumnsExcept("id")) - mock.ExpectQuery("SELECT *, `id` AS `__dalgo_record_id` FROM `customers`"). - WillReturnRows(sqlmock.NewRows([]string{"id", "__dalgo_record_id", "__dalgo_record_id"}). - AddRow("c1", "source-value", "c1")) + expectSQLiteSourceColumns(mock, "customers", "id", "__dalgo_record_id") + mock.ExpectQuery("SELECT `__dalgo_record_id`, `id` AS `__dalgo_record_id` FROM `customers`"). + WillReturnRows(sqlmock.NewRows([]string{"__dalgo_record_id", "__dalgo_record_id"}). + AddRow("source-value", "c1")) r, err := getRecordsReaderWithOptions(context.Background(), q, db.QueryContext, DbOptions{ StructuredQueryDialect: "sqlite", @@ -329,9 +469,10 @@ func TestWildcardExclusionMaskIdentityHelperDoesNotHideSameNamedSourceColumn(t * q := dal.From(dal.NewRootCollectionRef("customers", "")).NewQuery(). SelectColumns(dal.AllColumnsExcept("ID*")) - mock.ExpectQuery("SELECT *, `id` AS `__dalgo_record_id` FROM `customers`"). - WillReturnRows(sqlmock.NewRows([]string{"id", "__dalgo_record_id", "__dalgo_record_id"}). - AddRow("c1", "source-value", "c1")) + expectSQLiteSourceColumns(mock, "customers", "id", "__dalgo_record_id") + mock.ExpectQuery("SELECT `__dalgo_record_id`, `id` AS `__dalgo_record_id` FROM `customers`"). + WillReturnRows(sqlmock.NewRows([]string{"__dalgo_record_id", "__dalgo_record_id"}). + AddRow("source-value", "c1")) r, err := getRecordsReaderWithOptions(context.Background(), q, db.QueryContext, DbOptions{ StructuredQueryDialect: "sqlite", diff --git a/reader_base.go b/reader_base.go index e82ff3d..f496581 100644 --- a/reader_base.go +++ b/reader_base.go @@ -62,6 +62,19 @@ func getReaderBaseWithDialect(ctx context.Context, query dal.Query, execute exec if err != nil { return readerBase{}, &access.DeniedError{Decision: access.Decision{Operation: access.Query, Code: access.CodeEnforcementUnsupported, Scope: access.DecisionScopeOperation, Explanation: "structured SQLite query is unsupported"}} } + if projection != nil { + var names []string + if names, err = sqliteSourceColumns(ctx, q, execute); err != nil { + return readerBase{}, fmt.Errorf("failed to inspect SQLite wildcard source: %w", err) + } + if expanded, ok := projection.expandSQLiteWildcard(q, names); ok { + text, a, err = compileStructuredSQL(expanded) + if err != nil { + return readerBase{}, fmt.Errorf("failed to compile expanded SQLite projection: %w", err) + } + projection = nil + } + } default: return readerBase{}, fmt.Errorf("unsupported structured query dialect %q", dialect) } @@ -105,6 +118,33 @@ func getReaderBaseWithDialect(ctx context.Context, query dal.Query, execute exec return rb, nil } +func sqliteSourceColumns(ctx context.Context, q dal.StructuredQuery, execute executeQueryFunc) ([]string, error) { + source := q.From().Base() + // table_xinfo gives the actual identifiers, not SELECT * result labels, + // which SQLite may prefix when full_column_names is enabled. It also + // includes generated columns (hidden 2/3), which SELECT * returns. + rows, err := execute(ctx, "SELECT name, hidden FROM pragma_table_xinfo(?) ORDER BY cid", source.Name()) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + var names []string + for rows.Next() { + var name string + var hidden int + if err := rows.Scan(&name, &hidden); err != nil { + return nil, err + } + if hidden == 0 || hidden == 2 || hidden == 3 { // virtual-table hidden columns (1) are absent from SELECT * + names = append(names, name) + } + } + if err := rows.Err(); err != nil { + return nil, err + } + return names, nil +} + func (rb readerBase) scanValues() (values []any, err error) { rawValues := make([]any, len(rb.scanColNames)) scanArgs := make([]any, len(rb.scanColNames))