docs: add sqlc scripts documentation for developers

.github/workflows/main.yml

@@ -63,6 +63,15 @@ jobs:
       - name: Check RPC format
         run: make rpc-check
 
+      - name: Check generated SQL code is up-to-date
+        run: make sqlc-check
+
+      - name: Check SQL formatting
+        run: make sql-format-check
+
+      - name: Check SQL linting
+        run: make sql-lint-check
+
       - name: compile code
         run: go install -v ./...
 
@@ -149,3 +158,40 @@ jobs:
         with:
           path-to-profile: coverage.txt
           parallel: true
+
+  ########################
+  # run integration tests (SQLite + Postgres)
+  ########################
+  itest-db:
+    name: integration tests (${{ matrix.db }}, ${{ matrix.target }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        db: [sqlite, postgres]
+        target: [itest-db, itest-db-race]
+    steps:
+      - name: git checkout
+        uses: actions/checkout@v5
+
+      - name: Clean up runner space
+        uses: ./.github/actions/cleanup-space
+
+      - name: go cache
+        uses: actions/cache@v4
+        with:
+          path: /home/runner/work/go
+          key: btcwallet-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            btcwallet-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}-${{ hashFiles('**/go.sum') }}
+            btcwallet-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}-
+            btcwallet-${{ runner.os }}-go-${{ env.GO_VERSION }}-
+            btcwallet-${{ runner.os }}-go-
+
+      - name: setup go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: '${{ env.GO_VERSION }}'
+
+      - name: run ${{ matrix.db }} ${{ matrix.target }}
+        run: make ${{ matrix.target }} db=${{ matrix.db }} verbose=1

Makefile

@@ -6,12 +6,23 @@ GOINSTALL := GO111MODULE=on go install -v
 
 GOFILES = $(shell find . -type f -name '*.go' -not -name "*.pb.go")
 
+
+# SQL directories.
+SQL_MIGRATIONS_DIR := wallet/internal/db/migrations
+SQL_QUERIES_DIR := wallet/internal/db/queries
+
+# SQL file paths.
+SQL_POSTGRES_MIGRATIONS := $(SQL_MIGRATIONS_DIR)/postgres
+SQL_POSTGRES_QUERIES := $(SQL_QUERIES_DIR)/postgres
+SQL_SQLITE_MIGRATIONS := $(SQL_MIGRATIONS_DIR)/sqlite
+SQL_SQLITE_QUERIES := $(SQL_QUERIES_DIR)/sqlite
+
 RM := rm -f
 CP := cp
 MAKE := make
 XARGS := xargs -L 1
 
-include make/testing_flags.mk
+include config/testing_flags.mk
 
 # Linting uses a lot of memory, so keep it under control by limiting the number
 # of workers if requested.
@@ -24,6 +35,13 @@ DOCKER_TOOLS = docker run \
   -v $(shell bash -c "mkdir -p /tmp/go-build-cache; echo /tmp/go-build-cache"):/root/.cache/go-build \
   -v $$(pwd):/build btcwallet-tools
 
+SQLFLUFF = docker run \
+	--rm \
+	--user $$(id -u):$$(id -g) \
+    -v $$(pwd):/sql \
+    -w /sql \
+    sqlfluff/sqlfluff
+
 GREEN := "\\033[0;32m"
 NC := "\\033[0m"
 define print
@@ -84,6 +102,24 @@ unit-bench:
 	@$(call print, "Running benchmark tests.")
 	$(UNIT_BENCH)
 
+#? itest-db: Run integration tests for wallet database
+itest-db:
+	@if [ -z "$(IT_DB_LABEL)" ]; then \
+		echo "Unknown integration test database '$(db)'. Use db=sqlite or db=postgres." ; \
+		exit 1 ; \
+	fi
+	@$(call print, "Running $(IT_DB_LABEL) integration tests.")
+	$(ITEST_DB)
+
+#? itest-db-race: Run integration tests for wallet database with race detector
+itest-db-race:
+	@if [ -z "$(IT_DB_LABEL)" ]; then \
+		echo "Unknown integration test database '$(db)'. Use db=sqlite or db=postgres." ; \
+		exit 1 ; \
+	fi
+	@$(call print, "Running $(IT_DB_LABEL) integration tests (race).")
+	env CGO_ENABLED=1 GORACE="history_size=7 halt_on_errors=1" $(ITEST_DB_RACE)
+
 # =========
 # UTILITIES
 # =========
@@ -108,17 +144,17 @@ rpc-format:
 #? lint-config-check: Verify golangci-lint configuration
 lint-config-check: docker-tools
 	@$(call print, "Verifying golangci-lint configuration.")
-	$(DOCKER_TOOLS) golangci-lint config verify -v
+	$(DOCKER_TOOLS) golangci-lint config verify -v --config config/.golangci.yml
 
 #? lint: Lint source and check errors
 lint-check: lint-config-check
 	@$(call print, "Linting source.")
-	$(DOCKER_TOOLS) golangci-lint run -v $(LINT_WORKERS)
+	$(DOCKER_TOOLS) golangci-lint run -v --config config/.golangci.yml $(LINT_WORKERS)
 
 #? lint: Lint source and fix
 lint: lint-config-check
 	@$(call print, "Linting source.")
-	$(DOCKER_TOOLS) golangci-lint run -v --fix $(LINT_WORKERS)
+	$(DOCKER_TOOLS) golangci-lint run -v --fix --config config/.golangci.yml $(LINT_WORKERS)
 
 #? docker-tools: Build tools docker image
 docker-tools:
@@ -138,7 +174,7 @@ rpc-check: rpc
 #? protolint: Lint proto files using protolint
 protolint:
 	@$(call print, "Linting proto files.")
-	docker run --rm --volume "$$(pwd):/workspace" --workdir /workspace yoheimuta/protolint lint rpc/
+	$(DOCKER_TOOLS) protolint lint -config_dir_path=config rpc/
 
 #? sample-conf-check: Make sure default values in the sample-btcwallet.conf file are set correctly
 sample-conf-check: install
@@ -159,6 +195,65 @@ tidy-module:
 tidy-module-check: tidy-module
 	if test -n "$$(git status --porcelain)"; then echo "modules not updated, please run `make tidy-module` again!"; git status; exit 1; fi
 
+#? sql-parse: Ensures SQL files are syntactically valid
+sql-parse:
+	@$(call print, "Validating SQL files (postgres migrations).")
+	$(SQLFLUFF) parse --config /sql/config/sqlfluff.cfg --dialect postgres $(SQL_POSTGRES_MIGRATIONS)
+	@$(call print, "Validating SQL files (postgres queries).")
+	$(SQLFLUFF) parse --config /sql/config/sqlfluff.cfg --dialect postgres $(SQL_POSTGRES_QUERIES)
+	@$(call print, "Validating SQL files (sqlite migrations).")
+	$(SQLFLUFF) parse --config /sql/config/sqlfluff.cfg --dialect sqlite $(SQL_SQLITE_MIGRATIONS)
+	@$(call print, "Validating SQL files (sqlite queries).")
+	$(SQLFLUFF) parse --config /sql/config/sqlfluff.cfg --dialect sqlite $(SQL_SQLITE_QUERIES)
+
+#? sqlc: Generate Go code from SQL queries and migrations
+sqlc: sql-parse docker-tools
+	@$(call print, "Generating sql models and queries in Go")
+	$(DOCKER_TOOLS) sqlc generate -f config/sqlc.yaml
+
+#? sqlc-check: Verify generated Go SQL queries and migrations are up-to-date
+sqlc-check: sqlc
+	@$(call print, "Verifying sql code generation.")
+	if test -n "$$(git status --porcelain '*.go')"; then echo "SQL models not properly generated!"; git status --porcelain '*.go'; exit 1; fi
+
+#? sql-format: Format SQL migration and query files (like 'make fmt')
+sql-format:
+	@$(call print, "Formatting SQL files (postgres migrations).")
+	$(SQLFLUFF) format --config /sql/config/sqlfluff.cfg --dialect postgres $(SQL_POSTGRES_MIGRATIONS)
+	@$(call print, "Formatting SQL files (postgres queries).")
+	$(SQLFLUFF) format --config /sql/config/sqlfluff.cfg --dialect postgres $(SQL_POSTGRES_QUERIES)
+	@$(call print, "Formatting SQL files (sqlite migrations).")
+	$(SQLFLUFF) format --config /sql/config/sqlfluff.cfg --dialect sqlite $(SQL_SQLITE_MIGRATIONS)
+	@$(call print, "Formatting SQL files (sqlite queries).")
+	$(SQLFLUFF) format --config /sql/config/sqlfluff.cfg --dialect sqlite $(SQL_SQLITE_QUERIES)
+
+#? sql-check: Verify SQL migration and query files are formatted correctly (like 'make fmt-check')
+sql-format-check: sql-format
+	@$(call print, "Checking SQL formatting.")
+	if test -n "$$(git status --porcelain '$(SQL_MIGRATIONS_DIR)/**/*.sql' '$(SQL_QUERIES_DIR)/**/*.sql')"; then echo "SQL files not formatted correctly, please run 'make sql-format' again!"; git status; git diff; exit 1; fi
+
+#? sql-lint: Lint SQL migration and query files and fix issues (like 'make lint')
+sql-lint:
+	@$(call print, "Linting SQL files (postgres migrations).")
+	$(SQLFLUFF) fix --config /sql/config/sqlfluff.cfg --dialect postgres $(SQL_POSTGRES_MIGRATIONS)
+	@$(call print, "Linting SQL files (postgres queries).")
+	$(SQLFLUFF) fix --config /sql/config/sqlfluff.cfg --dialect postgres $(SQL_POSTGRES_QUERIES)
+	@$(call print, "Linting SQL files (sqlite migrations).")
+	$(SQLFLUFF) fix --config /sql/config/sqlfluff.cfg --dialect sqlite $(SQL_SQLITE_MIGRATIONS)
+	@$(call print, "Linting SQL files (sqlite queries).")
+	$(SQLFLUFF) fix --config /sql/config/sqlfluff.cfg --dialect sqlite $(SQL_SQLITE_QUERIES)
+
+#? sql-lint-check: Lint SQL files and report errors (like 'make lint-check')
+sql-lint-check:
+	@$(call print, "Linting SQL files (postgres migrations).")
+	$(SQLFLUFF) lint --config /sql/config/sqlfluff.cfg --dialect postgres $(SQL_POSTGRES_MIGRATIONS)
+	@$(call print, "Linting SQL files (postgres queries).")
+	$(SQLFLUFF) lint --config /sql/config/sqlfluff.cfg --dialect postgres $(SQL_POSTGRES_QUERIES)
+	@$(call print, "Linting SQL files (sqlite migrations).")
+	$(SQLFLUFF) lint --config /sql/config/sqlfluff.cfg --dialect sqlite $(SQL_SQLITE_MIGRATIONS)
+	@$(call print, "Linting SQL files (sqlite queries).")
+	$(SQLFLUFF) lint --config /sql/config/sqlfluff.cfg --dialect sqlite $(SQL_SQLITE_QUERIES)
+
 .PHONY: all \
 	default \
 	build \
@@ -168,10 +263,19 @@ tidy-module-check: tidy-module
 	unit-race \
 	unit-debug \
 	unit-bench \
+	itest-db \
+	itest-db-race \
 	fmt \
 	fmt-check \
 	tidy-module \
 	tidy-module-check \
+	sql-parse \
+	sqlc \
+	sqlc-check \
+	sql-format \
+	sql-lint \
+	sql-lint-check \
+	sql-format-check \
 	rpc-format \
 	lint \
 	lint-config-check \

config/sqlc.yaml

@@ -0,0 +1,45 @@
+# sqlc configuration file for generating type-safe Go code from SQL.
+# For full documentation, see: https://docs.sqlc.dev/en/stable/reference/config.html
+version: "2"
+sql:
+  - engine: "postgresql"
+    schema: "../wallet/internal/db/migrations/postgres"
+    queries: "../wallet/internal/db/queries/postgres"
+    gen:
+      go:
+        out: "../wallet/internal/db/sqlc/postgres"
+        package: "sqlcpg"
+
+        # This is the driver package that sqlc will use in the generated code.
+        sql_package: database/sql
+
+        # Generate a `Querier` interface of all query methods. It's useful for
+        # mocking in tests.
+        emit_interface: true
+
+        # Export generated SQL statements so they're usable from other packages.
+        emit_exported_queries: true
+
+        # Generate prepared statements for better performance with database/sql.
+        emit_prepared_queries: true
+
+  - engine: "sqlite"
+    schema: "../wallet/internal/db/migrations/sqlite"
+    queries: "../wallet/internal/db/queries/sqlite"
+    gen:
+      go:
+        out: "../wallet/internal/db/sqlc/sqlite"
+        package: "sqlcsqlite"
+
+        # This is the driver package that sqlc will use in the generated code.
+        sql_package: database/sql
+
+        # Generate a `Querier` interface of all query methods. It's useful for
+        # mocking in tests.
+        emit_interface: true
+
+        # Export generated SQL statements so they're usable from other packages.
+        emit_exported_queries: true
+
+        # Generate prepared statements for better performance with database/sql.
+        emit_prepared_queries: true

config/sqlfluff.cfg

@@ -0,0 +1,41 @@
+# SQLFluff configuration
+# Source: https://docs.sqlfluff.com/en/stable/configuration/setting_configuration.html
+
+[sqlfluff]
+# Supported dialects: postgres, sqlite
+# We specify dialect via CLI (-d flag) to handle both postgres and sqlite
+# sql_file_exts = .sql
+# exclude_rules = None
+
+# Standard max line length
+max_line_length = 120
+
+# Use all available CPU cores for better performance
+processes = 0
+
+[sqlfluff:indentation]
+# Use spaces for indentation
+indent_unit = space
+tab_space_size = 4
+indented_joins = False
+indented_using_on = True
+
+[sqlfluff:rules:capitalisation.keywords]
+# Keywords should be uppercase (e.g., SELECT, FROM, WHERE)
+capitalisation_policy = upper
+
+[sqlfluff:rules:capitalisation.identifiers]
+# Table and column names should be lowercase
+extended_capitalisation_policy = lower
+
+[sqlfluff:rules:capitalisation.functions]
+# Function names should be lowercase
+extended_capitalisation_policy = lower
+
+[sqlfluff:rules:capitalisation.types]
+# Data types should be uppercase (e.g., INTEGER, BYTEA, BLOB)
+extended_capitalisation_policy = upper
+
+[sqlfluff:rules:convention.not_equal]
+# Prefer != over <> for not equal
+preferred_not_equal_style = c_style

make/testing_flags.mk → config/testing_flags.mk

@@ -1,5 +1,16 @@
 DEV_TAGS = dev
 LOG_TAGS =
+IT_TAGS ?=
+
+# Integration test DB selections derived from the `db` variable (sqlite, postgres).
+# Defaults to sqlite if not specified.
+ifeq ($(db),postgres)
+IT_TAGS += test_db_postgres
+IT_DB_LABEL := PostgreSQL
+else
+IT_TAGS :=
+IT_DB_LABEL := SQLite
+endif
 
 GOCC ?= go
 GOLIST := $(GOCC) list -tags="$(DEV_TAGS)" -deps $(PKG)/... | grep '$(PKG)'
@@ -39,6 +50,10 @@ ifneq ($(nocache),)
 TEST_FLAGS += -test.count=1
 endif
 
+ifneq ($(benchmem),)
+TEST_FLAGS += -test.benchmem
+endif
+
 # Define the log tags that will be applied only when running unit tests. If none
 # are provided, we default to "debug stdlog" which will be standard debug log
 # output.
@@ -61,7 +76,7 @@ UNIT_RACE := $(GOTEST) -tags="$(DEV_TAGS) $(LOG_TAGS)" $(TEST_FLAGS) -race $(UNI
 
 # NONE is a special value which selects no other tests but only executes the
 # benchmark tests here.
-UNIT_BENCH := $(GOTEST) -tags="$(DEV_TAGS) $(LOG_TAGS)" -test.bench=. -test.run=NONE $(UNITPKG)
+UNIT_BENCH := $(GOTEST) -tags="$(DEV_TAGS) $(LOG_TAGS)" $(TEST_FLAGS) -test.bench=. -test.run=NONE $(UNITPKG)
 endif
 
 ifeq ($(UNIT_TARGETED), no)
@@ -70,8 +85,11 @@ UNIT_DEBUG := $(GOLIST) | $(XARGS) env $(GOTEST) -v -tags="$(DEV_TAGS) $(LOG_TAG
 
 # NONE is a special value which selects no other tests but only executes the
 # benchmark tests here.
-UNIT_BENCH := $(GOLIST) | $(XARGS) env $(GOTEST) -tags="$(DEV_TAGS) $(LOG_TAGS)" -test.bench=. -test.run=NONE
+UNIT_BENCH := $(GOLIST) | $(XARGS) env $(GOTEST) -tags="$(DEV_TAGS) $(LOG_TAGS)" $(TEST_FLAGS) -test.bench=. -test.run=NONE
 UNIT_RACE := $(UNIT) -race
 endif
 
 UNIT_COVER := $(GOTEST) $(COVER_FLAGS) -tags="$(DEV_TAGS) $(LOG_TAGS)" $(TEST_FLAGS) $(COVER_PKG)
+
+ITEST_DB := $(GOTEST) -tags="itest $(DEV_TAGS) $(LOG_TAGS) $(IT_TAGS)" $(TEST_FLAGS) $(PKG)/wallet/internal/db/itest
+ITEST_DB_RACE := $(GOTEST) -race -tags="itest $(DEV_TAGS) $(LOG_TAGS) $(IT_TAGS)" $(TEST_FLAGS) $(PKG)/wallet/internal/db/itest

docs/developer/README.md

@@ -36,6 +36,14 @@ A deep dive into the core design philosophy, architectural patterns, and Go impl
 
 ---
 
+## 🗄️ SQLC Development Guide
+
+Guide for working with the SQLC code generation infrastructure, including how to write SQL queries and migrations for both PostgreSQL and SQLite backends.
+
+**[➡️ Read the SQLC Development Guide](./sqlc_development_guide.md)**
+
+---
+
 ## 📝 Architecture Decision Records (ADRs)
 
 Formal documentation of significant architectural decisions, their context, and consequences.