Skip to content

Commit 67bcab3

Browse files
authored
Merge pull request #165 from Joyful12-tech/feat/125-immutable-migration-tooling
feat(db): establish immutable migration tooling for PostgreSQL/TimescaleDB
2 parents ade2c61 + f89014c commit 67bcab3

10 files changed

Lines changed: 460 additions & 67 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 38 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -103,61 +103,68 @@ jobs:
103103
working-directory: zk
104104
run: npm test
105105

106-
zk-tests:
107-
name: ZK Circuit Tests
106+
migration-dry-run:
107+
name: Migration Dry-Run
108108
runs-on: ubuntu-latest
109+
env:
110+
# Custom runner reads TIMESCALEDB_* env vars; point them at the ephemeral
111+
# timescaledb service container spun up for this job.
112+
TIMESCALEDB_HOST: localhost
113+
TIMESCALEDB_PORT: 5432
114+
TIMESCALEDB_DATABASE: pactum_migration_test
115+
TIMESCALEDB_USER: postgres
116+
TIMESCALEDB_PASSWORD: postgres
117+
services:
118+
postgres:
119+
# Same image docker-compose.yml uses for the real service, so
120+
# CREATE EXTENSION timescaledb (001_timescale_setup.sql) succeeds here
121+
# instead of only in the full docker-compose e2e stack.
122+
image: timescale/timescaledb:latest-pg16
123+
env:
124+
POSTGRES_PASSWORD: postgres
125+
POSTGRES_DB: pactum_migration_test
126+
ports:
127+
- 5432:5432
128+
options: >-
129+
--health-cmd "pg_isready -U postgres -d pactum_migration_test"
130+
--health-interval 10s
131+
--health-timeout 5s
132+
--health-retries 5
109133
110134
steps:
111135
- name: Checkout repository
112136
uses: actions/checkout@v4
113137

114-
# Node 22.18+ runs the package's TypeScript sources directly, so there is no
115-
# build step to keep in sync with the tests.
116-
- name: Set up Node.js 22.x
138+
- name: Set up Node.js 22.x (LTS)
117139
uses: actions/setup-node@v4
118140
with:
119141
node-version: '22.x'
120142

121-
- name: Install Rust stable toolchain
122-
uses: dtolnay/rust-toolchain@stable
123-
124-
# circom is not published to npm, so it is built from source and cached — the
125-
# circuit tests skip without it, and a skipped circuit test proves nothing.
126-
- name: Cache circom binary
127-
id: cache_circom
128-
uses: actions/cache@v4
129-
with:
130-
path: ~/.cargo/bin/circom
131-
key: ${{ runner.os }}-circom-2.2.3
132-
133-
- name: Build circom from source
134-
if: steps.cache_circom.outputs.cache-hit != 'true'
135-
run: cargo install --locked --git https://github.com/iden3/circom.git --tag v2.2.3 circom
136-
137143
- name: Cache npm dependencies
138144
uses: actions/cache@v4
139145
with:
140146
path: ~/.npm
141-
key: ${{ runner.os }}-node-zk-${{ hashFiles('zk/package-lock.json', 'zk/package.json') }}
147+
key: ${{ runner.os }}-node-${{ hashFiles('backend/package-lock.json', 'backend/package.json') }}
142148
restore-keys: |
143-
${{ runner.os }}-node-zk-
149+
${{ runner.os }}-node-
144150
145-
- name: Install zk dependencies
146-
working-directory: zk
151+
- name: Install backend dependencies
152+
working-directory: backend
147153
run: |
148154
if [ -f package-lock.json ]; then
149155
npm ci
150156
else
151157
npm install
152158
fi
153159
154-
- name: Compile circuit and run the Groth16 setup
155-
working-directory: zk
156-
run: npm run build:circuit
160+
- name: Build backend
161+
working-directory: backend
162+
run: npm run build
163+
164+
- name: Run migration dry-run against ephemeral TimescaleDB
165+
working-directory: backend
166+
run: node dist/db/migrate-dry-run.js
157167

158-
- name: Run zk test suite
159-
working-directory: zk
160-
run: npm test
161168

162169
backend-tests:
163170
name: Backend Tests

‎README.md‎

Lines changed: 69 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ Pactum lets any two parties register a real-world commitment on Stellar — a re
1313

1414
## The problem
1515

16-
Escrow contracts handle *one-time* deals — money goes in, money comes out once conditions are met. But a lot of real commitments aren't one-time payments at all:
16+
Escrow contracts handle _one-time_ deals — money goes in, money comes out once conditions are met. But a lot of real commitments aren't one-time payments at all:
1717

1818
- A landlord promising to return a deposit within 30 days of move-out
1919
- An API provider guaranteeing 99.9% uptime this quarter
@@ -81,43 +81,43 @@ See [`docs/architecture.md`](./docs/architecture.md) for the full breakdown.
8181

8282
## Tech stack
8383

84-
| Layer | Technology |
85-
|---|---|
86-
| Smart contract | Rust + Soroban SDK |
87-
| Contract network | Stellar Testnet |
88-
| Backend API | Node.js + TypeScript + Express |
89-
| Indexer | Soroban RPC event listener |
90-
| Database | PostgreSQL + TimescaleDB (time-series analytics) |
91-
| Cache | Redis (optional read cache for reputation lookups) |
92-
| SDK | TypeScript, published as `@pactum/sdk` |
93-
| ZK proofs | Circom 2 + snarkjs (Groth16 over BN254) |
94-
| Testing | Cargo test (contract) · Jest (backend) · `node --test` (zk) |
95-
| CI/CD | GitHub Actions |
84+
| Layer | Technology |
85+
| ---------------- | ----------------------------------------------------------- |
86+
| Smart contract | Rust + Soroban SDK |
87+
| Contract network | Stellar Testnet |
88+
| Backend API | Node.js + TypeScript + Express |
89+
| Indexer | Soroban RPC event listener |
90+
| Database | PostgreSQL + TimescaleDB (time-series analytics) |
91+
| Cache | Redis (optional read cache for reputation lookups) |
92+
| SDK | TypeScript, published as `@pactum/sdk` |
93+
| ZK proofs | Circom 2 + snarkjs (Groth16 over BN254) |
94+
| Testing | Cargo test (contract) · Jest (backend) · `node --test` (zk) |
95+
| CI/CD | GitHub Actions |
9696

9797
---
9898

9999
## Contract interface (early draft)
100100

101-
| Method | Kind | Description |
102-
|---|---|---|
103-
| `create_commitment(issuer, counterparty, terms_hash, due_at)` | write | Register a new commitment between two addresses |
104-
| `create_milestone_commitment(issuer, counterparty, terms_hash, due_at, resolver, milestone_count)` | write | Register a commitment fulfilled across several milestones |
105-
| `attest(commitment_id, outcome)` | write | Mark the next pending milestone fulfilled, late, or breached — resolving the commitment if it is the last one |
106-
| `attest_milestone(commitment_id, milestone_index, outcome)` | write | Mark one milestone of a commitment; the commitment resolves on the last one |
107-
| `get_milestone(commitment_id, milestone_index)` | read | Fetch a single milestone's outcome, or nothing while it is pending |
108-
| `dispute(commitment_id, reason)` | write | Flag a commitment as contested rather than resolved |
109-
| `resolve_dispute(commitment_id, outcome)` | write | Designated arbitrator/oracle settles a disputed commitment |
110-
| `get_commitment(commitment_id)` | read | Fetch a single commitment's details and status |
111-
| `get_reputation(address)` | read | Aggregate fulfilled / late / breached counts for an address |
101+
| Method | Kind | Description |
102+
| -------------------------------------------------------------------------------------------------- | ----- | ------------------------------------------------------------------------------------------------------------- |
103+
| `create_commitment(issuer, counterparty, terms_hash, due_at)` | write | Register a new commitment between two addresses |
104+
| `create_milestone_commitment(issuer, counterparty, terms_hash, due_at, resolver, milestone_count)` | write | Register a commitment fulfilled across several milestones |
105+
| `attest(commitment_id, outcome)` | write | Mark the next pending milestone fulfilled, late, or breached — resolving the commitment if it is the last one |
106+
| `attest_milestone(commitment_id, milestone_index, outcome)` | write | Mark one milestone of a commitment; the commitment resolves on the last one |
107+
| `get_milestone(commitment_id, milestone_index)` | read | Fetch a single milestone's outcome, or nothing while it is pending |
108+
| `dispute(commitment_id, reason)` | write | Flag a commitment as contested rather than resolved |
109+
| `resolve_dispute(commitment_id, outcome)` | write | Designated arbitrator/oracle settles a disputed commitment |
110+
| `get_commitment(commitment_id)` | read | Fetch a single commitment's details and status |
111+
| `get_reputation(address)` | read | Aggregate fulfilled / late / breached counts for an address |
112112

113113
Full spec lives in [`docs/contract-reference.md`](./docs/contract-reference.md) as the contract develops.
114114

115115
## Production Deployment
116116

117117
The contract is currently deployed on the Stellar Testnet for testing and integration.
118118

119-
| Contract | Network | Contract ID | Explorer |
120-
|----------|---------|-------------|----------|
119+
| Contract | Network | Contract ID | Explorer |
120+
| -------- | ------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- |
121121
| Registry | Testnet | `CBADTVTJ6IN332HIKZ7LWUYMYTLPZYCEBV3X2HS47VHR5UDBHQ3GAA7E` | [Stellar Expert](https://stellar.expert/explorer/testnet/contract/CBADTVTJ6IN332HIKZ7LWUYMYTLPZYCEBV3X2HS47VHR5UDBHQ3GAA7E) |
122122

123123
### Example Transactions
@@ -153,13 +153,50 @@ cd ../backend
153153
npm install
154154
cp .env.example .env # fill in DATABASE_URL, SOROBAN_RPC_URL, TIMESCALEDB_* config, etc.
155155
npm run build
156-
npm run migrate:timescale # Run TimescaleDB migrations
156+
npm run migrate:up # Run all pending migrations
157157
npm run dev
158158

159159
# 5. Start analytics worker (optional, for background data processing)
160160
npm run analytics:worker
161161
```
162162

163+
### Database migrations
164+
165+
Pactum uses a custom immutable migration runner built into the backend. Every migration file is content-addressed with a SHA-256 checksum frozen on first apply — editing an already-applied file is detected and rejected at startup, preventing silent schema drift between environments.
166+
167+
Migration files live in `backend/src/db/migrations/` and are numbered with a three-digit prefix (`001_`, `002_`, …) so they execute in a deterministic lexicographic order.
168+
169+
| NPM script | What it does |
170+
| --------------------------------------- | ----------------------------------------------------------- |
171+
| `npm run migrate:up` | Build the backend and apply all pending migrations |
172+
| `npm run migrate:make -- <description>` | Create a new numbered migration stub |
173+
| `npm run migrate:status` | Print which migrations are applied, pending, or tampered |
174+
| `npm run migrate:dry-run` | Validate checksums and SQL syntax without writing to the DB |
175+
176+
#### Day-to-day workflow
177+
178+
```bash
179+
# Check what is currently applied
180+
cd backend
181+
npm run migrate:status
182+
183+
# Create a new migration
184+
npm run migrate:make -- add_webhook_deliveries_table
185+
# → writes backend/src/db/migrations/008_add_webhook_deliveries_table.sql
186+
187+
# Edit the generated stub, then apply
188+
npm run migrate:up
189+
190+
# Validate all migrations without touching the database (also runs in CI)
191+
npm run migrate:dry-run
192+
```
193+
194+
#### Immutability rules
195+
196+
- **Never edit a migration that has already been applied to any environment.** The runner stores the SHA-256 of each file at apply time and compares it on every subsequent run. A mismatch aborts the migration cycle and exits non-zero.
197+
- To change something that was already applied, create a new migration that issues the corrective `ALTER`, `DROP`, or `CREATE` statement.
198+
- The `migrate:dry-run` script is run automatically on every pull request by CI (see `.github/workflows/ci.yml` → `migration-dry-run` job).
199+
163200
### Running the whole stack with Docker
164201

165202
**Prerequisites:** Docker with Compose v2.
@@ -170,11 +207,11 @@ docker compose up --build
170207

171208
That boots TimescaleDB, the backend (which applies `backend/src/db/migrations/*.sql` on startup) and an nginx-served frontend build.
172209

173-
| Service | URL | Override |
174-
|---|---|---|
175-
| Frontend | http://localhost | `FRONTEND_PORT` |
176-
| Backend | http://localhost:3000 | `BACKEND_PORT` |
177-
| TimescaleDB | `localhost:5432` | `TIMESCALEDB_PORT` |
210+
| Service | URL | Override |
211+
| ----------- | --------------------- | ------------------ |
212+
| Frontend | http://localhost | `FRONTEND_PORT` |
213+
| Backend | http://localhost:3000 | `BACKEND_PORT` |
214+
| TimescaleDB | `localhost:5432` | `TIMESCALEDB_PORT` |
178215

179216
The frontend is built to call the API on its own origin, and nginx proxies `/api`, `/reputation`, `/commitments` and `/health` to the backend container. Database credentials and Soroban settings come from the same `TIMESCALEDB_*` / `SOROBAN_*` variables as `backend/.env.example`; set them in a root `.env` to override the defaults.
180217

‎backend/package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@
1717
"indexer": "node dist/workers/indexer.js",
1818
"analytics:worker": "node dist/workers/analyticsWorker.js",
1919
"migrate:timescale": "node dist/db/migrate.js",
20+
"migrate:up": "npm run build && node dist/db/migrate.js",
21+
"migrate:make": "node scripts/migrate-make.js",
22+
"migrate:status": "npm run build && node dist/db/migrate-status.js",
23+
"migrate:dry-run": "npm run build && node dist/db/migrate-dry-run.js",
2024
"backfill:commitments": "node dist/indexer/backfill-commitments.js"
2125
},
2226
"keywords": [],

‎backend/scripts/migrate-make.js‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/usr/bin/env node
2+
/**
3+
* migrate-make.js — create a new numbered migration file.
4+
*
5+
* Usage:
6+
* npm run migrate:make -- <description>
7+
*
8+
* Example:
9+
* npm run migrate:make -- add_webhook_deliveries_table
10+
*
11+
* The script finds the highest existing migration number, increments it by 1,
12+
* and writes an empty SQL stub:
13+
*
14+
* backend/src/db/migrations/006_add_webhook_deliveries_table.sql
15+
*
16+
* Rules enforced here intentionally mirror those in timescale.ts:
17+
* - Files are sorted lexicographically, so zero-padded three-digit prefixes
18+
* are required for correct ordering up to migration 999.
19+
* - Each file's checksum is frozen on first run; never edit an applied file.
20+
*/
21+
22+
// eslint-disable-next-line @typescript-eslint/no-require-imports
23+
const fs = require('fs');
24+
// eslint-disable-next-line @typescript-eslint/no-require-imports
25+
const path = require('path');
26+
27+
const MIGRATIONS_DIR = path.join(__dirname, '..', 'src', 'db', 'migrations');
28+
29+
function main() {
30+
const args = process.argv.slice(2);
31+
const raw = args.join('_').trim();
32+
33+
if (!raw) {
34+
console.error('Usage: npm run migrate:make -- <description>');
35+
console.error('Example: npm run migrate:make -- add_webhook_deliveries_table');
36+
process.exit(1);
37+
}
38+
39+
// Sanitise: lowercase, replace spaces/hyphens with underscores, strip non-word chars
40+
const description = raw
41+
.toLowerCase()
42+
.replace(/[\s-]+/g, '_')
43+
.replace(/[^\w]/g, '');
44+
45+
if (!description) {
46+
console.error('Error: description must contain at least one alphanumeric character.');
47+
process.exit(1);
48+
}
49+
50+
// Find the highest existing sequence number
51+
const existing = fs
52+
.readdirSync(MIGRATIONS_DIR)
53+
.filter((f) => /^\d{3}_.*\.sql$/.test(f))
54+
.map((f) => parseInt(f.slice(0, 3), 10));
55+
56+
const next = existing.length > 0 ? Math.max(...existing) + 1 : 1;
57+
const prefix = String(next).padStart(3, '0');
58+
const filename = `${prefix}_${description}.sql`;
59+
const fullPath = path.join(MIGRATIONS_DIR, filename);
60+
61+
if (fs.existsSync(fullPath)) {
62+
console.error(`Error: migration file already exists: ${filename}`);
63+
process.exit(1);
64+
}
65+
66+
const timestamp = new Date().toISOString();
67+
const template = `-- Migration ${prefix}: ${description}
68+
-- Created: ${timestamp}
69+
--
70+
-- IMPORTANT: Once this migration has been applied to any environment its
71+
-- checksum is frozen. Do NOT edit this file; create a new migration instead.
72+
--
73+
-- Write your SQL below this line.
74+
`;
75+
76+
fs.writeFileSync(fullPath, template, 'utf8');
77+
console.log(`Created migration: src/db/migrations/${filename}`);
78+
}
79+
80+
main();

‎backend/src/db/migrate-dry-run.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import pool, { dryRunMigrations } from './timescale';
2+
3+
// Validates all migration files (checksum integrity + SQL syntax) against an
4+
// ephemeral database without making any persistent changes. Used by CI to
5+
// gate PRs that touch migration files.
6+
//
7+
// Exit codes:
8+
// 0 — all migrations are valid (applied checksums match, pending SQL is clean)
9+
// 1 — at least one migration is tampered or contains a syntax error
10+
dryRunMigrations()
11+
.then(async (report) => {
12+
let hasError = false;
13+
14+
for (const entry of report) {
15+
const icon =
16+
entry.status === 'applied'
17+
? '✓'
18+
: entry.status === 'pending'
19+
? '⧖'
20+
: entry.status === 'tampered'
21+
? '✗'
22+
: '✗';
23+
24+
const detail = entry.message ? ` — ${entry.message}` : '';
25+
console.log(`[migrate:dry-run] ${icon} ${entry.file} (${entry.status})${detail}`);
26+
27+
if (entry.status === 'tampered' || entry.status === 'error') {
28+
hasError = true;
29+
}
30+
}
31+
32+
if (hasError) {
33+
console.error('\n[migrate:dry-run] FAILED — fix the errors above before merging.');
34+
} else {
35+
console.log(`\n[migrate:dry-run] OK — ${report.length} migration(s) validated successfully.`);
36+
}
37+
38+
await pool.end();
39+
process.exit(hasError ? 1 : 0);
40+
})
41+
.catch(async (err) => {
42+
console.error('[migrate:dry-run] Error:', err);
43+
try {
44+
await pool.end();
45+
} catch {
46+
// ignore teardown errors
47+
}
48+
process.exit(1);
49+
});

0 commit comments

Comments
 (0)