Skip to content

sql: retry connect failures while queries are waiting #32028

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alii merged 17 commits into from
Jun 10, 2026
Merged

sql: retry connect failures while queries are waiting #32028

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
5ac00d4
sql: report connect failures distinctly from closed connections
alii Jun 9, 2026
587336b
[autofix.ci] apply automated fixes
autofix-ci[bot] Jun 9, 2026
e4f9437
sql: retry connect failures while queries are waiting
alii Jun 9, 2026
d9ee4b7
docs: clarify which failures map to ERR_POSTGRES_CONNECTION_FAILED
robobun Jun 9, 2026
6cc03d1
sql: add a ConnectionFailed variant to AnyPostgresError
alii Jun 9, 2026
0119420
Merge branch 'ali/postgres-startup-connection-close-race' into ali/sq…
alii Jun 9, 2026
580f594
sql: fix close() deadlock and onclose spam in the connect-retry path
alii Jun 9, 2026
f1cd450
Merge remote-tracking branch 'origin/main' into ali/sql-retry-connect…
alii Jun 9, 2026
4a72176
sql: fail fast on refused connections, retry only pre-handshake closes
alii Jun 9, 2026
a0b2beb
[autofix.ci] apply automated fixes
autofix-ci[bot] Jun 9, 2026
e8ba216
test: cover mysql onclose-once and connectionTimeout: 0 retry semantics
alii Jun 9, 2026
89b5711
docs: split sql connection error descriptions into separate sentences
robobun Jun 9, 2026
f1c7168
Merge branch 'main' into ali/sql-retry-connect-failures
robobun Jun 10, 2026
7d706ea
ci: retrigger
robobun Jun 10, 2026
f831018
Merge branch 'main' into ali/sql-retry-connect-failures
alii Jun 10, 2026
735f41d
sql: hoist retry timer callbacks and type the connect-failure checks
robobun Jun 10, 2026
b81a131
ci: retrigger
robobun Jun 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 10 additions & 9 deletions docs/runtime/sql.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1041,15 +1041,16 @@ try {

### PostgreSQL Connection Errors

| Connection Errors | Description |
| --------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `ERR_POSTGRES_CONNECTION_CLOSED` | An established connection was terminated |
| `ERR_POSTGRES_CONNECTION_FAILED` | Connection could not be established (refused, or socket closed before the handshake completed, e.g. an intermediary accepted then closed); errors the server sends during startup, like `57P03`, surface as `ERR_POSTGRES_SERVER_ERROR` |
| `ERR_POSTGRES_CONNECTION_TIMEOUT` | Failed to establish connection within timeout period |
| `ERR_POSTGRES_IDLE_TIMEOUT` | Connection closed due to inactivity |
| `ERR_POSTGRES_LIFETIME_TIMEOUT` | Connection exceeded maximum lifetime |
| `ERR_POSTGRES_TLS_NOT_AVAILABLE` | SSL/TLS connection not available |
| `ERR_POSTGRES_TLS_UPGRADE_FAILED` | Failed to upgrade connection to SSL/TLS |
| Connection Errors | Description |
| --------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `ERR_POSTGRES_CONNECTION_CLOSED` | An established connection was terminated |
| `ERR_POSTGRES_CONNECTION_FAILED` | Connection was accepted but closed before the handshake completed (e.g. the server is still starting up). Retried with backoff until `connectionTimeout` while queries are waiting. Note: errors the server sends during startup, like `57P03`, surface as `ERR_POSTGRES_SERVER_ERROR` |
| `ERR_POSTGRES_CONNECTION_REFUSED` | Connection was refused because nothing is listening at the address. Fails immediately and is not retried |
| `ERR_POSTGRES_CONNECTION_TIMEOUT` | Failed to establish connection within timeout period |
| `ERR_POSTGRES_IDLE_TIMEOUT` | Connection closed due to inactivity |
| `ERR_POSTGRES_LIFETIME_TIMEOUT` | Connection exceeded maximum lifetime |
| `ERR_POSTGRES_TLS_NOT_AVAILABLE` | SSL/TLS connection not available |
| `ERR_POSTGRES_TLS_UPGRADE_FAILED` | Failed to upgrade connection to SSL/TLS |

### Authentication Errors

Expand Down
96 changes: 95 additions & 1 deletion src/js/internal/sql/mysql.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,12 @@ class PooledMySQLConnection {
flags: number = 0;
/// queryCount is used to indicate the number of queries using the connection, if a connection is reserved or if its a transaction queryCount will be 1 independently of the number of queries
queryCount: number = 0;
/// when the current connect cycle started; 0 when not connecting. Connect
/// failures (server not yet accepting connections) are retried until
/// connectionTimeout elapses from this point.
connectStartedAt: number = 0;
connectAttempts: number = 0;
retryTimer: ReturnType<typeof setTimeout> | null = null;

#onConnected(err, connection) {
if (err) {
Expand All @@ -318,6 +324,7 @@ class PooledMySQLConnection {
}
this.storedError = err;
if (!err) {
this.connectStartedAt = 0;
this.flags |= PooledConnectionFlags.canBeConnected;
}
this.state = err ? PooledConnectionState.closed : PooledConnectionState.connected;
Expand All @@ -343,12 +350,38 @@ class PooledMySQLConnection {
if (err) {
err = wrapError(err);
}
this.connection = null;
this.storedError = err;
if (this.#shouldRetryConnecting(err)) {
// The server is not accepting connections yet (e.g. still starting
// up). Keep the slot pending and retry with backoff instead of
// failing the queries that are waiting for a connection. The user's
// onclose callback only fires when the slot actually closes.
this.connectAttempts++;
const delay = Math.min(20 * 2 ** this.connectAttempts, 1000);
this.retryTimer = setTimeout(() => {
this.retryTimer = null;
// conditions may have changed during the backoff (pool closing,
// waiters gone, retry budget elapsed) — re-check before dialing
if (this.#canKeepRetrying()) {
this.#startConnection();
} else {
this.#finishClose(this.storedError);
}
}, delay);
return;
}
// this connect cycle is over; a later retry() starts a fresh one
this.connectStartedAt = 0;
this.#finishClose(err);
}

#finishClose(err) {
const connectionInfo = this.connectionInfo;
if (connectionInfo?.onclose) {
connectionInfo.onclose(err);
}
this.state = PooledConnectionState.closed;
this.connection = null;
this.storedError = err;

// remove from ready connections if its there
Expand Down Expand Up @@ -378,9 +411,58 @@ class PooledMySQLConnection {
}

#startConnection() {
if (this.connectStartedAt === 0) {
this.connectStartedAt = Date.now();
this.connectAttempts = 0;
}
PooledMySQLConnection.createConnection(this.connectionInfo, this.#onConnected.bind(this), this.#onClose.bind(this));
}

/// Connect failures (ERR_MYSQL_CONNECTION_FAILED) mean the server
/// accepted the TCP connection but closed it before the handshake
/// completed — typically it is still starting up, or an intermediary
/// (like a container port proxy) is up before the database is. Those are
/// retried until connectionTimeout elapses, as long as queries are
/// waiting on the pool. Refused connections
/// (ERR_MYSQL_CONNECTION_REFUSED) fail fast: nothing is listening,
/// and probes/healthchecks rely on the immediate error. Real server errors (authentication,
/// handshake errors) and closes of established connections are not
/// retried here.
#shouldRetryConnecting(err: Error | null): boolean {
if ((err as any)?.code !== "ERR_MYSQL_CONNECTION_FAILED") {
Comment thread
robobun marked this conversation as resolved.
Outdated
return false;
}
return this.#canKeepRetrying();
}

#canKeepRetrying(): boolean {
if (this.adapter.closed || this.onFinish !== null) {
return false;
}
// only retry while queries are actually waiting for a connection
if (this.adapter.waitingQueue.length === 0 && this.adapter.reservedQueue.length === 0) {
return false;
}
// an explicit connectionTimeout of 0 disables the connect timer, and with
// it the retry budget
const connectionTimeout = this.connectionInfo.connectionTimeout ?? 30 * 1000;
if (connectionTimeout <= 0) {
return false;
}
return this.connectStartedAt !== 0 && Date.now() - this.connectStartedAt < connectionTimeout;
}

/// Returns true if a scheduled connect retry was cancelled — in that case
/// nothing is in flight and no onClose/onConnected callback will fire.
cancelRetry(): boolean {
if (this.retryTimer !== null) {
clearTimeout(this.retryTimer);
this.retryTimer = null;
return true;
}
return false;
}

onClose(onClose: (err: Error) => void) {
this.queries.add(onClose);
}
Expand All @@ -396,6 +478,7 @@ class PooledMySQLConnection {
}
// reset error and state
this.storedError = null;
this.connectStartedAt = 0;
this.state = PooledConnectionState.pending;
// retry connection
this.#startConnection();
Expand Down Expand Up @@ -672,6 +755,11 @@ class MySQLAdapter
for (const pending of reservedQueue) {
pending(connection.storedError, connection);
}
// draining the queues may have been the last pending work — a
// graceful close() is waiting on this callback
if (this.onAllQueriesFinished && !this.hasPendingQueries()) {
this.onAllQueriesFinished();
}
}
return;
}
Expand Down Expand Up @@ -766,6 +854,12 @@ class MySQLAdapter
switch (connection.state) {
case PooledConnectionState.pending:
{
if (connection.cancelRetry()) {
// a connect retry was scheduled; nothing is in flight so
// there is no onClose/onConnected to wait for
connection.state = PooledConnectionState.closed;
break;
}
const { promise, resolve } = Promise.withResolvers();
connection.onFinish = resolve;
promises.push(promise);
Expand Down
96 changes: 95 additions & 1 deletion src/js/internal/sql/postgres.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -530,6 +530,12 @@ class PooledPostgresConnection {
flags: number = 0;
/// queryCount is used to indicate the number of queries using the connection, if a connection is reserved or if its a transaction queryCount will be 1 independently of the number of queries
queryCount: number = 0;
/// when the current connect cycle started; 0 when not connecting. Connect
/// failures (server not yet accepting connections) are retried until
/// connectionTimeout elapses from this point.
connectStartedAt: number = 0;
connectAttempts: number = 0;
retryTimer: ReturnType<typeof setTimeout> | null = null;

#onConnected(err, _) {
if (err) {
Expand All @@ -541,6 +547,7 @@ class PooledPostgresConnection {
}
this.storedError = err;
if (!err) {
this.connectStartedAt = 0;
this.flags |= PooledConnectionFlags.canBeConnected;
}
this.state = err ? PooledConnectionState.closed : PooledConnectionState.connected;
Expand All @@ -566,12 +573,38 @@ class PooledPostgresConnection {
if (err) {
err = wrapPostgresError(err);
}
this.connection = null;
this.storedError = err;
if (this.#shouldRetryConnecting(err)) {
// The server is not accepting connections yet (e.g. still starting
// up). Keep the slot pending and retry with backoff instead of
// failing the queries that are waiting for a connection. The user's
// onclose callback only fires when the slot actually closes.
this.connectAttempts++;
const delay = Math.min(20 * 2 ** this.connectAttempts, 1000);
this.retryTimer = setTimeout(() => {
Comment thread
robobun marked this conversation as resolved.
Outdated
Comment thread
robobun marked this conversation as resolved.
Outdated
this.retryTimer = null;
// conditions may have changed during the backoff (pool closing,
// waiters gone, retry budget elapsed) — re-check before dialing
if (this.#canKeepRetrying()) {
this.#startConnection();
} else {
this.#finishClose(this.storedError);
}
}, delay);
return;
}
// this connect cycle is over; a later retry() starts a fresh one
this.connectStartedAt = 0;
this.#finishClose(err);
}

#finishClose(err) {
const connectionInfo = this.connectionInfo;
if (connectionInfo?.onclose) {
connectionInfo.onclose(err);
}
this.state = PooledConnectionState.closed;
this.connection = null;
this.storedError = err;

// remove from ready connections if its there
Expand Down Expand Up @@ -601,13 +634,62 @@ class PooledPostgresConnection {
}

async #startConnection() {
if (this.connectStartedAt === 0) {
this.connectStartedAt = Date.now();
this.connectAttempts = 0;
}
this.connection = await PooledPostgresConnection.createConnection(
this.connectionInfo,
this.#onConnected.bind(this),
this.#onClose.bind(this),
);
}

/// Connect failures (ERR_POSTGRES_CONNECTION_FAILED) mean the server
/// accepted the TCP connection but closed it before the handshake
/// completed — typically it is still starting up, or an intermediary
/// (like a container port proxy) is up before the database is. Those are
/// retried until connectionTimeout elapses, as long as queries are
/// waiting on the pool. Refused connections
/// (ERR_POSTGRES_CONNECTION_REFUSED) fail fast: nothing is listening,
/// and probes/healthchecks rely on the immediate error. Real server errors (authentication,
/// ErrorResponse during startup) and closes of established connections are
/// not retried here.
#shouldRetryConnecting(err: Error | null): boolean {
if ((err as any)?.code !== "ERR_POSTGRES_CONNECTION_FAILED") {
Comment thread
robobun marked this conversation as resolved.
Outdated
return false;
}
return this.#canKeepRetrying();
}

#canKeepRetrying(): boolean {
if (this.adapter.closed || this.onFinish !== null) {
return false;
}
// only retry while queries are actually waiting for a connection
if (this.adapter.waitingQueue.length === 0 && this.adapter.reservedQueue.length === 0) {
return false;
}
// an explicit connectionTimeout of 0 disables the connect timer, and with
// it the retry budget
const connectionTimeout = this.connectionInfo.connectionTimeout ?? 30 * 1000;
if (connectionTimeout <= 0) {
return false;
}
return this.connectStartedAt !== 0 && Date.now() - this.connectStartedAt < connectionTimeout;
}

/// Returns true if a scheduled connect retry was cancelled — in that case
/// nothing is in flight and no onClose/onConnected callback will fire.
cancelRetry(): boolean {
if (this.retryTimer !== null) {
clearTimeout(this.retryTimer);
this.retryTimer = null;
return true;
}
return false;
}

onClose(onClose: (err: Error) => void) {
this.queries.add(onClose);
}
Expand All @@ -623,6 +705,7 @@ class PooledPostgresConnection {
}
// reset error and state
this.storedError = null;
this.connectStartedAt = 0;
this.state = PooledConnectionState.pending;
// retry connection
this.#startConnection();
Expand Down Expand Up @@ -909,6 +992,11 @@ class PostgresAdapter
for (const pending of reservedQueue) {
pending(connection.storedError, connection);
}
// draining the queues may have been the last pending work — a
// graceful close() is waiting on this callback
if (this.onAllQueriesFinished && !this.hasPendingQueries()) {
this.onAllQueriesFinished();
}
}
return;
}
Expand Down Expand Up @@ -1003,6 +1091,12 @@ class PostgresAdapter
switch (connection.state) {
case PooledConnectionState.pending:
{
if (connection.cancelRetry()) {
// a connect retry was scheduled; nothing is in flight so
// there is no onClose/onConnected to wait for
connection.state = PooledConnectionState.closed;
break;
}
const { promise, resolve } = Promise.withResolvers();
connection.onFinish = resolve;
promises.push(promise);
Expand Down
Loading
Loading