Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion server/src/auth/organization-principal.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import type { WorkOSUser } from "../types.js";

export type OrgAuthorizationPrincipal = Pick<
WorkOSUser,
"id" | "authWorkosUserId"
"id" | "authWorkosUserId" | "authorizationSnapshot"
>;

/**
Expand Down
175 changes: 159 additions & 16 deletions server/src/db/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,109 @@ export function withDatabaseDeadline<T>(
return queryDeadline.run({ deadlineMs, readOnly: options.readOnly ?? true }, work);
}

export class DatabaseQueryDeadlineExceededError extends Error {
readonly code = '57014' as const;
readonly retryable = true as const;

constructor() {
super('Database query deadline exceeded');
this.name = 'DatabaseQueryDeadlineExceededError';
}
}

function databaseDeadlineExceededError(): DatabaseQueryDeadlineExceededError {
return new DatabaseQueryDeadlineExceededError();
}

/**
* Await one client operation for no longer than the caller's absolute
* deadline. Promise.race keeps observing a late rejection, while callers
* destroy the client because a late result leaves its protocol/transaction
* state unknowable.
*/
async function clientOperationBeforeDeadline<T>(
operation: () => Promise<T>,
deadlineMs: number,
): Promise<T> {
const remainingMs = deadlineMs - Date.now();
if (remainingMs <= 0) throw databaseDeadlineExceededError();

const pending = Promise.resolve().then(operation);
let timeout: ReturnType<typeof setTimeout> | undefined;
const deadline = new Promise<never>((_resolve, reject) => {
timeout = setTimeout(() => reject(databaseDeadlineExceededError()), remainingMs);
});

try {
return await Promise.race([pending, deadline]);
} finally {
if (timeout !== undefined) clearTimeout(timeout);
}
}

function isDefinitivePostgresError(error: unknown): boolean {
if (!(error instanceof Error) || error instanceof DatabaseQueryDeadlineExceededError) return false;
const code = (error as { code?: unknown }).code;
return typeof code === 'string'
&& /^[0-9A-Z]{5}$/.test(code)
&& !isTransientConnectionError(error);
}

/**
* Execute one query with server-enforced statement and lock deadlines.
* Check out a pooled client without allowing pool saturation to outlive the
* caller's absolute deadline. A client delivered after the timer wins is
* released immediately so a timed-out request cannot leak a pool slot.
*/
async function checkoutClientBeforeDeadline(deadlineMs: number): Promise<PoolClient> {
const remainingMs = deadlineMs - Date.now();
if (remainingMs <= 0) throw databaseDeadlineExceededError();

const checkout = getPool().connect();
let timeout: ReturnType<typeof setTimeout> | undefined;
let timedOut = false;
const deadline = new Promise<never>((_resolve, reject) => {
timeout = setTimeout(() => {
timedOut = true;
reject(databaseDeadlineExceededError());
}, remainingMs);
});

try {
return await Promise.race([checkout, deadline]);
} catch (error) {
if (timedOut) {
void checkout.then(
(lateClient) => lateClient.release(),
() => undefined,
);
}
throw error;
} finally {
if (timeout !== undefined) clearTimeout(timeout);
}
}

async function getClientBeforeDeadline(
deadlineMs: number,
retryTransientCheckout: boolean,
): Promise<PoolClient> {
const attempts = retryTransientCheckout ? 2 : 1;
for (let attempt = 0; attempt < attempts; attempt++) {
try {
return await checkoutClientBeforeDeadline(deadlineMs);
} catch (error) {
if (attempt === attempts - 1 || !isTransientConnectionError(error) || Date.now() >= deadlineMs) {
throw error;
}
console.warn('Transient DB connection error, retrying client checkout:', (error as Error).message);
}
}
throw databaseDeadlineExceededError();
}

/**
* Execute one query with deadline-bounded pool checkout plus server-enforced
* statement and lock deadlines.
*
* Use this for public read paths whose inputs can select unusually large
* registry fan-outs. The transaction-local settings ensure PostgreSQL stops
Expand All @@ -159,39 +260,81 @@ export async function queryWithTimeout<T extends QueryResultRow = any>(
text: string,
params: any[] | undefined,
timeoutMs: number,
options: { retryTransientCheckout?: boolean; deadlineMs?: number } = {},
): Promise<QueryResult<T>> {
const inheritedDeadline = queryDeadline.getStore();
const deadlineMs = Math.min(
Date.now() + timeoutMs,
options.deadlineMs ?? Date.now() + timeoutMs,
inheritedDeadline?.deadlineMs ?? Number.POSITIVE_INFINITY,
);
const client = await getClient();
const client = await getClientBeforeDeadline(
deadlineMs,
options.retryTransientCheckout ?? true,
);
let transactionStarted = false;
let clientStateUncertain = false;

const run = async <R>(
operation: () => Promise<R>,
phase: 'begin' | 'configuration' | 'statement' | 'commit' | 'rollback',
): Promise<R> => {
try {
return await clientOperationBeforeDeadline(operation, deadlineMs);
} catch (error) {
// A client-side deadline can win while pg is still processing any query.
// Connection-style failures have no trustworthy server acknowledgement.
// COMMIT failures are always ambiguous: the transaction may be durable
// even though its acknowledgement never reached this process.
if (error instanceof DatabaseQueryDeadlineExceededError
|| !isDefinitivePostgresError(error)
|| phase === 'commit'
|| phase === 'rollback') {
clientStateUncertain = true;
}
throw error;
}
};

try {
const effectiveTimeoutMs = deadlineMs - Date.now();
if (effectiveTimeoutMs <= 0) {
throw Object.assign(new Error('Database query deadline exceeded'), { code: '57014' });
throw databaseDeadlineExceededError();
}
await client.query(inheritedDeadline?.readOnly === false ? 'BEGIN' : 'BEGIN READ ONLY');
await run(
() => client.query(inheritedDeadline?.readOnly === false ? 'BEGIN' : 'BEGIN READ ONLY'),
'begin',
);
transactionStarted = true;
await client.query("SELECT set_config('statement_timeout', $1, true)", [
`${effectiveTimeoutMs}ms`,
]);
await client.query("SELECT set_config('lock_timeout', $1, true)", [
`${Math.min(effectiveTimeoutMs, 2_000)}ms`,
]);
const result = await client.query<T>(text, params);
await client.query('COMMIT');
await run(
() => client.query("SELECT set_config('statement_timeout', $1, true)", [
`${effectiveTimeoutMs}ms`,
]),
'configuration',
);
await run(
() => client.query("SELECT set_config('lock_timeout', $1, true)", [
`${Math.min(effectiveTimeoutMs, 2_000)}ms`,
]),
'configuration',
);
const result = await run(() => client.query<T>(text, params), 'statement');
await run(() => client.query('COMMIT'), 'commit');
transactionStarted = false;
return result;
} catch (error) {
if (transactionStarted) {
await client.query('ROLLBACK').catch((rollbackError) => {
// Never queue ROLLBACK behind an operation that may still be in flight.
// If the prior outcome is definite, cleanup must still finish inside the
// same deadline before this client can be considered reusable.
if (transactionStarted && !clientStateUncertain) {
await run(() => client.query('ROLLBACK'), 'rollback').catch((rollbackError) => {
logger.warn({ err: rollbackError }, 'Timed query rollback failed');
});
if (!clientStateUncertain) transactionStarted = false;
}
throw error;
} finally {
client.release();
if (clientStateUncertain || transactionStarted) client.release(true);
else client.release();
}
}

Expand Down
Loading
Loading