From f30374d939ea5ad6d5eb00c177c50f2c0802d306 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 07:43:14 +0000 Subject: [PATCH 1/3] feat: never store login tokens in plain text Tokens are kept in the data directory only as SHA-256 hashes (for login lookup) and sealed with AES-256-GCM under a key held outside it, in a Docker volume at /var/lib/actor-runtime. Proxy passwords are sealed too. Existing data directories are rewritten on start. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017ocuAKna7JTHKG2TiQi5qH --- .gitignore | 1 + CLAUDE.MD | 1 + CONTRIBUTING.md | 1 + README.md | 3 + package.json | 2 +- requirements/storage.md | 3 +- requirements/system.md | 9 +- skills/actor-runtime/SKILL.md | 4 + src/api/auth.ts | 9 +- src/config.ts | 4 + src/console/server.ts | 2 +- src/index.ts | 13 ++- src/services/scheduler.ts | 4 +- src/services/standby.ts | 4 +- src/services/token-vault.ts | 76 ++++++++++++++++ src/services/user-codec.ts | 54 +++++++++++ src/services/users.ts | 55 +++++++++--- src/services/webhook-dispatches.ts | 2 +- src/storage/entities.ts | 10 ++- src/storage/registries.ts | 3 +- src/storage/registry.ts | 33 +++++-- test/integration/identity-resolution.test.ts | 3 +- test/integration/token-sealing.test.ts | 94 ++++++++++++++++++++ test/setup-token-vault.ts | 8 ++ vitest.config.ts | 1 + 25 files changed, 360 insertions(+), 39 deletions(-) create mode 100644 src/services/token-vault.ts create mode 100644 src/services/user-codec.ts create mode 100644 test/integration/token-sealing.test.ts create mode 100644 test/setup-token-vault.ts diff --git a/.gitignore b/.gitignore index 3a0dd775..f59627ef 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.db-wal *.db-shm /data/ +/data-key/ ca-bundle.crt # Durable run-pipeline data directory - never a project artifact. diff --git a/CLAUDE.MD b/CLAUDE.MD index 66fc59c3..848bc5c3 100644 --- a/CLAUDE.MD +++ b/CLAUDE.MD @@ -22,6 +22,7 @@ mkdir -p data docker run --rm -p 3333:3333 -p 3000:3000 \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$(pwd)/data:/data" \ + -v actor-runtime-keys:/var/lib/actor-runtime \ actor-runtime ``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ab139774..bef2cf29 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,6 +8,7 @@ mkdir -p data docker run --rm -p 3333:3333 -p 3000:3000 \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$(pwd)/data:/data" \ + -v actor-runtime-keys:/var/lib/actor-runtime \ actor-runtime ``` diff --git a/README.md b/README.md index 5eed466c..791b5148 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,7 @@ mkdir -p data podman run --rm -p 3333:3333 -p 3000:3000 \ -v "$XDG_RUNTIME_DIR/podman/podman.sock:/var/run/docker.sock" \ -v "$(pwd)/data:/data" \ + -v actor-runtime-keys:/var/lib/actor-runtime \ docker.io/apify/actor-runtime ``` @@ -232,6 +233,8 @@ podman run --rm -p 3333:3333 -p 3000:3000 \ - Under rootless Podman 3.x, add `--network slirp4netns:allow_host_loopback=true` to the `podman run` above (`apify runtime start` does) so the runtime reaches the web servers of runs; without it only standby runs' servers are reachable. +- Rootless Podman keeps the `actor-runtime-keys` volume under your home directory, where a process + running as you can read it. Rootful Podman and Docker keep it out of reach of your user (`system.md`). - Podman does not create a missing bind-mount directory, hence `mkdir -p data`. `apify runtime start` creates its data directory itself. - Short image names in an Actor's `FROM` line (`apify/actor-node:20`) resolve to Docker Hub, as on the diff --git a/package.json b/package.json index 40f54e03..a3438d00 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "scripts": { "build": "tsc -p tsconfig.json", "start": "node dist/index.js", - "dev": "ACTOR_RUNTIME_DATA_DIR=./data tsx src/index.ts", + "dev": "ACTOR_RUNTIME_DATA_DIR=./data ACTOR_RUNTIME_KEY_DIR=./data-key tsx src/index.ts", "lint": "eslint .", "lint:fix": "eslint . --fix", "format": "prettier --write .", diff --git a/requirements/storage.md b/requirements/storage.md index e4d54fa4..8b83511b 100644 --- a/requirements/storage.md +++ b/requirements/storage.md @@ -38,8 +38,7 @@ - `key` is the `userId` - `value` is the metadata of the user - name - - token - - proxyPassword (optional) + - token, every earlier token, and proxyPassword (optional) - never in plain text (`system.md`) - The system stores Actors in dedicated key-value store called `__ACTORS__`: - `key` is the id of the Actor `actorId` - `value` is the metadata of the Actor diff --git a/requirements/system.md b/requirements/system.md index 464a30a3..b6f6c7b3 100644 --- a/requirements/system.md +++ b/requirements/system.md @@ -38,7 +38,8 @@ (`-v /var/run/docker.sock:/var/run/docker.sock`) so the runtime can build and run Actor containers, and mount a persistent data directory (`-v :/data`, e.g. `-v "$(pwd)/data:/data"`) so storages survive a restart and are easy to inspect from the host; the directory must exist before the - runtime starts. Publish both ports + runtime starts. Mount a named volume at `/var/lib/actor-runtime` (`-v actor-runtime-keys:/var/lib/actor-runtime`) + to keep the runtime's key across restarts. Publish both ports (`-p 3333:3333 -p 3000:3000`). The canonical start command is: ```bash @@ -47,9 +48,15 @@ docker run --rm -p 3333:3333 -p 3000:3000 \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$(pwd)/data:/data" \ + -v actor-runtime-keys:/var/lib/actor-runtime \ actor-runtime ``` +- **Login tokens are never stored in plain text.** Neither tokens nor proxy passwords appear anywhere + in the data directory: they can be read back only with a key the runtime keeps outside it, in the + `/var/lib/actor-runtime` volume. A data directory written before this is rewritten on start. + - Without that volume the key is new on every start: logins keep working, but a user's schedules and + standby Actors cannot start runs until that user makes an API request again; the runtime warns. - **Docker and Podman are equally supported**, rootful or rootless: everything the system offers works the same on either engine. The user picks the engine by mounting its Docker-compatible API socket in place of the Docker one (e.g. `-v /run/podman/podman.sock:/var/run/docker.sock`). One exception: diff --git a/skills/actor-runtime/SKILL.md b/skills/actor-runtime/SKILL.md index 4ddfad9d..a640a37e 100644 --- a/skills/actor-runtime/SKILL.md +++ b/skills/actor-runtime/SKILL.md @@ -35,6 +35,10 @@ Any non-empty token authenticates - `apify login --token local-dev-token` is eno `APIFY_DISABLE_KEYRING=1` first in a sandbox with no OS keyring. To act as a second user, pass a different token on a single call: `apify api v2/datasets -H '{"authorization": "Bearer OTHER"}'`. +Tokens are never written to the data directory in plain text, so do not look for them there. If a +schedule or standby Actor fails with "the token of user ... cannot be read", the runtime lost its key +(started without its `/var/lib/actor-runtime` volume): any API call with that user's token fixes it. + ### Other ports When 3333 or 3000 is taken on this machine, start the runtime on other ports: diff --git a/src/api/auth.ts b/src/api/auth.ts index e8090a86..3b8e4c2a 100644 --- a/src/api/auth.ts +++ b/src/api/auth.ts @@ -54,9 +54,12 @@ export function auth(): RequestHandler { * practice (surfaces as a 500 through `h`'s error middleware, not a user-facing error) - a route that * calls this without `auth()` having run first is a wiring bug, not a request the caller can cause. */ -export function requireUser(req: Request): UserRecord { - if (!req.user) { +export function requireUser(req: Request): UserRecord & { token: string } { + const { user } = req; + if (!user) { throw new Error('requireUser() called outside the authenticated router - auth() did not run first'); } - return req.user; + // `getOrCreateUserForToken` always hands back a user holding a token: the request just brought one. + if (user.token === undefined) throw new Error(`User ${user.id} has no token after authenticating`); + return { ...user, token: user.token }; } diff --git a/src/config.ts b/src/config.ts index ce72006d..d2fca74f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -39,6 +39,10 @@ export const CONSOLE_BASE_URL = `http://localhost:${CONSOLE_PORT}`; export const DEFAULT_DATA_DIR = process.env.ACTOR_RUNTIME_DATA_DIR ?? '/data'; +/** Holds the key login tokens are sealed with (`services/token-vault.ts`). Never inside the data + * directory: in the image it is a Docker volume of its own, out of reach of a scan of the host's files. */ +export const KEY_DIR = process.env.ACTOR_RUNTIME_KEY_DIR ?? '/var/lib/actor-runtime'; + /** Where the Python debug-mode payload lives inside the runtime's own image. Read fresh on every call * (not cached) so tests can point `ACTOR_RUNTIME_DEBUGPY_PAYLOAD_DIR` at a fixture directory. */ function debugpyPayloadDir(): string { diff --git a/src/console/server.ts b/src/console/server.ts index 4a44ecb3..b5979286 100644 --- a/src/console/server.ts +++ b/src/console/server.ts @@ -234,7 +234,7 @@ export function createConsoleServer(deps: ConsoleServerDeps): Express { const pool = standbyPoolSnapshot(service); return ( '

Actor Standby

' + - (owner + (owner?.token !== undefined ? "

Open or copy the standby URL with the owner's token already in it:

" + standbyLink(url, owner.token) : '') + diff --git a/src/index.ts b/src/index.ts index d702e7dc..122f7e4c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,11 +14,22 @@ import { startWebhookDispatcher } from './services/webhook-dispatches.js'; import { startScheduler } from './services/scheduler.js'; import { buildDto, runDto } from './api/dto/actors.js'; import { gracefulShutdown } from './shutdown.js'; -import { API_PORT, CONSOLE_PORT, DEFAULT_DATA_DIR } from './config.js'; +import { API_PORT, CONSOLE_PORT, DEFAULT_DATA_DIR, KEY_DIR } from './config.js'; +import { initTokenVault } from './services/token-vault.js'; +import { sealStoredUsers } from './services/users.js'; async function main(): Promise { + const vault = initTokenVault(KEY_DIR); bootstrapStorage(DEFAULT_DATA_DIR); await openRegistries(); + const { unreadable } = await sealStoredUsers(); + if (vault.created && unreadable > 0) { + console.warn( + `The key login tokens were sealed with is gone (${KEY_DIR} is new), so ${unreadable} user(s) need an ` + + 'API request with their token before their schedules and standby Actors can start runs. Mount a ' + + `named volume at ${KEY_DIR} (-v actor-runtime-keys:${KEY_DIR}) to keep the key across restarts.`, + ); + } // No default user is created here any more - users are created ad-hoc, per previously-unseen token, // at the first API request that carries it (`services/users.ts: getOrCreateUserForToken()`, // `cli.md`'s User bootstrap). diff --git a/src/services/scheduler.ts b/src/services/scheduler.ts index 0f9bb7c3..4caca788 100644 --- a/src/services/scheduler.ts +++ b/src/services/scheduler.ts @@ -26,7 +26,7 @@ import { listOwnedRuns, startRun } from './runs.js'; import { MAX_LOG_MESSAGES, nextRunAtOf } from './schedules.js'; import { standbyUrl } from './standby-config.js'; import { getOwnedTask } from './tasks.js'; -import { getUserById, resolveProxyPassword } from './users.js'; +import { getUserById, resolveProxyPassword, tokenOf } from './users.js'; /** `setTimeout` overflows past this; a longer wait is re-armed when it elapses. */ const MAX_TIMER_DELAY_MILLIS = 2 ** 31 - 1; @@ -231,7 +231,7 @@ async function startActionRun( standbyUrl: standbyUrl(actor, user.username), proxyPassword: resolveProxyPassword(user), apiBaseUrl: CONTAINER_API_BASE_URL, - token: user.token, + token: tokenOf(user), }); } diff --git a/src/services/standby.ts b/src/services/standby.ts index 99e694c3..d53d5759 100644 --- a/src/services/standby.ts +++ b/src/services/standby.ts @@ -17,7 +17,7 @@ import { publishAborting, publishPersistState } from './events-channel.js'; import { isTerminalJobStatus } from './job-status.js'; import { resolveTaggedBuild } from './builds.js'; import { resolveBuildInput } from './input-schema.js'; -import { resolveProxyPassword } from './users.js'; +import { resolveProxyPassword, tokenOf } from './users.js'; import { containerServerPortFor, startRun } from './runs.js'; import { findVersion } from './actors.js'; import { markStandbyRunFinishing } from './standby-finish.js'; @@ -167,7 +167,7 @@ async function startStandbyRun( standbyUrl: standbyUrl(task ?? actor, user.username), proxyPassword: resolveProxyPassword(user), apiBaseUrl: CONTAINER_API_BASE_URL, - token: user.token, + token: tokenOf(user), }); } diff --git a/src/services/token-vault.ts b/src/services/token-vault.ts new file mode 100644 index 00000000..709f1f62 --- /dev/null +++ b/src/services/token-vault.ts @@ -0,0 +1,76 @@ +import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto'; +import { chmodSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +/** + * Keeps login tokens out of the data directory in plain text (`storage.md`): a token is stored only as + * its SHA-256 (enough to recognise a login) and sealed with AES-256-GCM under a key that lives in its own + * directory, never inside the data directory. In the image that directory is a Docker volume, so a scan of + * the host's files finds neither the tokens nor the key that opens them. + */ + +const KEY_FILE = 'token-vault.key'; +const KEY_BYTES = 32; +const IV_BYTES = 12; +const TAG_BYTES = 16; +const SEALED_PREFIX = 'v1:'; + +let key: Buffer | undefined; + +/** Loads the key from `keyDir`, creating it on first start. `created` with users already stored means the + * previous key is gone (e.g. the key volume was not mounted), so their sealed tokens no longer open. */ +export function initTokenVault(keyDir: string): { created: boolean } { + mkdirSync(keyDir, { recursive: true, mode: 0o700 }); + const path = join(keyDir, KEY_FILE); + try { + const existing = readFileSync(path); + if (existing.length !== KEY_BYTES) throw new Error(`${path} is not a ${KEY_BYTES}-byte key`); + key = existing; + return { created: false }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + const fresh = randomBytes(KEY_BYTES); + // `wx`: never overwrite a key another process wrote in the meantime - that would orphan its tokens. + writeFileSync(path, fresh, { mode: 0o600, flag: 'wx' }); + chmodSync(path, 0o600); + key = fresh; + return { created: true }; +} + +function requireKey(): Buffer { + if (!key) throw new Error('initTokenVault() has not been called yet.'); + return key; +} + +/** Unsalted on purpose: lookups must match across restarts and even after the key is lost. API tokens + * are long random strings, so the hash cannot be reversed by guessing. */ +export function hashToken(token: string): string { + return createHash('sha256').update(token, 'utf8').digest('hex'); +} + +/** `context` (the record id) is bound in as associated data: a sealed value copied onto another record + * does not open there. */ +export function seal(value: unknown, context: string): string { + const iv = randomBytes(IV_BYTES); + const cipher = createCipheriv('aes-256-gcm', requireKey(), iv); + cipher.setAAD(Buffer.from(context, 'utf8')); + const encrypted = Buffer.concat([cipher.update(JSON.stringify(value), 'utf8'), cipher.final()]); + return SEALED_PREFIX + Buffer.concat([iv, cipher.getAuthTag(), encrypted]).toString('base64'); +} + +/** `undefined` when the value was sealed under another key, for another record, or was tampered with. */ +export function unseal(sealed: string, context: string): T | undefined { + if (!sealed.startsWith(SEALED_PREFIX)) return undefined; + const raw = Buffer.from(sealed.slice(SEALED_PREFIX.length), 'base64'); + if (raw.length < IV_BYTES + TAG_BYTES) return undefined; + try { + const decipher = createDecipheriv('aes-256-gcm', requireKey(), raw.subarray(0, IV_BYTES)); + decipher.setAAD(Buffer.from(context, 'utf8')); + decipher.setAuthTag(raw.subarray(IV_BYTES, IV_BYTES + TAG_BYTES)); + const plain = Buffer.concat([decipher.update(raw.subarray(IV_BYTES + TAG_BYTES)), decipher.final()]); + return JSON.parse(plain.toString('utf8')) as T; + } catch { + return undefined; + } +} diff --git a/src/services/user-codec.ts b/src/services/user-codec.ts new file mode 100644 index 00000000..4c45e83f --- /dev/null +++ b/src/services/user-codec.ts @@ -0,0 +1,54 @@ +import type { RecordCodec } from '../storage/registry.js'; +import type { UserRecord } from '../storage/entities.js'; +import { hashToken, seal, unseal } from './token-vault.js'; + +/** A `__USERS__` record as written to disk: the user's credentials only ever sealed (`token-vault.ts`). */ +interface StoredUser { + id: string; + username: string; + createdAt: string; + tokenHashes: string[]; + sealed: string; +} + +interface SealedCredentials { + token?: string; + proxyPassword?: string; +} + +/** A record written before tokens were sealed, read once and rewritten sealed (`sealStoredUsers`). */ +interface LegacyUser { + id: string; + username: string; + createdAt: string; + token: string; + tokens?: string[]; + proxyPassword?: string; +} + +export function isLegacyUser(stored: unknown): stored is LegacyUser { + return typeof (stored as LegacyUser).token === 'string'; +} + +export const userCodec: RecordCodec = { + encode({ id, username, createdAt, tokenHashes, token, proxyPassword }): StoredUser { + const credentials: SealedCredentials = { token, proxyPassword }; + return { id, username, createdAt, tokenHashes, sealed: seal(credentials, id) }; + }, + decode(stored): UserRecord { + if (isLegacyUser(stored)) { + const { id, username, createdAt, token, tokens, proxyPassword } = stored; + return { id, username, createdAt, token, tokenHashes: (tokens ?? [token]).map(hashToken), proxyPassword }; + } + const { id, username, createdAt, tokenHashes, sealed } = stored as StoredUser; + const credentials = unseal(sealed, id) ?? {}; + return { + id, + username, + createdAt, + tokenHashes, + ...(credentials.token !== undefined ? { token: credentials.token } : {}), + ...(credentials.proxyPassword !== undefined ? { proxyPassword: credentials.proxyPassword } : {}), + }; + }, +}; diff --git a/src/services/users.ts b/src/services/users.ts index db760aa1..5aa82b3d 100644 --- a/src/services/users.ts +++ b/src/services/users.ts @@ -3,6 +3,7 @@ import { getRegistries } from '../storage/registries.js'; import { KeyedMutex } from '../storage/mutex.js'; import { isApifyProxyEnabled } from './apify-proxy.js'; import { fetchRealIdentity, type RealIdentity, upstreamApiBaseUrl } from './identity-resolution.js'; +import { hashToken } from './token-vault.js'; /** Serialises "resolve token -> create-if-missing" per token, so two concurrent first-requests for the * same brand-new token can never both pass the not-found check and mint two users - the same @@ -43,7 +44,7 @@ async function fabricateUser(token: string): Promise { id: `0000000000000000${n}`, username: `local-user-${n}`, token, - tokens: [token], + tokenHashes: [hashToken(token)], createdAt: new Date().toISOString(), }; } @@ -51,25 +52,23 @@ async function fabricateUser(token: string): Promise { /** Enough for a run container started with a token that has since rotated hourly for days. */ const MAX_REMEMBERED_TOKENS = 100; -function tokensOf(user: UserRecord): string[] { - return user.tokens ?? [user.token]; -} - async function findUserByToken(token: string): Promise { + const hash = hashToken(token); const all = await getRegistries().users.list(); - return all.find((user) => tokensOf(user).includes(token)) ?? null; + return all.find((user) => user.tokenHashes.includes(hash)) ?? null; } /** A token that resolved to an account the runtime already knows (an OAuth login's token rotates every * hour) joins that user: its earlier tokens keep resolving, so a run container or a restart does not * lose the account, and the new one becomes the token handed out from now on. */ -function withToken(existing: UserRecord, token: string, identity: RealIdentity): UserRecord { +function withToken(existing: UserRecord, token: string, identity: RealIdentity | null): UserRecord { + const hash = hashToken(token); return { ...existing, - username: identity.username, + username: identity?.username ?? existing.username, token, - tokens: [...tokensOf(existing).filter((known) => known !== token), token].slice(-MAX_REMEMBERED_TOKENS), - proxyPassword: identity.proxyPassword ?? existing.proxyPassword, + tokenHashes: [...existing.tokenHashes.filter((known) => known !== hash), hash].slice(-MAX_REMEMBERED_TOKENS), + proxyPassword: identity?.proxyPassword ?? existing.proxyPassword, }; } @@ -103,10 +102,18 @@ export async function getOrCreateUserForToken(token: string): Promise { + const { users } = getRegistries(); + let unreadable = 0; + for (const user of await users.list()) { + if (user.token === undefined) unreadable++; + await users.update(user.id, (current) => current); + } + return { unreadable }; +} + +/** The user's token for work the runtime starts on its own (schedules, standby). */ +export function tokenOf(user: UserRecord): string { + if (user.token !== undefined) return user.token; + throw new Error( + `The token of user ${user.username} cannot be read: the runtime's key it was sealed with is gone. ` + + 'Any API request with that token restores it.', + ); +} + export async function getUserById(userId: string): Promise { return getRegistries().users.get(userId); } diff --git a/src/services/webhook-dispatches.ts b/src/services/webhook-dispatches.ts index 2440017d..970c19a9 100644 --- a/src/services/webhook-dispatches.ts +++ b/src/services/webhook-dispatches.ts @@ -399,7 +399,7 @@ async function deliver(dispatch: WebhookDispatchRecord, call: WebhookDispatchCal // As on the platform, a request to the Apify API without a token is authenticated as the webhook's owner. if (ownApi && !url.searchParams.get('token')) { const user = await getUserById(dispatch.userId); - if (user) headers.Authorization = `Bearer ${user.token}`; + if (user?.token !== undefined) headers.Authorization = `Bearer ${user.token}`; } try { diff --git a/src/storage/entities.ts b/src/storage/entities.ts index d80d2baf..e3448b23 100644 --- a/src/storage/entities.ts +++ b/src/storage/entities.ts @@ -38,10 +38,12 @@ export interface DatasetSchema { export interface UserRecord { id: string; username: string; - /** The token seen last, handed to new runs, schedules and standby links. */ - token: string; - /** Every token this user was seen with, oldest first; absent on records written before tokens rotated. */ - tokens?: string[]; + /** The token seen last, handed to new runs, schedules and standby links. Never written to disk as is + * (`services/user-codec.ts`). Absent when it was sealed under a runtime key that is gone, until the + * user's next request brings a token again. */ + token?: string; + /** SHA-256 of every token this user was seen with, oldest first - what a login is matched against. */ + tokenHashes: string[]; createdAt: string; /** Real Apify Proxy password for this account, harvested from the upstream `/v2/users/me` response * at creation time when the token resolved against the real platform (see diff --git a/src/storage/registries.ts b/src/storage/registries.ts index 9281e98c..e1bc3fa6 100644 --- a/src/storage/registries.ts +++ b/src/storage/registries.ts @@ -5,6 +5,7 @@ */ import { Registry } from './registry.js'; import { openKeyValueStore } from './open.js'; +import { userCodec } from '../services/user-codec.js'; import type { ActorRecord, BuildRecord, @@ -38,7 +39,7 @@ export async function openRegistries(): Promise { if (current) return current; current = { storages: await Registry.open('__STORAGES__'), - users: await Registry.open('__USERS__'), + users: await Registry.open('__USERS__', userCodec), actors: await Registry.open('__ACTORS__'), runs: await Registry.open('__RUNS__'), builds: await Registry.open('__BUILDS__'), diff --git a/src/storage/registry.ts b/src/storage/registry.ts index 6b4490ae..cb54c17f 100644 --- a/src/storage/registry.ts +++ b/src/storage/registry.ts @@ -13,17 +13,36 @@ import { KeyedMutex } from './mutex.js'; /** Called after a write lands, inside the same per-id lock, with the record before and after it. */ export type RegistryChangeListener = (previous: T | null, next: T | null) => void; +/** Translates between a record and what is written to disk, for a registry whose records must not be + * stored as they are held in memory (`__USERS__`' tokens, `services/user-codec.ts`). */ +export interface RecordCodec { + encode(value: T): unknown; + decode(stored: unknown): T; +} + +const identityCodec: RecordCodec = { encode: (value) => value, decode: (stored) => stored as never }; + export class Registry { private readonly listeners = new Set>(); private constructor( private readonly store: KeyValueStore, private readonly mutex: KeyedMutex, + private readonly codec: RecordCodec, ) {} - static async open(name: string): Promise> { + static async open(name: string, codec: RecordCodec = identityCodec): Promise> { const store = await openKeyValueStore(name); - return new Registry(store, new KeyedMutex()); + return new Registry(store, new KeyedMutex(), codec); + } + + private async load(id: string): Promise { + const stored = await this.store.getValue(id); + return stored === null || stored === undefined ? null : this.codec.decode(stored); + } + + private save(id: string, value: T | null): Promise { + return this.store.setValue(id, value === null ? null : this.codec.encode(value)); } /** Serialised with writes to the same id: a read racing a delete can otherwise fail in the store. */ @@ -32,13 +51,13 @@ export class Registry { } private read(id: string): Promise { - return this.mutex.run(id, () => this.store.getValue(id)); + return this.mutex.run(id, () => this.load(id)); } async set(id: string, value: T): Promise { await this.mutex.run(id, async () => { - const previous = this.listeners.size > 0 ? await this.store.getValue(id) : null; - await this.store.setValue(id, value); + const previous = this.listeners.size > 0 ? await this.load(id) : null; + await this.save(id, value); this.notify(previous, value); }); } @@ -46,9 +65,9 @@ export class Registry { /** Read-modify-write, serialised per id. Returning `null` from `mutator` deletes the record. */ async update(id: string, mutator: (current: T | null) => T | null): Promise { return this.mutex.run(id, async () => { - const current = await this.store.getValue(id); + const current = await this.load(id); const next = mutator(current); - await this.store.setValue(id, next); + await this.save(id, next); this.notify(current, next); return next; }); diff --git a/test/integration/identity-resolution.test.ts b/test/integration/identity-resolution.test.ts index fa472ab9..738c3fb1 100644 --- a/test/integration/identity-resolution.test.ts +++ b/test/integration/identity-resolution.test.ts @@ -17,6 +17,7 @@ import { ApifyClient } from 'apify-client'; import { capturingDriver, startTestServer, type TestServerHandle } from './helpers/test-server.js'; import { getRegistries } from '../../src/storage/registries.js'; +import { hashToken } from '../../src/services/token-vault.js'; interface StubUpstream { baseUrl: string; @@ -197,7 +198,7 @@ describe('identity resolution against the real platform (stubbed upstream)', () const all = (await getRegistries().users.list()).filter((user) => user.id === 'rotating-real-id'); expect(all).toHaveLength(1); expect(all[0]?.token).toBe('rotating-token-2'); - expect(all[0]?.tokens).toEqual(['rotating-token-1', 'rotating-token-2']); + expect(all[0]?.tokenHashes).toEqual(['rotating-token-1', 'rotating-token-2'].map(hashToken)); const { resetUsersForTests } = await import('../../src/services/users.js'); resetUsersForTests(); diff --git a/test/integration/token-sealing.test.ts b/test/integration/token-sealing.test.ts new file mode 100644 index 00000000..32a4662a --- /dev/null +++ b/test/integration/token-sealing.test.ts @@ -0,0 +1,94 @@ +import { mkdtempSync } from 'node:fs'; +import { readdir, readFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { ApifyClient } from 'apify-client'; + +import { startTestServer, type TestServerHandle } from './helpers/test-server.js'; +import { getRegistries } from '../../src/storage/registries.js'; +import { openKeyValueStore } from '../../src/storage/open.js'; +import { initTokenVault } from '../../src/services/token-vault.js'; +import { resetUsersForTests, sealStoredUsers, tokenOf } from '../../src/services/users.js'; + +async function filesContaining(dir: string, needle: string): Promise { + const hits: string[] = []; + for (const entry of await readdir(dir, { withFileTypes: true, recursive: true })) { + if (!entry.isFile()) continue; + const path = join(entry.parentPath, entry.name); + if ((await readFile(path, 'utf8')).includes(needle)) hits.push(path); + } + return hits; +} + +const newKeyDir = () => mkdtempSync(join(tmpdir(), 'actor-runtime-test-key-')); + +describe('login tokens at rest', () => { + let server: TestServerHandle; + + beforeEach(async () => { + initTokenVault(newKeyDir()); + server = await startTestServer(undefined, 'apify_api_never-on-disk-0123456789'); + }); + + afterEach(async () => { + await server.close(); + }); + + it('a token, current or earlier, and a proxy password never appear in the data directory', async () => { + const me = await server.client.user('me').get(); + await getRegistries().users.update(me.id, (user) => user && { ...user, proxyPassword: 'proxy-secret-42' }); + await server.client.actors().create({ name: 'some-actor' }); + + expect(await filesContaining(server.dataDir, server.token)).toEqual([]); + expect(await filesContaining(server.dataDir, 'proxy-secret-42')).toEqual([]); + const stored = await getRegistries().users.get(me.id); + expect(stored?.token).toBe(server.token); + expect(stored?.proxyPassword).toBe('proxy-secret-42'); + }); + + it('a user stored with a plain-text token before sealing existed is rewritten sealed and still logs in', async () => { + const legacy = { + id: 'legacy-user-id', + username: 'legacy-user', + createdAt: new Date().toISOString(), + token: 'legacy-plain-token', + tokens: ['legacy-older-token', 'legacy-plain-token'], + proxyPassword: 'legacy-proxy', + }; + await (await openKeyValueStore('__USERS__')).setValue(legacy.id, legacy); + + await sealStoredUsers(); + + for (const secret of ['legacy-plain-token', 'legacy-older-token', 'legacy-proxy']) { + expect(await filesContaining(server.dataDir, secret)).toEqual([]); + } + for (const token of ['legacy-plain-token', 'legacy-older-token']) { + const client = new ApifyClient({ baseUrl: server.baseUrl, token, maxRetries: 0 }); + expect((await client.user('me').get()).id).toBe(legacy.id); + } + }); + + it('with the key gone, logins still resolve and the next request restores the token', async () => { + const me = await server.client.user('me').get(); + + resetUsersForTests(); + expect(initTokenVault(newKeyDir()).created).toBe(true); + expect(await sealStoredUsers()).toEqual({ unreadable: 1 }); + const orphaned = await getRegistries().users.get(me.id); + expect(orphaned?.token).toBeUndefined(); + expect(() => tokenOf(orphaned!)).toThrow(/cannot be read/); + + expect((await server.client.user('me').get()).id).toBe(me.id); + expect((await getRegistries().users.get(me.id))?.token).toBe(server.token); + }); + + it('a sealed value copied onto another user does not open there', async () => { + const me = await server.client.user('me').get(); + const store = await openKeyValueStore('__USERS__'); + const raw = (await store.getValue(me.id)) as Record; + await store.setValue('someone-else', { ...raw, id: 'someone-else', tokenHashes: [] }); + + expect((await getRegistries().users.get('someone-else'))?.token).toBeUndefined(); + }); +}); diff --git a/test/setup-token-vault.ts b/test/setup-token-vault.ts new file mode 100644 index 00000000..21cc51e8 --- /dev/null +++ b/test/setup-token-vault.ts @@ -0,0 +1,8 @@ +import { mkdtempSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { initTokenVault } from '../src/services/token-vault.js'; + +// The runtime loads its key at startup (`index.ts`); tests start storage on their own, so each worker gets one. +initTokenVault(mkdtempSync(join(tmpdir(), 'actor-runtime-test-key-'))); diff --git a/vitest.config.ts b/vitest.config.ts index ed7cae41..ee57b0c2 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -6,5 +6,6 @@ export default defineConfig({ testTimeout: 30_000, hookTimeout: 30_000, globals: false, + setupFiles: ['test/setup-token-vault.ts'], }, }); From e80115dbfe2631ccc0c1bcddfb14e54ec8f6e1e9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 07:57:04 +0000 Subject: [PATCH 2/3] feat: keep the token key in the data directory, owner-only Drop the separate key volume: the runtime creates the key file in its data directory with mode 0600, re-tightened on every start, so only the runtime's own user and root can open the sealed tokens. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017ocuAKna7JTHKG2TiQi5qH --- .gitignore | 1 - CLAUDE.MD | 1 - CONTRIBUTING.md | 1 - README.md | 3 --- package.json | 2 +- requirements/system.md | 14 ++++++-------- skills/actor-runtime/SKILL.md | 7 ++++--- src/config.ts | 4 ---- src/index.ts | 9 ++++----- src/services/token-vault.ts | 23 +++++++++++++---------- src/services/users.ts | 6 +++--- src/storage/entities.ts | 2 +- test/integration/token-sealing.test.ts | 22 ++++++++++++++-------- 13 files changed, 46 insertions(+), 49 deletions(-) diff --git a/.gitignore b/.gitignore index f59627ef..3a0dd775 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ *.db-wal *.db-shm /data/ -/data-key/ ca-bundle.crt # Durable run-pipeline data directory - never a project artifact. diff --git a/CLAUDE.MD b/CLAUDE.MD index 848bc5c3..66fc59c3 100644 --- a/CLAUDE.MD +++ b/CLAUDE.MD @@ -22,7 +22,6 @@ mkdir -p data docker run --rm -p 3333:3333 -p 3000:3000 \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$(pwd)/data:/data" \ - -v actor-runtime-keys:/var/lib/actor-runtime \ actor-runtime ``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bef2cf29..ab139774 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,6 @@ mkdir -p data docker run --rm -p 3333:3333 -p 3000:3000 \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$(pwd)/data:/data" \ - -v actor-runtime-keys:/var/lib/actor-runtime \ actor-runtime ``` diff --git a/README.md b/README.md index 791b5148..5eed466c 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,6 @@ mkdir -p data podman run --rm -p 3333:3333 -p 3000:3000 \ -v "$XDG_RUNTIME_DIR/podman/podman.sock:/var/run/docker.sock" \ -v "$(pwd)/data:/data" \ - -v actor-runtime-keys:/var/lib/actor-runtime \ docker.io/apify/actor-runtime ``` @@ -233,8 +232,6 @@ podman run --rm -p 3333:3333 -p 3000:3000 \ - Under rootless Podman 3.x, add `--network slirp4netns:allow_host_loopback=true` to the `podman run` above (`apify runtime start` does) so the runtime reaches the web servers of runs; without it only standby runs' servers are reachable. -- Rootless Podman keeps the `actor-runtime-keys` volume under your home directory, where a process - running as you can read it. Rootful Podman and Docker keep it out of reach of your user (`system.md`). - Podman does not create a missing bind-mount directory, hence `mkdir -p data`. `apify runtime start` creates its data directory itself. - Short image names in an Actor's `FROM` line (`apify/actor-node:20`) resolve to Docker Hub, as on the diff --git a/package.json b/package.json index a3438d00..40f54e03 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "scripts": { "build": "tsc -p tsconfig.json", "start": "node dist/index.js", - "dev": "ACTOR_RUNTIME_DATA_DIR=./data ACTOR_RUNTIME_KEY_DIR=./data-key tsx src/index.ts", + "dev": "ACTOR_RUNTIME_DATA_DIR=./data tsx src/index.ts", "lint": "eslint .", "lint:fix": "eslint . --fix", "format": "prettier --write .", diff --git a/requirements/system.md b/requirements/system.md index b6f6c7b3..810b29b7 100644 --- a/requirements/system.md +++ b/requirements/system.md @@ -38,8 +38,7 @@ (`-v /var/run/docker.sock:/var/run/docker.sock`) so the runtime can build and run Actor containers, and mount a persistent data directory (`-v :/data`, e.g. `-v "$(pwd)/data:/data"`) so storages survive a restart and are easy to inspect from the host; the directory must exist before the - runtime starts. Mount a named volume at `/var/lib/actor-runtime` (`-v actor-runtime-keys:/var/lib/actor-runtime`) - to keep the runtime's key across restarts. Publish both ports + runtime starts. Publish both ports (`-p 3333:3333 -p 3000:3000`). The canonical start command is: ```bash @@ -48,15 +47,14 @@ docker run --rm -p 3333:3333 -p 3000:3000 \ -v /var/run/docker.sock:/var/run/docker.sock \ -v "$(pwd)/data:/data" \ - -v actor-runtime-keys:/var/lib/actor-runtime \ actor-runtime ``` -- **Login tokens are never stored in plain text.** Neither tokens nor proxy passwords appear anywhere - in the data directory: they can be read back only with a key the runtime keeps outside it, in the - `/var/lib/actor-runtime` volume. A data directory written before this is rewritten on start. - - Without that volume the key is new on every start: logins keep working, but a user's schedules and - standby Actors cannot start runs until that user makes an API request again; the runtime warns. +- **Login tokens are never stored in plain text.** Tokens and proxy passwords in the data directory can + be read only with a key file the runtime creates there, readable by the runtime's own user and root + alone. A data directory written before this is rewritten on start. + - If the key file is deleted, logins keep working, but a user's schedules and standby Actors cannot + start runs until that user makes an API request again; the runtime warns. - **Docker and Podman are equally supported**, rootful or rootless: everything the system offers works the same on either engine. The user picks the engine by mounting its Docker-compatible API socket in place of the Docker one (e.g. `-v /run/podman/podman.sock:/var/run/docker.sock`). One exception: diff --git a/skills/actor-runtime/SKILL.md b/skills/actor-runtime/SKILL.md index a640a37e..c18d4b17 100644 --- a/skills/actor-runtime/SKILL.md +++ b/skills/actor-runtime/SKILL.md @@ -35,9 +35,10 @@ Any non-empty token authenticates - `apify login --token local-dev-token` is eno `APIFY_DISABLE_KEYRING=1` first in a sandbox with no OS keyring. To act as a second user, pass a different token on a single call: `apify api v2/datasets -H '{"authorization": "Bearer OTHER"}'`. -Tokens are never written to the data directory in plain text, so do not look for them there. If a -schedule or standby Actor fails with "the token of user ... cannot be read", the runtime lost its key -(started without its `/var/lib/actor-runtime` volume): any API call with that user's token fixes it. +Tokens are never written to the data directory in plain text, so do not look for them there. Never +delete `.token-vault.key` in the data directory: it is the key to them. If a schedule or standby Actor +fails with "the token of user ... cannot be read", it was deleted: any API call with that user's token +fixes it. ### Other ports diff --git a/src/config.ts b/src/config.ts index d2fca74f..ce72006d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -39,10 +39,6 @@ export const CONSOLE_BASE_URL = `http://localhost:${CONSOLE_PORT}`; export const DEFAULT_DATA_DIR = process.env.ACTOR_RUNTIME_DATA_DIR ?? '/data'; -/** Holds the key login tokens are sealed with (`services/token-vault.ts`). Never inside the data - * directory: in the image it is a Docker volume of its own, out of reach of a scan of the host's files. */ -export const KEY_DIR = process.env.ACTOR_RUNTIME_KEY_DIR ?? '/var/lib/actor-runtime'; - /** Where the Python debug-mode payload lives inside the runtime's own image. Read fresh on every call * (not cached) so tests can point `ACTOR_RUNTIME_DEBUGPY_PAYLOAD_DIR` at a fixture directory. */ function debugpyPayloadDir(): string { diff --git a/src/index.ts b/src/index.ts index 122f7e4c..d516dbb2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,20 +14,19 @@ import { startWebhookDispatcher } from './services/webhook-dispatches.js'; import { startScheduler } from './services/scheduler.js'; import { buildDto, runDto } from './api/dto/actors.js'; import { gracefulShutdown } from './shutdown.js'; -import { API_PORT, CONSOLE_PORT, DEFAULT_DATA_DIR, KEY_DIR } from './config.js'; +import { API_PORT, CONSOLE_PORT, DEFAULT_DATA_DIR } from './config.js'; import { initTokenVault } from './services/token-vault.js'; import { sealStoredUsers } from './services/users.js'; async function main(): Promise { - const vault = initTokenVault(KEY_DIR); + const vault = initTokenVault(DEFAULT_DATA_DIR); bootstrapStorage(DEFAULT_DATA_DIR); await openRegistries(); const { unreadable } = await sealStoredUsers(); if (vault.created && unreadable > 0) { console.warn( - `The key login tokens were sealed with is gone (${KEY_DIR} is new), so ${unreadable} user(s) need an ` + - 'API request with their token before their schedules and standby Actors can start runs. Mount a ' + - `named volume at ${KEY_DIR} (-v actor-runtime-keys:${KEY_DIR}) to keep the key across restarts.`, + `The runtime's key file was missing, so the tokens of ${unreadable} user(s) cannot be read. Their ` + + 'schedules and standby Actors start runs again after that user makes an API request with their token.', ); } // No default user is created here any more - users are created ad-hoc, per previously-unseen token, diff --git a/src/services/token-vault.ts b/src/services/token-vault.ts index 709f1f62..1ebf26b7 100644 --- a/src/services/token-vault.ts +++ b/src/services/token-vault.ts @@ -3,13 +3,13 @@ import { chmodSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; /** - * Keeps login tokens out of the data directory in plain text (`storage.md`): a token is stored only as - * its SHA-256 (enough to recognise a login) and sealed with AES-256-GCM under a key that lives in its own - * directory, never inside the data directory. In the image that directory is a Docker volume, so a scan of - * the host's files finds neither the tokens nor the key that opens them. + * Keeps login tokens out of the data directory in plain text (`system.md`): a token is stored only as its + * SHA-256 (enough to recognise a login) and sealed with AES-256-GCM under a key file the runtime creates in + * the data directory, readable only by the runtime's own user. Root and that user can still open the + * tokens; any other user on the host, reading the rest of the data directory, cannot. */ -const KEY_FILE = 'token-vault.key'; +const KEY_FILE = '.token-vault.key'; const KEY_BYTES = 32; const IV_BYTES = 12; const TAG_BYTES = 16; @@ -17,14 +17,16 @@ const SEALED_PREFIX = 'v1:'; let key: Buffer | undefined; -/** Loads the key from `keyDir`, creating it on first start. `created` with users already stored means the - * previous key is gone (e.g. the key volume was not mounted), so their sealed tokens no longer open. */ -export function initTokenVault(keyDir: string): { created: boolean } { - mkdirSync(keyDir, { recursive: true, mode: 0o700 }); - const path = join(keyDir, KEY_FILE); +/** Loads the key from `dataDir`, creating it on first start. `created` with users already stored means the + * previous key file was deleted, so their sealed tokens no longer open. */ +export function initTokenVault(dataDir: string): { created: boolean } { + mkdirSync(dataDir, { recursive: true }); + const path = join(dataDir, KEY_FILE); try { const existing = readFileSync(path); if (existing.length !== KEY_BYTES) throw new Error(`${path} is not a ${KEY_BYTES}-byte key`); + // Re-tightened on every start: a copy or restore of the data directory may have loosened it. + chmodSync(path, 0o600); key = existing; return { created: false }; } catch (error) { @@ -33,6 +35,7 @@ export function initTokenVault(keyDir: string): { created: boolean } { const fresh = randomBytes(KEY_BYTES); // `wx`: never overwrite a key another process wrote in the meantime - that would orphan its tokens. writeFileSync(path, fresh, { mode: 0o600, flag: 'wx' }); + // `mode` above is narrowed by the umask only, never widened; this pins it whatever the umask. chmodSync(path, 0o600); key = fresh; return { created: true }; diff --git a/src/services/users.ts b/src/services/users.ts index 5aa82b3d..00822566 100644 --- a/src/services/users.ts +++ b/src/services/users.ts @@ -107,7 +107,7 @@ export async function getOrCreateUserForToken(token: string): Promise { const { users } = getRegistries(); @@ -163,7 +163,7 @@ export async function sealStoredUsers(): Promise<{ unreadable: number }> { export function tokenOf(user: UserRecord): string { if (user.token !== undefined) return user.token; throw new Error( - `The token of user ${user.username} cannot be read: the runtime's key it was sealed with is gone. ` + + `The token of user ${user.username} cannot be read: the runtime's key file it was sealed with was deleted. ` + 'Any API request with that token restores it.', ); } diff --git a/src/storage/entities.ts b/src/storage/entities.ts index e3448b23..fcc5cde6 100644 --- a/src/storage/entities.ts +++ b/src/storage/entities.ts @@ -39,7 +39,7 @@ export interface UserRecord { id: string; username: string; /** The token seen last, handed to new runs, schedules and standby links. Never written to disk as is - * (`services/user-codec.ts`). Absent when it was sealed under a runtime key that is gone, until the + * (`services/user-codec.ts`). Absent when it was sealed under a key file that was deleted, until the * user's next request brings a token again. */ token?: string; /** SHA-256 of every token this user was seen with, oldest first - what a login is matched against. */ diff --git a/test/integration/token-sealing.test.ts b/test/integration/token-sealing.test.ts index 32a4662a..b3d70f57 100644 --- a/test/integration/token-sealing.test.ts +++ b/test/integration/token-sealing.test.ts @@ -1,6 +1,4 @@ -import { mkdtempSync } from 'node:fs'; -import { readdir, readFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; +import { chmod, readdir, readFile, rm, stat } from 'node:fs/promises'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { ApifyClient } from 'apify-client'; @@ -14,27 +12,34 @@ import { resetUsersForTests, sealStoredUsers, tokenOf } from '../../src/services async function filesContaining(dir: string, needle: string): Promise { const hits: string[] = []; for (const entry of await readdir(dir, { withFileTypes: true, recursive: true })) { - if (!entry.isFile()) continue; + if (!entry.isFile() || entry.name === '.token-vault.key') continue; const path = join(entry.parentPath, entry.name); if ((await readFile(path, 'utf8')).includes(needle)) hits.push(path); } return hits; } -const newKeyDir = () => mkdtempSync(join(tmpdir(), 'actor-runtime-test-key-')); - describe('login tokens at rest', () => { let server: TestServerHandle; beforeEach(async () => { - initTokenVault(newKeyDir()); server = await startTestServer(undefined, 'apify_api_never-on-disk-0123456789'); + initTokenVault(server.dataDir); }); afterEach(async () => { await server.close(); }); + it('the key file sits in the data directory, readable by its owner only, even after its mode was loosened', async () => { + const keyFile = join(server.dataDir, '.token-vault.key'); + expect((await stat(keyFile)).mode & 0o777).toBe(0o600); + + await chmod(keyFile, 0o644); + expect(initTokenVault(server.dataDir).created).toBe(false); + expect((await stat(keyFile)).mode & 0o777).toBe(0o600); + }); + it('a token, current or earlier, and a proxy password never appear in the data directory', async () => { const me = await server.client.user('me').get(); await getRegistries().users.update(me.id, (user) => user && { ...user, proxyPassword: 'proxy-secret-42' }); @@ -73,7 +78,8 @@ describe('login tokens at rest', () => { const me = await server.client.user('me').get(); resetUsersForTests(); - expect(initTokenVault(newKeyDir()).created).toBe(true); + await rm(join(server.dataDir, '.token-vault.key')); + expect(initTokenVault(server.dataDir).created).toBe(true); expect(await sealStoredUsers()).toEqual({ unreadable: 1 }); const orphaned = await getRegistries().users.get(me.id); expect(orphaned?.token).toBeUndefined(); From e6697f7e77ba6f2728179463ed65b63c951e0bbe Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 18 Nov 2026 08:05:57 +0000 Subject: [PATCH 3/3] feat: seal Actors' secret passphrases with the runtime's key file The passphrase protecting each Actor's private key is no longer stored in plain text, so the data directory alone no longer opens secret env vars or secret input. Existing records are rewritten on start; only outdated ones are touched. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017ocuAKna7JTHKG2TiQi5qH --- requirements/system.md | 10 +- skills/actor-runtime/SKILL.md | 8 +- src/index.ts | 4 +- src/services/actor-codec.ts | 38 +++++++ src/services/secrets.ts | 21 +++- src/services/user-codec.ts | 5 +- src/services/users.ts | 18 ++- src/storage/entities.ts | 2 +- src/storage/registries.ts | 3 +- src/storage/registry.ts | 21 ++++ test/integration/actor-secret-sealing.test.ts | 106 ++++++++++++++++++ test/integration/token-sealing.test.ts | 6 +- 12 files changed, 210 insertions(+), 32 deletions(-) create mode 100644 src/services/actor-codec.ts create mode 100644 test/integration/actor-secret-sealing.test.ts diff --git a/requirements/system.md b/requirements/system.md index 810b29b7..2799908a 100644 --- a/requirements/system.md +++ b/requirements/system.md @@ -50,11 +50,13 @@ actor-runtime ``` -- **Login tokens are never stored in plain text.** Tokens and proxy passwords in the data directory can - be read only with a key file the runtime creates there, readable by the runtime's own user and root - alone. A data directory written before this is rewritten on start. +- **Credentials are never stored in plain text.** Tokens, proxy passwords and the keys to Actors' + secret env vars and secret input in the data directory can be read only with a key file the runtime + creates there, readable by the runtime's own user and root alone. A data directory written before this + is rewritten on start. - If the key file is deleted, logins keep working, but a user's schedules and standby Actors cannot - start runs until that user makes an API request again; the runtime warns. + start runs until that user makes an API request again; the runtime warns. Secret env vars set before + then must be set again. - **Docker and Podman are equally supported**, rootful or rootless: everything the system offers works the same on either engine. The user picks the engine by mounting its Docker-compatible API socket in place of the Docker one (e.g. `-v /run/podman/podman.sock:/var/run/docker.sock`). One exception: diff --git a/skills/actor-runtime/SKILL.md b/skills/actor-runtime/SKILL.md index c18d4b17..a54ad371 100644 --- a/skills/actor-runtime/SKILL.md +++ b/skills/actor-runtime/SKILL.md @@ -35,10 +35,10 @@ Any non-empty token authenticates - `apify login --token local-dev-token` is eno `APIFY_DISABLE_KEYRING=1` first in a sandbox with no OS keyring. To act as a second user, pass a different token on a single call: `apify api v2/datasets -H '{"authorization": "Bearer OTHER"}'`. -Tokens are never written to the data directory in plain text, so do not look for them there. Never -delete `.token-vault.key` in the data directory: it is the key to them. If a schedule or standby Actor -fails with "the token of user ... cannot be read", it was deleted: any API call with that user's token -fixes it. +Tokens and secret env vars are never readable in the data directory, so do not look for them there. +Never delete `.token-vault.key` in the data directory: it is the key to them. If it was deleted, a +schedule or standby Actor fails with "the token of user ... cannot be read" until any API call with that +user's token, and a run with "Secret env var ... cannot be decrypted" until that env var is set again. ### Other ports diff --git a/src/index.ts b/src/index.ts index d516dbb2..793257ee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,13 +16,13 @@ import { buildDto, runDto } from './api/dto/actors.js'; import { gracefulShutdown } from './shutdown.js'; import { API_PORT, CONSOLE_PORT, DEFAULT_DATA_DIR } from './config.js'; import { initTokenVault } from './services/token-vault.js'; -import { sealStoredUsers } from './services/users.js'; +import { sealStoredCredentials } from './services/users.js'; async function main(): Promise { const vault = initTokenVault(DEFAULT_DATA_DIR); bootstrapStorage(DEFAULT_DATA_DIR); await openRegistries(); - const { unreadable } = await sealStoredUsers(); + const { unreadableUsers: unreadable } = await sealStoredCredentials(); if (vault.created && unreadable > 0) { console.warn( `The runtime's key file was missing, so the tokens of ${unreadable} user(s) cannot be read. Their ` + diff --git a/src/services/actor-codec.ts b/src/services/actor-codec.ts new file mode 100644 index 00000000..87c9c7ad --- /dev/null +++ b/src/services/actor-codec.ts @@ -0,0 +1,38 @@ +import type { RecordCodec } from '../storage/registry.js'; +import type { ActorRecord, ActorSecretKeys } from '../storage/entities.js'; +import { seal, unseal } from './token-vault.js'; + +/** `secretKeys` as written to disk: the private key is already protected by the passphrase, so sealing the + * passphrase alone keeps the Actor's secrets unreadable to anyone without the runtime's key file. */ +interface StoredSecretKeys { + publicKey: string; + privateKey: string; + sealedPassphrase: string; +} + +type StoredActor = Omit & { secretKeys?: StoredSecretKeys | ActorSecretKeys }; + +export const actorCodec: RecordCodec = { + isOutdated: (stored) => { + const { secretKeys } = stored as StoredActor; + return secretKeys !== undefined && 'passphrase' in secretKeys; + }, + encode(actor): StoredActor { + if (!actor.secretKeys) return actor; + const { publicKey, privateKey, passphrase } = actor.secretKeys; + return { ...actor, secretKeys: { publicKey, privateKey, sealedPassphrase: seal(passphrase, actor.id) } }; + }, + decode(stored): ActorRecord { + const { secretKeys, ...actor } = stored as StoredActor; + // Written before passphrases were sealed: read as is, rewritten sealed on start (`Registry.rewriteOutdated`). + if (!secretKeys || 'passphrase' in secretKeys) return stored as ActorRecord; + const passphrase = unseal(secretKeys.sealedPassphrase, actor.id); + // Unreadable once the key file is deleted: a key pair that cannot open is no key pair, so a new one + // is made on the next write or run (`secrets.ts: ensureSecretKeys`). + if (typeof passphrase !== 'string') return actor; + return { + ...actor, + secretKeys: { publicKey: secretKeys.publicKey, privateKey: secretKeys.privateKey, passphrase }, + }; + }, +}; diff --git a/src/services/secrets.ts b/src/services/secrets.ts index efc20f5e..063099d6 100644 --- a/src/services/secrets.ts +++ b/src/services/secrets.ts @@ -107,17 +107,28 @@ export function inputSecretsEnv(secretKeys: ActorSecretKeys): Record envVar.isSecret && envVar.encryptedAes256Password); if (sealed.length === 0) return envVars ?? []; - if (!actor.secretKeys) throw new Error(`Actor ${actor.id} has an encrypted secret but no key pair`); + const cannotDecrypt = (name: string) => + new Error( + `Secret env var "${name}" of Actor ${actor.id} cannot be decrypted: the runtime's key file was deleted. ` + + 'Set it again.', + ); + // No key pair: the one it was sealed with no longer opens (`actor-codec.ts`). + if (!actor.secretKeys) throw cannotDecrypt(sealed[0]!.name); const privateKey = createPrivateKey({ key: actor.secretKeys.privateKey, passphrase: actor.secretKeys.passphrase }); return (envVars ?? []).map((envVar) => { if (!sealed.includes(envVar)) return envVar; - const opened = JSON.parse( - privateDecrypt({ + let plain: string; + try { + plain = privateDecrypt({ privateKey, encryptedPassword: envVar.encryptedAes256Password!, encryptedValue: envVar.value, - }), - ) as { actId?: string; name?: string; value?: string }; + }); + } catch { + // Sealed under a key pair replaced since (`ensureSecretKeys`), after the old one stopped opening. + throw cannotDecrypt(envVar.name); + } + const opened = JSON.parse(plain) as { actId?: string; name?: string; value?: string }; if (opened.actId !== actor.id || opened.name !== envVar.name || typeof opened.value !== 'string') { throw new Error(`Secret env var "${envVar.name}" was not encrypted for Actor ${actor.id}`); } diff --git a/src/services/user-codec.ts b/src/services/user-codec.ts index 4c45e83f..5e89ace7 100644 --- a/src/services/user-codec.ts +++ b/src/services/user-codec.ts @@ -16,7 +16,7 @@ interface SealedCredentials { proxyPassword?: string; } -/** A record written before tokens were sealed, read once and rewritten sealed (`sealStoredUsers`). */ +/** A record written before tokens were sealed, rewritten sealed on start (`Registry.rewriteOutdated`). */ interface LegacyUser { id: string; username: string; @@ -26,11 +26,12 @@ interface LegacyUser { proxyPassword?: string; } -export function isLegacyUser(stored: unknown): stored is LegacyUser { +function isLegacyUser(stored: unknown): stored is LegacyUser { return typeof (stored as LegacyUser).token === 'string'; } export const userCodec: RecordCodec = { + isOutdated: isLegacyUser, encode({ id, username, createdAt, tokenHashes, token, proxyPassword }): StoredUser { const credentials: SealedCredentials = { token, proxyPassword }; return { id, username, createdAt, tokenHashes, sealed: seal(credentials, id) }; diff --git a/src/services/users.ts b/src/services/users.ts index 00822566..e3274d11 100644 --- a/src/services/users.ts +++ b/src/services/users.ts @@ -145,18 +145,16 @@ export async function getOrCreateUserForToken(token: string): Promise { - const { users } = getRegistries(); - let unreadable = 0; - for (const user of await users.list()) { - if (user.token === undefined) unreadable++; - await users.update(user.id, (current) => current); - } - return { unreadable }; +export async function sealStoredCredentials(): Promise<{ unreadableUsers: number }> { + const { users, actors } = getRegistries(); + await users.rewriteOutdated(); + await actors.rewriteOutdated(); + const unreadableUsers = (await users.list()).filter((user) => user.token === undefined).length; + return { unreadableUsers }; } /** The user's token for work the runtime starts on its own (schedules, standby). */ diff --git a/src/storage/entities.ts b/src/storage/entities.ts index fcc5cde6..da37ea3b 100644 --- a/src/storage/entities.ts +++ b/src/storage/entities.ts @@ -83,7 +83,7 @@ export interface ActorEnvVarRecord { } /** PEM-encoded, the private key protected by `passphrase`; created with the Actor and never returned by - * the API. */ + * the API. `passphrase` is never written to disk as is (`services/actor-codec.ts`). */ export interface ActorSecretKeys { publicKey: string; privateKey: string; diff --git a/src/storage/registries.ts b/src/storage/registries.ts index e1bc3fa6..66caac5d 100644 --- a/src/storage/registries.ts +++ b/src/storage/registries.ts @@ -5,6 +5,7 @@ */ import { Registry } from './registry.js'; import { openKeyValueStore } from './open.js'; +import { actorCodec } from '../services/actor-codec.js'; import { userCodec } from '../services/user-codec.js'; import type { ActorRecord, @@ -40,7 +41,7 @@ export async function openRegistries(): Promise { current = { storages: await Registry.open('__STORAGES__'), users: await Registry.open('__USERS__', userCodec), - actors: await Registry.open('__ACTORS__'), + actors: await Registry.open('__ACTORS__', actorCodec), runs: await Registry.open('__RUNS__'), builds: await Registry.open('__BUILDS__'), tasks: await Registry.open('__TASKS__'), diff --git a/src/storage/registry.ts b/src/storage/registry.ts index cb54c17f..5adf7548 100644 --- a/src/storage/registry.ts +++ b/src/storage/registry.ts @@ -18,6 +18,8 @@ export type RegistryChangeListener = (previous: T | null, next: T | null) => export interface RecordCodec { encode(value: T): unknown; decode(stored: unknown): T; + /** A record still in an older on-disk form, which `rewriteOutdated` brings up to date. */ + isOutdated?(stored: unknown): boolean; } const identityCodec: RecordCodec = { encode: (value) => value, decode: (stored) => stored as never }; @@ -93,6 +95,25 @@ export class Registry { } } + /** Rewrites every record the codec reports outdated. Only those: a record that is current but no longer + * opens (its key file was deleted) is left as it is, in case that file comes back. */ + async rewriteOutdated(): Promise { + const { isOutdated } = this.codec; + if (!isOutdated) return; + const ids: string[] = []; + await this.store.forEachKey(async (id) => { + ids.push(id); + }); + for (const id of ids) { + await this.mutex.run(id, async () => { + const stored = await this.store.getValue(id); + if (stored !== null && stored !== undefined && isOutdated(stored)) { + await this.store.setValue(id, this.codec.encode(this.codec.decode(stored))); + } + }); + } + } + /** All non-deleted records, in no particular order. Fine at POC scale (<100 records per type). */ async list(): Promise { const ids: string[] = []; diff --git a/test/integration/actor-secret-sealing.test.ts b/test/integration/actor-secret-sealing.test.ts new file mode 100644 index 00000000..59e55bc7 --- /dev/null +++ b/test/integration/actor-secret-sealing.test.ts @@ -0,0 +1,106 @@ +import { readdir, readFile, rm } from 'node:fs/promises'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { startTestServer, type TestServerHandle } from './helpers/test-server.js'; +import { getRegistries } from '../../src/storage/registries.js'; +import { openKeyValueStore } from '../../src/storage/open.js'; +import { initTokenVault } from '../../src/services/token-vault.js'; +import { sealStoredCredentials } from '../../src/services/users.js'; +import { decryptedEnvVars, ensureSecretKeys } from '../../src/services/secrets.js'; + +async function filesContaining(dir: string, needle: string): Promise { + const hits: string[] = []; + for (const entry of await readdir(dir, { withFileTypes: true, recursive: true })) { + if (!entry.isFile() || entry.name === '.token-vault.key') continue; + const path = join(entry.parentPath, entry.name); + if ((await readFile(path, 'utf8')).includes(needle)) hits.push(path); + } + return hits; +} + +const SECRET_VALUE = 'my-secret-env-value-123'; + +interface RawActor { + modifiedAt: string; + secretKeys: { passphrase?: string; sealedPassphrase?: string }; +} + +describe("Actors' secret passphrases at rest", () => { + let server: TestServerHandle; + let actorId: string; + + const rawActor = async () => (await (await openKeyValueStore('__ACTORS__')).getValue(actorId)) as RawActor; + const actor = async () => (await getRegistries().actors.get(actorId))!; + + beforeEach(async () => { + server = await startTestServer(); + initTokenVault(server.dataDir); + const created = await server.client.actors().create({ + name: 'secret-actor', + versions: [ + { + versionNumber: '0.0', + sourceType: 'SOURCE_FILES', + buildTag: 'latest', + sourceFiles: [], + envVars: [{ name: 'MY_SECRET', value: SECRET_VALUE, isSecret: true }], + }, + ], + } as never); + actorId = created.id; + }); + + afterEach(async () => { + await server.close(); + }); + + const secretValue = async () => decryptedEnvVars(await actor(), (await actor()).versions[0]!.envVars)[0]!.value; + + it('the passphrase is stored sealed, never in plain text, and the secret still decrypts', async () => { + const { passphrase } = (await actor()).secretKeys!; + const stored = await rawActor(); + + expect(stored.secretKeys.passphrase).toBeUndefined(); + expect(stored.secretKeys.sealedPassphrase).toMatch(/^v1:/); + expect(await filesContaining(server.dataDir, passphrase)).toEqual([]); + expect(await filesContaining(server.dataDir, SECRET_VALUE)).toEqual([]); + expect(await secretValue()).toBe(SECRET_VALUE); + }); + + it('an Actor stored with a plain-text passphrase is rewritten sealed on start, modifiedAt untouched', async () => { + const decoded = await actor(); + await (await openKeyValueStore('__ACTORS__')).setValue(actorId, decoded); + expect((await rawActor()).secretKeys.passphrase).toBe(decoded.secretKeys!.passphrase); + + await sealStoredCredentials(); + + const stored = await rawActor(); + expect(stored.secretKeys.passphrase).toBeUndefined(); + expect(stored.modifiedAt).toBe(decoded.modifiedAt); + expect(await filesContaining(server.dataDir, decoded.secretKeys!.passphrase)).toEqual([]); + expect(await secretValue()).toBe(SECRET_VALUE); + }); + + it('with the key file deleted, the secret fails with a clear error, and the start leaves the record alone', async () => { + const before = await rawActor(); + await rm(join(server.dataDir, '.token-vault.key')); + initTokenVault(server.dataDir); + + await sealStoredCredentials(); + expect(await rawActor()).toEqual(before); + expect((await actor()).secretKeys).toBeUndefined(); + await expect(secretValue()).rejects.toThrow(/MY_SECRET.*cannot be decrypted.*Set it again/); + + // A new key pair (made by the next run or write) does not open the old secret either - same error. + const rekeyed = await ensureSecretKeys(await actor()); + expect(() => decryptedEnvVars(rekeyed, rekeyed.versions[0]!.envVars)).toThrow(/cannot be decrypted/); + }); + + it('a sealed passphrase copied onto another Actor does not open there', async () => { + const store = await openKeyValueStore('__ACTORS__'); + await store.setValue('other-actor-id', { ...(await rawActor()), id: 'other-actor-id' }); + + expect((await getRegistries().actors.get('other-actor-id'))?.secretKeys).toBeUndefined(); + }); +}); diff --git a/test/integration/token-sealing.test.ts b/test/integration/token-sealing.test.ts index b3d70f57..df348556 100644 --- a/test/integration/token-sealing.test.ts +++ b/test/integration/token-sealing.test.ts @@ -7,7 +7,7 @@ import { startTestServer, type TestServerHandle } from './helpers/test-server.js import { getRegistries } from '../../src/storage/registries.js'; import { openKeyValueStore } from '../../src/storage/open.js'; import { initTokenVault } from '../../src/services/token-vault.js'; -import { resetUsersForTests, sealStoredUsers, tokenOf } from '../../src/services/users.js'; +import { resetUsersForTests, sealStoredCredentials, tokenOf } from '../../src/services/users.js'; async function filesContaining(dir: string, needle: string): Promise { const hits: string[] = []; @@ -63,7 +63,7 @@ describe('login tokens at rest', () => { }; await (await openKeyValueStore('__USERS__')).setValue(legacy.id, legacy); - await sealStoredUsers(); + await sealStoredCredentials(); for (const secret of ['legacy-plain-token', 'legacy-older-token', 'legacy-proxy']) { expect(await filesContaining(server.dataDir, secret)).toEqual([]); @@ -80,7 +80,7 @@ describe('login tokens at rest', () => { resetUsersForTests(); await rm(join(server.dataDir, '.token-vault.key')); expect(initTokenVault(server.dataDir).created).toBe(true); - expect(await sealStoredUsers()).toEqual({ unreadable: 1 }); + expect(await sealStoredCredentials()).toEqual({ unreadableUsers: 1 }); const orphaned = await getRegistries().users.get(me.id); expect(orphaned?.token).toBeUndefined(); expect(() => tokenOf(orphaned!)).toThrow(/cannot be read/);