diff --git a/.devcontainer/compose.yml b/.devcontainer/compose.yml index fcf61df4..2aa586bd 100644 --- a/.devcontainer/compose.yml +++ b/.devcontainer/compose.yml @@ -83,7 +83,6 @@ services: dragonfly --maxmemory=128mb --proactor_threads=1 - --cache_mode=true --default_lua_flags=allow-undeclared-keys ports: - "6379:6379" diff --git a/.env.example b/.env.example index b5bc00bb..205d814b 100644 --- a/.env.example +++ b/.env.example @@ -219,8 +219,10 @@ ENABLE_REGISTRATION=true # Redis URL for caching and job queues # Required for ActivityPub delivery queue in production -# (DRAGONFLY_URL is also accepted as a fallback) -REDIS_URL=redis://localhost:6379 +# Defaults to redis://localhost:6379. DRAGONFLY_URL is also accepted as a +# fallback; docker/compose.yml and the devcontainer set DRAGONFLY_URL, so leave +# this commented out there (REDIS_URL takes precedence over DRAGONFLY_URL) +# REDIS_URL=redis://localhost:6379 # Disable queue and use synchronous delivery (default: false) # Set to "true" to disable BullMQ queue (useful when Redis/Dragonfly has issues) diff --git a/bun.lock b/bun.lock index 2ade59e4..c3d8cca8 100644 --- a/bun.lock +++ b/bun.lock @@ -25,7 +25,6 @@ "@sentry/bun": "^11.7.0", "@sentry/core": "11.7.0", "@simplewebauthn/server": "^14.0.3", - "@simplewebauthn/types": "^12.0.0", "blurhash": "^2.0.5", "bullmq": "^6.3.14", "drizzle-orm": "^0.45.4", @@ -691,8 +690,6 @@ "@simplewebauthn/server": ["@simplewebauthn/server@14.0.3", "", { "dependencies": { "@hexagon/base64": "^1.1.27", "@levischuck/tiny-cbor": "^0.2.2", "@peculiar/asn1-android": "^2.6.0", "@peculiar/asn1-ecc": "^2.6.1", "@peculiar/asn1-rsa": "^2.6.1", "@peculiar/asn1-schema": "^2.6.0", "@peculiar/asn1-x509": "^2.6.1", "@peculiar/asn1-x509-post-quantum": "^2.9.4", "@peculiar/x509": "^2.1.0", "reflect-metadata": "^0.2.2" } }, "sha512-D39/0fiDiYmGV+nraY9s63D1Ho2AX/waMLy22/9gUadTUuvbAnstfb4bpbHYbwZuV5ZLDYN1SmzDKuiHzKAA4Q=="], - "@simplewebauthn/types": ["@simplewebauthn/types@12.0.0", "", {}, "sha512-q6y8MkoV8V8jB4zzp18Uyj2I7oFp2/ONL8c3j8uT06AOWu3cIChc1au71QYHrP2b+xDapkGTiv+9lX7xkTlAsA=="], - "@sinclair/typebox": ["@sinclair/typebox@0.27.12", "", {}, "sha512-hhyNJ+nbR6ZR7pToHvllEFun9TL0sbL+tk/ON75lo+Xas054uez98qRbsuNt7MBCyZKK4+8Yli/OAGZhmfBZ/g=="], "@smithy/core": ["@smithy/core@3.35.2", "", { "dependencies": { "@smithy/types": "^4.19.0", "tslib": "^2.6.2" } }, "sha512-ejTukBuP/Pg3t9N8Ux+0zKsXeHWm7p6W3HbXFOyltKasSxZwCFmvTut7kBdZSCF3m9y1bXXyVu6p2xuMFcsTyQ=="], diff --git a/docker/compose.dev.yml b/docker/compose.dev.yml index 2cea1697..7417334c 100644 --- a/docker/compose.dev.yml +++ b/docker/compose.dev.yml @@ -51,7 +51,6 @@ services: dragonfly --maxmemory=128mb --proactor_threads=1 - --cache_mode=true --default_lua_flags=allow-undeclared-keys ports: - "6379:6379" diff --git a/docker/compose.yml b/docker/compose.yml index c663e584..826a3b38 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -41,7 +41,7 @@ services: postgres: condition: service_healthy dragonfly: - condition: service_started + condition: service_healthy environment: - NODE_ENV=production - PORT=3000 @@ -195,7 +195,6 @@ services: dragonfly --maxmemory=256mb --proactor_threads=2 - --cache_mode=true --default_lua_flags=allow-undeclared-keys volumes: - dragonfly-data:/data diff --git a/packages/backend/package.json b/packages/backend/package.json index bc2a8620..3ac35ce7 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -25,7 +25,6 @@ "@sentry/bun": "^11.7.0", "@sentry/core": "11.7.0", "@simplewebauthn/server": "^14.0.3", - "@simplewebauthn/types": "^12.0.0", "blurhash": "^2.0.5", "bullmq": "^6.3.14", "drizzle-orm": "^0.45.4", diff --git a/packages/backend/src/adapters/cache/DragonflyCacheAdapter.ts b/packages/backend/src/adapters/cache/DragonflyCacheAdapter.ts index c1746103..3135b67d 100644 --- a/packages/backend/src/adapters/cache/DragonflyCacheAdapter.ts +++ b/packages/backend/src/adapters/cache/DragonflyCacheAdapter.ts @@ -2,14 +2,15 @@ * Dragonfly Cache Adapter * * Implements ICacheService using Dragonfly (Redis-compatible) as the backend. - * Uses ioredis for connection management. + * Uses ioredis for connection management (see lib/redis.ts). * * @module adapters/cache/DragonflyCacheAdapter */ -import { Redis } from "ioredis"; +import type { Redis } from "ioredis"; import type { ICacheService, CacheSetOptions } from "../../interfaces/ICacheService.js"; import { logger } from "../../lib/logger.js"; +import { createRedisClient, waitForRedisReady } from "../../lib/redis.js"; import { recordCacheOperation } from "../../lib/metrics.js"; /** @@ -61,31 +62,22 @@ export class DragonflyCacheAdapter implements ICacheService { * Initialize Redis connection */ private async initialize(): Promise { - // docker/compose.yml と K8s secret は DRAGONFLY_URL を設定しているため両方を受け付ける - const redisUrl = process.env.REDIS_URL || process.env.DRAGONFLY_URL || "redis://localhost:6379"; - try { - this.redis = new Redis(redisUrl, { + this.redis = createRedisClient("cache", { maxRetriesPerRequest: 3, - // ioredis v6 は既定で RESP3 になったため、v5 と同じ RESP2 を明示（Dragonfly/BullMQ 互換性のため） - protocol: 2, - retryStrategy: (times) => { - if (times > 3) return null; - return Math.min(times * 100, 1000); - }, - connectTimeout: 5000, - enableOfflineQueue: false, - lazyConnect: true, + enableOfflineQueue: false, // Cache misses while disconnected instead of queueing commands }); - await this.redis.connect(); - await this.redis.ping(); + // 起動直後は Dragonfly がまだ接続を受け付けていないことがあるため、一定時間は再試行を待つ + await waitForRedisReady(this.redis); this.available = true; - logger.debug("Dragonfly cache connected"); + logger.info("Dragonfly cache connected"); } catch (error) { logger.warn({ err: error }, "Dragonfly cache not available, caching disabled"); this.available = false; + // 失敗した接続が裏で再接続を続けないように切断する + this.redis?.disconnect(); this.redis = null; } } diff --git a/packages/backend/src/lib/redis.ts b/packages/backend/src/lib/redis.ts new file mode 100644 index 00000000..aca69a63 --- /dev/null +++ b/packages/backend/src/lib/redis.ts @@ -0,0 +1,140 @@ +/** + * Redis / Dragonfly Connection Helpers + * + * Shared connection settings for the Dragonfly cache adapter and the + * ActivityPub delivery queue, so both always talk to the same server with + * the same protocol and reconnect behavior. + * + * @module lib/redis + */ + +import { Redis, type RedisOptions } from "ioredis"; +import { logger } from "./logger.js"; + +/** Default Redis URL when neither `REDIS_URL` nor `DRAGONFLY_URL` is set. */ +const DEFAULT_REDIS_URL = "redis://localhost:6379"; + +/** Upper bound for the delay between reconnect attempts. */ +const MAX_RECONNECT_DELAY_MS = 5000; + +/** How long to wait for the first connection at startup. */ +export const DEFAULT_STARTUP_TIMEOUT_MS = 15000; + +/** + * Resolve the Redis connection URL. + * + * `REDIS_URL` takes precedence; `DRAGONFLY_URL` is accepted as a fallback + * because docker/compose.yml and the K8s secret set that name. + * + * @returns Redis connection URL + */ +export function getRedisUrl(): string { + return process.env.REDIS_URL || process.env.DRAGONFLY_URL || DEFAULT_REDIS_URL; +} + +/** + * Strip credentials from a Redis URL so it can be logged safely. + * + * @param url - Redis connection URL (may contain `user:password@`) + * @returns URL with username and password replaced by `***` + */ +export function redactRedisUrl(url: string): string { + try { + const parsed = new URL(url); + if (parsed.username) parsed.username = "***"; + if (parsed.password) parsed.password = "***"; + return parsed.toString(); + } catch { + return ""; + } +} + +/** + * Create an ioredis client with the shared defaults. + * + * Defaults: + * - `protocol: 2`: ioredis v6 defaults to RESP3; RESP2 keeps v5 behavior + * for Dragonfly/BullMQ compatibility. + * - Never gives up reconnecting (capped backoff), so a Dragonfly restart + * does not permanently disable the cache or the delivery queue. + * - Logs a warning when an established connection drops and an info + * message when it comes back. + * + * @param name - Label used in log messages (e.g. "cache", "queue") + * @param options - ioredis options merged over the defaults + * @returns ioredis client (connecting unless `lazyConnect` is set) + */ +export function createRedisClient(name: string, options: RedisOptions = {}): Redis { + const redis = new Redis(getRedisUrl(), { + protocol: 2, + connectTimeout: 5000, + retryStrategy: (times) => Math.min(times * 200, MAX_RECONNECT_DELAY_MS), + ...options, + }); + + // Log once per outage. "reconnecting" (unlike "close") does not fire on a + // deliberate quit()/disconnect(), so graceful shutdown stays quiet. + let everReady = false; + let connected = false; + redis.on("ready", () => { + if (everReady) { + logger.info({ client: name }, "Redis connection restored"); + } + everReady = true; + connected = true; + }); + redis.on("reconnecting", () => { + if (connected) { + logger.warn({ client: name }, "Redis connection lost, reconnecting"); + connected = false; + } + }); + // Without an error listener ioredis prints "Unhandled error event" on + // every reconnect attempt. Connection state is reported via ready/close. + redis.on("error", (err) => { + logger.debug({ err, client: name }, "Redis connection error"); + }); + + return redis; +} + +/** + * Wait until a client is ready, tolerating transient connection errors. + * + * Unlike listening for the first `error` event, this keeps waiting while + * ioredis retries, so a backend that starts before Dragonfly accepts + * connections still picks it up. + * + * @param redis - Client to wait for + * @param timeoutMs - Maximum time to wait + * @throws Error if the client is not ready within `timeoutMs` + */ +export async function waitForRedisReady( + redis: Redis, + timeoutMs: number = DEFAULT_STARTUP_TIMEOUT_MS, +): Promise { + if (redis.status === "ready") return; + + await new Promise((resolve, reject) => { + const onReady = () => { + cleanup(); + resolve(); + }; + const onEnd = () => { + cleanup(); + reject(new Error("Redis connection ended before becoming ready")); + }; + const timer = setTimeout(() => { + cleanup(); + reject(new Error(`Redis not ready within ${timeoutMs}ms (${redactRedisUrl(getRedisUrl())})`)); + }, timeoutMs); + const cleanup = () => { + clearTimeout(timer); + redis.off("ready", onReady); + redis.off("end", onEnd); + }; + + redis.once("ready", onReady); + redis.once("end", onEnd); + }); +} diff --git a/packages/backend/src/services/PasskeyService.ts b/packages/backend/src/services/PasskeyService.ts index c6799b27..56ac751b 100644 --- a/packages/backend/src/services/PasskeyService.ts +++ b/packages/backend/src/services/PasskeyService.ts @@ -14,9 +14,9 @@ import { verifyAuthenticationResponse, type VerifiedRegistrationResponse, type VerifiedAuthenticationResponse, + type AuthenticatorTransport, } from "@simplewebauthn/server"; import { isoBase64URL } from "@simplewebauthn/server/helpers"; -import type { AuthenticatorTransportFuture } from "@simplewebauthn/types"; import type { IPasskeyCredentialRepository } from "../interfaces/repositories/IPasskeyCredentialRepository.js"; import type { IPasskeyChallengeRepository } from "../interfaces/repositories/IPasskeyChallengeRepository.js"; import type { IUserRepository } from "../interfaces/repositories/IUserRepository.js"; @@ -144,7 +144,7 @@ export class PasskeyService { attestationType: "none", // Don't require attestation excludeCredentials: existingPasskeys.map((passkey) => ({ id: passkey.credentialId, - transports: (passkey.transports as AuthenticatorTransportFuture[]) || undefined, + transports: (passkey.transports as AuthenticatorTransport[]) || undefined, })), authenticatorSelection: { residentKey: "required", // Required for passkeys @@ -283,9 +283,7 @@ export class PasskeyService { * @returns Authentication options to pass to browser */ async generateAuthenticationOptions(username?: string): Promise { - let allowCredentials: - | Array<{ id: string; transports?: AuthenticatorTransportFuture[] }> - | undefined; + let allowCredentials: Array<{ id: string; transports?: AuthenticatorTransport[] }> | undefined; if (username) { const user = await this.userRepository.findByUsername(username); @@ -294,7 +292,7 @@ export class PasskeyService { if (passkeys.length > 0) { allowCredentials = passkeys.map((passkey) => ({ id: passkey.credentialId, - transports: (passkey.transports as AuthenticatorTransportFuture[]) || undefined, + transports: (passkey.transports as AuthenticatorTransport[]) || undefined, })); } } @@ -395,7 +393,7 @@ export class PasskeyService { id: passkey.credentialId, publicKey: isoBase64URL.toBuffer(passkey.publicKey), counter: passkey.counter, - transports: passkey.transports as AuthenticatorTransportFuture[] | undefined, + transports: passkey.transports as AuthenticatorTransport[] | undefined, }, requireUserVerification: false, }); diff --git a/packages/backend/src/services/ap/ActivityDeliveryQueue.ts b/packages/backend/src/services/ap/ActivityDeliveryQueue.ts index efd704f6..f6541dbe 100644 --- a/packages/backend/src/services/ap/ActivityDeliveryQueue.ts +++ b/packages/backend/src/services/ap/ActivityDeliveryQueue.ts @@ -7,10 +7,12 @@ * @module services/ap/ActivityDeliveryQueue */ +import { createHash } from "node:crypto"; import { Queue, Worker, type Job } from "bullmq"; -import { Redis } from "ioredis"; +import type { Redis } from "ioredis"; import { ActivityDeliveryService } from "./ActivityDeliveryService.js"; import { logger } from "../../lib/logger.js"; +import { createRedisClient, waitForRedisReady } from "../../lib/redis.js"; /** * Job priority levels @@ -149,32 +151,18 @@ export class ActivityDeliveryQueue { return; } - // docker/compose.yml と K8s secret は DRAGONFLY_URL を設定しているため両方を受け付ける - const redisUrl = process.env.REDIS_URL || process.env.DRAGONFLY_URL || "redis://localhost:6379"; - try { - // Attempt to connect to Redis // Note: maxRetriesPerRequest must be null for BullMQ compatibility // BullMQ handles its own retry logic and requires this setting - this.redis = new Redis(redisUrl, { + this.redis = createRedisClient("queue", { maxRetriesPerRequest: null, // Required for BullMQ - // ioredis v6 は既定で RESP3 になったため、v5 と同じ RESP2 を明示（Dragonfly/BullMQ 互換性のため） - protocol: 2, - retryStrategy: (times) => { - if (times > 3) return null; // Give up after 3 attempts - return Math.min(times * 100, 1000); // Exponential backoff up to 1 second - }, - connectTimeout: 5000, - enableOfflineQueue: false, // Fail fast if connection is not ready + enableOfflineQueue: false, // Fail fast (and fall back to sync) while disconnected }); - // Wait for connection to be ready - await new Promise((resolve, reject) => { - this.redis!.once("ready", () => resolve()); - this.redis!.once("error", (err) => reject(err)); - }); + // 起動直後は Dragonfly がまだ接続を受け付けていないことがあるため、一定時間は再試行を待つ + await waitForRedisReady(this.redis); - logger.debug("Redis connected, using BullMQ for delivery queue"); + logger.info("Redis connected, using BullMQ for delivery queue"); // Initialize queue // Note: Using simple prefix without hash tags for Dragonfly compatibility @@ -356,7 +344,7 @@ export class ActivityDeliveryQueue { removeOnFail: { age: 24 * 3600, // Keep failed jobs for 24 hours }, - // Deduplication: if same activity to same inbox within 5 seconds, skip + // Deduplication: skipped while a job for the same activity and inbox exists jobId: this.generateJobId(data), }); @@ -364,7 +352,7 @@ export class ActivityDeliveryQueue { } catch (error) { // Queue operation failed (e.g., Dragonfly cluster mode issue) // Fall back to synchronous delivery - logger.debug({ err: error }, "Queue operation failed, falling back to sync delivery"); + logger.warn({ err: error }, "Queue operation failed, falling back to sync delivery"); await this.deliverSync(data); } } else { @@ -386,10 +374,12 @@ export class ActivityDeliveryQueue { /** * Generate unique job ID for deduplication * - * Creates a job ID based on activity ID and inbox URL to prevent - * duplicate deliveries within a short time window. - * Uses a simple numeric hash to avoid special characters that may - * cause issues with Dragonfly's Lua scripts. + * Creates a job ID based on activity ID and inbox URL so the same activity + * is not queued twice for the same inbox while a job with that ID exists. + * Uses a truncated SHA-256 hex digest (128 bits) to make collisions between + * different deliveries practically impossible, with a `d-` prefix so the ID + * is never all digits (BullMQ rejects integer-like custom IDs). Hex + * characters only, which is safe for Dragonfly's Lua scripts. * * @param data - Delivery job data * @returns Job ID for deduplication @@ -398,16 +388,8 @@ export class ActivityDeliveryQueue { */ private generateJobId(data: DeliveryJobData): string { const activityId = data.activity.id || JSON.stringify(data.activity); - // Use a simple hash that avoids special characters - // Create a numeric hash from the string for cluster-safe job ID - const input = `${activityId}-${data.inboxUrl}`; - let hash = 0; - for (let i = 0; i < input.length; i++) { - const char = input.charCodeAt(i); - hash = (hash << 5) - hash + char; - hash = hash & hash; // Convert to 32bit integer - } - return `${Math.abs(hash).toString(36)}`; + const digest = createHash("sha256").update(`${activityId}\n${data.inboxUrl}`).digest("hex"); + return `d-${digest.slice(0, 32)}`; } /** diff --git a/packages/backend/src/tests/unit/ActivityDeliveryQueue.test.ts b/packages/backend/src/tests/unit/ActivityDeliveryQueue.test.ts index 22361bb2..ab7b7a5f 100644 --- a/packages/backend/src/tests/unit/ActivityDeliveryQueue.test.ts +++ b/packages/backend/src/tests/unit/ActivityDeliveryQueue.test.ts @@ -16,6 +16,10 @@ import { type DeliveryJobData, } from "../../services/ap/ActivityDeliveryQueue"; +// These tests cover synchronous delivery mode. Disable the queue explicitly so +// the queue does not wait for a Redis connection during initialization. +process.env.USE_QUEUE = "false"; + // Mock the ActivityDeliveryService mock.module("../../services/ap/ActivityDeliveryService", () => ({ ActivityDeliveryService: class { @@ -329,11 +333,24 @@ describe("ActivityDeliveryQueue job ID generation", () => { privateKey: "secret", }; - // Should not throw and return a valid base36 string + // Should not throw and return a hex digest with the "d-" prefix const id = generateJobId(jobData); - expect(typeof id).toBe("string"); - expect(id.length).toBeGreaterThan(0); - // Verify it's a valid base36 string (alphanumeric, no special characters) - expect(/^[0-9a-z]+$/.test(id)).toBe(true); + expect(id).toMatch(/^d-[0-9a-f]{32}$/); + }); + + test("should never generate an integer-like job ID", () => { + const queue = new ActivityDeliveryQueue(false); + const generateJobId = (queue as any).generateJobId.bind(queue); + + // BullMQ rejects custom job IDs that parse as integers + for (let i = 0; i < 1000; i++) { + const id: string = generateJobId({ + activity: { id: `https://local.example/activities/${i}`, type: "Create" }, + inboxUrl: "https://remote.example/inbox", + keyId: "key", + privateKey: "secret", + }); + expect(String(Number.parseInt(id, 10))).not.toBe(id); + } }); }); diff --git a/packages/backend/src/tests/unit/redis.test.ts b/packages/backend/src/tests/unit/redis.test.ts new file mode 100644 index 00000000..d968c391 --- /dev/null +++ b/packages/backend/src/tests/unit/redis.test.ts @@ -0,0 +1,24 @@ +/** + * Redis helper unit tests + */ + +import { describe, test, expect } from "bun:test"; +import { redactRedisUrl } from "../../lib/redis"; + +describe("redactRedisUrl", () => { + test("should mask password-only credentials", () => { + expect(redactRedisUrl("redis://:s3cret@dragonfly:6379")).toBe("redis://:***@dragonfly:6379"); + }); + + test("should mask username and password", () => { + expect(redactRedisUrl("rediss://user:pw@host:6380/0")).toBe("rediss://***:***@host:6380/0"); + }); + + test("should leave URLs without credentials unchanged", () => { + expect(redactRedisUrl("redis://localhost:6379")).toBe("redis://localhost:6379"); + }); + + test("should not echo invalid input", () => { + expect(redactRedisUrl("not a url :secret")).toBe(""); + }); +});