From b77a7b6aa7c6cab979331466c4a2e548e76c3247 Mon Sep 17 00:00:00 2001 From: flakey5 <73616808+flakey5@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:30:00 -0700 Subject: [PATCH] src, scripts: support unofficial builds site TODO: - update scripts, mainly so that we're updating kv directory cache - checking that this didn't break anything would be good perchacne - ??? Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com> --- src/constants/release-config.ts | 19 ++++++++++++++ src/env.ts | 4 +++ src/middleware/r2Middleware.ts | 39 ++++++++++++++++++----------- src/providers/kvProvider.ts | 16 +++--------- src/providers/r2Provider.ts | 20 ++++++++------- src/routes/index.ts | 44 +++++++++++++++++++++++++++------ src/worker.ts | 32 +++++++++++++++++++++--- wrangler.jsonc | 23 +++++++++++++++++ 8 files changed, 150 insertions(+), 47 deletions(-) create mode 100644 src/constants/release-config.ts diff --git a/src/constants/release-config.ts b/src/constants/release-config.ts new file mode 100644 index 00000000..6677b75d --- /dev/null +++ b/src/constants/release-config.ts @@ -0,0 +1,19 @@ +import { env } from 'cloudflare:workers'; +import { type Env } from '../env'; + +const typedEnv = env as Env; + +export type ReleaseConfig = { + bucket: R2Bucket; + directoryCache: KVNamespace; +}; + +export const MAINLINE_RELEASE_CONFIG: ReleaseConfig = { + bucket: typedEnv.R2_BUCKET, + directoryCache: typedEnv.DIRECTORY_CACHE, +}; + +export const UNOFFICIAL_BUILDS_RELEASE_CONFIG: ReleaseConfig = { + bucket: typedEnv.UNOFFICIAL_BUILDS_BUCKET, + directoryCache: typedEnv.UNOFFICIAL_BUILDS_DIRECTORY_CACHE, +}; diff --git a/src/env.ts b/src/env.ts index ed4151b0..ee1b3671 100644 --- a/src/env.ts +++ b/src/env.ts @@ -16,6 +16,10 @@ export interface Env { */ DIRECTORY_CACHE: KVNamespace; + UNOFFICIAL_BUILDS_BUCKET: R2Bucket; + + UNOFFICIAL_BUILDS_DIRECTORY_CACHE: KVNamespace; + /** * Temp flag for whether or not to use KV instead of S3 */ diff --git a/src/middleware/r2Middleware.ts b/src/middleware/r2Middleware.ts index 5665e1ec..737040eb 100644 --- a/src/middleware/r2Middleware.ts +++ b/src/middleware/r2Middleware.ts @@ -13,10 +13,19 @@ import type { Request } from '../routes/request'; import { renderDirectoryListing } from '../utils/directoryListing'; import { getOriginalUrl, parseConditionalHeaders } from '../utils/request'; import { once } from '../utils/memo'; - -const getProvider = once((ctx: Context) => new R2Provider({ ctx })); +import { type ReleaseConfig } from '../constants/release-config'; export class R2Middleware implements Middleware { + #releaseConfig: ReleaseConfig; + #getProvider = once( + (ctx: Context) => + new R2Provider({ ctx, releaseConfig: this.#releaseConfig }) + ); + + constructor(releaseConfig: ReleaseConfig) { + this.#releaseConfig = releaseConfig; + } + async handle(request: Request, ctx: Context): Promise { const path = getR2Path(request); const isPathADirectory = isDirectoryPath(path); @@ -29,16 +38,17 @@ export class R2Middleware implements Middleware { }, }); + const provider = this.#getProvider(ctx); return isPathADirectory - ? handleDirectory(request, path, ctx) - : handleFile(request, path, ctx); + ? handleDirectory(request, path, provider) + : handleFile(request, path, provider); } } async function handleDirectory( request: Request, r2Path: string, - ctx: Context + provider: R2Provider ): Promise { if (!hasTrailingSlash(request.urlObj.pathname)) { // We always want directory listing requests to have a trailing slash @@ -46,7 +56,7 @@ async function handleDirectory( return Response.redirect(`${url}/`, 301); } - const result = await getProvider(ctx).readDirectory(r2Path); + const result = await provider.readDirectory(r2Path); if (result === undefined) { return responses.directoryNotFound(request.method); @@ -54,7 +64,7 @@ async function handleDirectory( if (result.hasIndexHtmlFile) { // Prioritize showing index files over directory listings - return handleFile(request, r2Path + 'index.html', ctx); + return handleFile(request, r2Path + 'index.html', provider); } let responseBody; @@ -93,13 +103,13 @@ function responseHeaders( function handleFile( request: Request, r2Path: string, - ctx: Context + provider: R2Provider ): Promise { switch (request.method) { case 'HEAD': - return headFile(request, r2Path, ctx); + return headFile(request, r2Path, provider); case 'GET': - return getFile(request, r2Path, ctx); + return getFile(request, r2Path, provider); } throw new Error('R2Middleware handleFile unsupported method'); @@ -108,9 +118,9 @@ function handleFile( async function headFile( request: Request, r2Path: string, - ctx: Context + provider: R2Provider ): Promise { - const result = await getProvider(ctx).headFile(r2Path); + const result = await provider.headFile(r2Path); if (result === undefined) { return responses.fileNotFound(request.method); @@ -125,10 +135,8 @@ async function headFile( async function getFile( request: Request, r2Path: string, - ctx: Context + provider: R2Provider ): Promise { - const provider = getProvider(ctx); - let result: GetFileResult | undefined; try { result = await provider.getFile(r2Path, { @@ -161,6 +169,7 @@ async function getFile( }); } +// TODO what to do here cause this is very dist-prod specific function getR2Path({ urlObj, params, diff --git a/src/providers/kvProvider.ts b/src/providers/kvProvider.ts index 97e23900..8f0fe0fd 100644 --- a/src/providers/kvProvider.ts +++ b/src/providers/kvProvider.ts @@ -1,5 +1,4 @@ import { KV_RETRY_LIMIT } from '../../lib/limits.mjs'; -import type { Context } from '../context'; import { retryWrapper } from '../utils/provider'; import type { GetFileOptions, @@ -9,15 +8,11 @@ import type { ReadDirectoryResult, } from './provider'; -type KvProviderCtorOptions = { - ctx: Context; -}; - export class KvProvider implements Provider { - #ctx: Context; + #kvNamespace: KVNamespace; - constructor({ ctx }: KvProviderCtorOptions) { - this.#ctx = ctx; + constructor(kvNamespace: KVNamespace) { + this.#kvNamespace = kvNamespace; } headFile(_: string): Promise { @@ -30,10 +25,7 @@ export class KvProvider implements Provider { async readDirectory(path: string): Promise { const result = await retryWrapper(async () => { - return this.#ctx.env.DIRECTORY_CACHE.get( - path, - 'json' - ); + return this.#kvNamespace.get(path, 'json'); }, KV_RETRY_LIMIT); if (result === null) { diff --git a/src/providers/r2Provider.ts b/src/providers/r2Provider.ts index 270f9b45..c28c97e8 100644 --- a/src/providers/r2Provider.ts +++ b/src/providers/r2Provider.ts @@ -15,6 +15,7 @@ import type { } from './provider'; import { S3Provider } from './s3Provider'; import { KvProvider } from './kvProvider'; +import { type ReleaseConfig } from '../constants/release-config'; type CachedFile = { name: string; @@ -31,13 +32,18 @@ type CachedDirectory = { type R2ProviderCtorOptions = { ctx: Context; + releaseConfig: ReleaseConfig; }; export class R2Provider implements Provider { private ctx: Context; + #bucket: R2Bucket; + #kvProvider: KvProvider; - constructor({ ctx }: R2ProviderCtorOptions) { + constructor({ ctx, releaseConfig }: R2ProviderCtorOptions) { this.ctx = ctx; + this.#bucket = releaseConfig.bucket; + this.#kvProvider = new KvProvider(releaseConfig.directoryCache); } async headFile(path: string): Promise { @@ -47,7 +53,7 @@ export class R2Provider implements Provider { } const object = await retryWrapper( - async () => await this.ctx.env.R2_BUCKET.head(path), + async () => await this.#bucket.head(path), R2_RETRY_LIMIT ); @@ -75,7 +81,7 @@ export class R2Provider implements Provider { } const object = await retryWrapper(async () => { - return this.ctx.env.R2_BUCKET.get(path, { + return this.#bucket.get(path, { onlyIf: { etagMatches: options?.conditionalHeaders?.ifMatch, etagDoesNotMatch: options?.conditionalHeaders?.ifNoneMatch, @@ -101,19 +107,15 @@ export class R2Provider implements Provider { } async readDirectory(path: string): Promise { - const kvProvider = new KvProvider({ - ctx: this.ctx, - }); - if (this.ctx.env.USE_KV) { if (this.ctx.env.KV_DIRECTORIES !== undefined) { for (const prefix of this.ctx.env.KV_DIRECTORIES) { if (path.startsWith(prefix)) { - return await kvProvider.readDirectory(path); + return await this.#kvProvider.readDirectory(path); } } } else { - return await kvProvider.readDirectory(path); + return await this.#kvProvider.readDirectory(path); } } diff --git a/src/routes/index.ts b/src/routes/index.ts index 2bfb2ef5..e022f4ee 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,4 +1,8 @@ import latestVersions from '../constants/latestVersions.json' assert { type: 'json' }; +import { + MAINLINE_RELEASE_CONFIG, + UNOFFICIAL_BUILDS_RELEASE_CONFIG, +} from '../constants/release-config'; import { MethodNotAllowedMiddleware } from '../middleware/methodNotAllowedMiddleware'; import { NotFoundMiddleware } from '../middleware/notFoundMiddleware'; import { OptionsMiddleware } from '../middleware/optionsMiddleware'; @@ -7,18 +11,28 @@ import { R2Middleware } from '../middleware/r2Middleware'; import { RedirectionMiddleware } from '../middleware/redirectionMiddleware'; import { SubtitutionMiddleware } from '../middleware/subtituteMiddleware'; import { ThrowMiddleware } from '../middleware/throwMiddleware'; -import type { Router } from './router'; +import { Router } from './router'; -export function registerRoutes(router: Router): void { - const r2Middleware = new R2Middleware(); +/** + * Register routes shared by all routers + */ +function registerCommonRoutes(router: Router): void { + router.options('*', new OptionsMiddleware()); + router.get('*', new NotFoundMiddleware()); + router.head('*', new NotFoundMiddleware()); + router.all('*', new MethodNotAllowedMiddleware()); +} + +export function getMainlineRouter(): Router { + const router = new Router(); + + const r2Middleware = new R2Middleware(MAINLINE_RELEASE_CONFIG); const originMiddleware = new OriginMiddleware(); const corepackRedirectMiddleware = new RedirectionMiddleware( 'https://github.com/nodejs/corepack#readme' ); - router.options('*', new OptionsMiddleware()); - router.head('/metrics/?:filePath+', r2Middleware, originMiddleware); router.get('/metrics/?:filePath+', r2Middleware, originMiddleware); @@ -64,9 +78,25 @@ export function registerRoutes(router: Router): void { router.post('/_throw', new ThrowMiddleware()); - router.get('*', new NotFoundMiddleware()); + registerCommonRoutes(router); - router.all('*', new MethodNotAllowedMiddleware()); + return router; +} + +/** + * @see https://github.com/nodejs/build/blob/e4d53ee871b08a5ab34b0cde5727476db23b0c42/ansible/roles/nginx/templates/unofficial-builds.nodejs.org.conf.j2#L11 + */ +export function getUnofficialBuildsRouter(): Router { + const router = new Router(); + + const r2Middleware = new R2Middleware(UNOFFICIAL_BUILDS_RELEASE_CONFIG); + + router.head('*', r2Middleware); + router.get('*', r2Middleware); + + registerCommonRoutes(router); + + return router; } export * from './router'; diff --git a/src/worker.ts b/src/worker.ts index 65e713f9..501b98ce 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -2,11 +2,24 @@ import * as Sentry from '@sentry/cloudflare'; import type { Env } from './env'; import responses from './responses'; import type { Context } from './context'; -import { Router } from './routes/router'; -import { registerRoutes } from './routes'; +import { + getMainlineRouter, + getUnofficialBuildsRouter, + type Router, +} from './routes'; -const router: Router = new Router(); -registerRoutes(router); +const mainlineRouter = getMainlineRouter(); +const unofficialBuildsRouter = getUnofficialBuildsRouter(); + +// TODO better place for this? +const hostnameToRouterMap: Record = { + 'nodejs.org': mainlineRouter, + 'r2.nodejs.org': mainlineRouter, + 'dist-worker-prod.nodejs.workers.dev': mainlineRouter, + 'r2-staging.nodejs.org': mainlineRouter, + 'dist-worker-staging.nodejs.workers.dev': mainlineRouter, + 'unofficial-builds.nodejs.org': unofficialBuildsRouter, +}; const handler = { async fetch( @@ -29,6 +42,17 @@ const handler = { execution: ctx, }; + // todo pass this to the router so we're not doing it multiple times + const url = URL.parse(request.url); + if (url === null) { + return responses.badRequest(); + } + + const router = hostnameToRouterMap[url.hostname]; + if (router === undefined) { + return responses.badRequest(); + } + try { const response: unknown = await router.fetch(request, context); diff --git a/wrangler.jsonc b/wrangler.jsonc index f47bb9a0..88366c10 100644 --- a/wrangler.jsonc +++ b/wrangler.jsonc @@ -28,6 +28,7 @@ "account_id": "07be8d2fbc940503ca1be344714cb0d1", "routes": [ { "pattern": "r2-staging.nodejs.org/*", "zone_name": "nodejs.org" }, + { "pattern": "unofficial-builds-staging.nodejs.org/*", "zone_name": "nodejs.org" }, ], "vars": { "ENVIRONMENT": "staging", @@ -44,12 +45,22 @@ "preview_bucket_name": "dist-prod", "bucket_name": "dist-prod", }, + { + // TODO bucket needs creating + "binding": "R2_BUCKET", + "preview_bucket_name": "unofficial-builds", + "bucket_name": "unofficial-builds", + }, ], "kv_namespaces": [ { "binding": "DIRECTORY_CACHE", "id": "a8378d5444a2465ca7202ebaa5d8deb1", }, + { + "binding": "UNOFFICIAL_BUILDS_DIRECTORY_CACHE", + "id": "TODO", + }, ], "cache": { "enabled": true, @@ -77,6 +88,7 @@ "zone_name": "nodejs.org", }, { "pattern": "r2.nodejs.org/*", "zone_name": "nodejs.org" }, + { "pattern": "unofficial-builds.nodejs.org/*", "zone_name": "nodejs.org" }, ], "tail_consumers": [ { @@ -90,6 +102,7 @@ "S3_ENDPOINT": "https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com", "BUCKET_NAME": "dist-prod", "ORIGIN_HOST": "https://origin.nodejs.org", + // TODO atm this needs kv to be unconditionally used in prod "USE_KV": true, "KV_DIRECTORIES": [ "metrics/", @@ -109,12 +122,22 @@ "preview_bucket_name": "dist-prod", "bucket_name": "dist-prod", }, + { + // TODO bucket needs creating + "binding": "R2_BUCKET", + "preview_bucket_name": "unofficial-builds", + "bucket_name": "unofficial-builds", + }, ], "kv_namespaces": [ { "binding": "DIRECTORY_CACHE", "id": "a8378d5444a2465ca7202ebaa5d8deb1", }, + { + "binding": "UNOFFICIAL_BUILDS_DIRECTORY_CACHE", + "id": "TODO", + }, ], "cache": { "enabled": true,