From df07782544c3c9d16f51c1de8f7473bc303b1e56 Mon Sep 17 00:00:00 2001 From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:29:38 -0700 Subject: [PATCH] fix(site): serve real robots.txt, sitemap.xml and 404 page Cloudflare Pages answered /robots.txt, /sitemap.xml and every unknown path with the home page and a 200, so Google found no sitemap and saw soft 404s. Search Console lists /, /how-it-works/ and /archived/ as "Crawled - currently not indexed". robots.txt points at a sitemap.xml built from the category list, and a 404.astro gives Pages a real 404.html to serve. --- site/public/robots.txt | 4 ++++ site/src/pages/404.astro | 13 +++++++++++++ site/src/pages/sitemap.xml.ts | 18 ++++++++++++++++++ site/tests/dist.test.mjs | 22 ++++++++++++++++++++++ 4 files changed, 57 insertions(+) create mode 100644 site/public/robots.txt create mode 100644 site/src/pages/404.astro create mode 100644 site/src/pages/sitemap.xml.ts diff --git a/site/public/robots.txt b/site/public/robots.txt new file mode 100644 index 0000000..929b1e0 --- /dev/null +++ b/site/public/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://devstream.jamesconway.dev/sitemap.xml diff --git a/site/src/pages/404.astro b/site/src/pages/404.astro new file mode 100644 index 0000000..95a5361 --- /dev/null +++ b/site/src/pages/404.astro @@ -0,0 +1,13 @@ +--- +import Base from "../layouts/Base.astro"; +--- + + + +
+
+

Page not found

+

There's nothing at this address. Back to the leaderboard.

+
+
+ diff --git a/site/src/pages/sitemap.xml.ts b/site/src/pages/sitemap.xml.ts new file mode 100644 index 0000000..a177ffe --- /dev/null +++ b/site/src/pages/sitemap.xml.ts @@ -0,0 +1,18 @@ +import { loadAll } from "../lib/data.ts"; + +// Without this file Cloudflare Pages answers /sitemap.xml with the home page. +export function GET({ site }: { site: URL }) { + const paths = [ + "/", + "/how-it-works/", + "/archived/", + ...loadAll().categories.map((c) => `/category/${c.slug}/`), + ]; + const urls = paths + .map((p) => ` ${new URL(p, site)}`) + .join("\n"); + return new Response( + `\n\n${urls}\n\n`, + { headers: { "Content-Type": "application/xml" } }, + ); +} diff --git a/site/tests/dist.test.mjs b/site/tests/dist.test.mjs index 4edf847..75d0ab4 100644 --- a/site/tests/dist.test.mjs +++ b/site/tests/dist.test.mjs @@ -154,3 +154,25 @@ test("the Trending tab marks exactly the rows trending() allows", async () => { ); } }); + +test("robots.txt, sitemap.xml and 404.html are real files", () => { + assert.match( + read("robots.txt"), + /Sitemap: https:\/\/devstream\.jamesconway\.dev\/sitemap\.xml/, + ); + const sitemap = read("sitemap.xml"); + const cats = JSON.parse( + readFileSync(join(fixtures, "categories.json"), "utf8"), + ); + for (const p of [ + "/", + "/how-it-works/", + "/archived/", + ...cats.map((c) => `/category/${c.slug}/`), + ]) + assert.ok( + sitemap.includes(`https://devstream.jamesconway.dev${p}`), + p, + ); + assert.ok(read("404.html").includes("Page not found")); +});