diff --git a/site/public/robots.txt b/site/public/robots.txt new file mode 100644 index 0000000..929b1e0 --- /dev/null +++ b/site/public/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://devstream.jamesconway.dev/sitemap.xml diff --git a/site/src/pages/404.astro b/site/src/pages/404.astro new file mode 100644 index 0000000..95a5361 --- /dev/null +++ b/site/src/pages/404.astro @@ -0,0 +1,13 @@ +--- +import Base from "../layouts/Base.astro"; +--- + + + + + + Page not found + There's nothing at this address. Back to the leaderboard. + + + diff --git a/site/src/pages/sitemap.xml.ts b/site/src/pages/sitemap.xml.ts new file mode 100644 index 0000000..a177ffe --- /dev/null +++ b/site/src/pages/sitemap.xml.ts @@ -0,0 +1,18 @@ +import { loadAll } from "../lib/data.ts"; + +// Without this file Cloudflare Pages answers /sitemap.xml with the home page. +export function GET({ site }: { site: URL }) { + const paths = [ + "/", + "/how-it-works/", + "/archived/", + ...loadAll().categories.map((c) => `/category/${c.slug}/`), + ]; + const urls = paths + .map((p) => ` ${new URL(p, site)}`) + .join("\n"); + return new Response( + `\n\n${urls}\n\n`, + { headers: { "Content-Type": "application/xml" } }, + ); +} diff --git a/site/tests/dist.test.mjs b/site/tests/dist.test.mjs index 4edf847..75d0ab4 100644 --- a/site/tests/dist.test.mjs +++ b/site/tests/dist.test.mjs @@ -154,3 +154,25 @@ test("the Trending tab marks exactly the rows trending() allows", async () => { ); } }); + +test("robots.txt, sitemap.xml and 404.html are real files", () => { + assert.match( + read("robots.txt"), + /Sitemap: https:\/\/devstream\.jamesconway\.dev\/sitemap\.xml/, + ); + const sitemap = read("sitemap.xml"); + const cats = JSON.parse( + readFileSync(join(fixtures, "categories.json"), "utf8"), + ); + for (const p of [ + "/", + "/how-it-works/", + "/archived/", + ...cats.map((c) => `/category/${c.slug}/`), + ]) + assert.ok( + sitemap.includes(`https://devstream.jamesconway.dev${p}`), + p, + ); + assert.ok(read("404.html").includes("Page not found")); +});
There's nothing at this address. Back to the leaderboard.