From df07782544c3c9d16f51c1de8f7473bc303b1e56 Mon Sep 17 00:00:00 2001
From: jamesconwaydev <148910373+jamesconwaydev@users.noreply.github.com>
Date: Tue, 17 Nov 2026 21:29:38 -0700
Subject: [PATCH] fix(site): serve real robots.txt, sitemap.xml and 404 page
Cloudflare Pages answered /robots.txt, /sitemap.xml and every unknown
path with the home page and a 200, so Google found no sitemap and saw
soft 404s. Search Console lists /, /how-it-works/ and /archived/ as
"Crawled - currently not indexed".
robots.txt points at a sitemap.xml built from the category list, and a
404.astro gives Pages a real 404.html to serve.
---
site/public/robots.txt | 4 ++++
site/src/pages/404.astro | 13 +++++++++++++
site/src/pages/sitemap.xml.ts | 18 ++++++++++++++++++
site/tests/dist.test.mjs | 22 ++++++++++++++++++++++
4 files changed, 57 insertions(+)
create mode 100644 site/public/robots.txt
create mode 100644 site/src/pages/404.astro
create mode 100644 site/src/pages/sitemap.xml.ts
diff --git a/site/public/robots.txt b/site/public/robots.txt
new file mode 100644
index 0000000..929b1e0
--- /dev/null
+++ b/site/public/robots.txt
@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://devstream.jamesconway.dev/sitemap.xml
diff --git a/site/src/pages/404.astro b/site/src/pages/404.astro
new file mode 100644
index 0000000..95a5361
--- /dev/null
+++ b/site/src/pages/404.astro
@@ -0,0 +1,13 @@
+---
+import Base from "../layouts/Base.astro";
+---
+
+
+
+
+