Make documentation discoverable and readable by AI agents

README.md

@@ -5,6 +5,19 @@ this repository’s `docs/` folder during the build;
 generated API content is in `lib/api-docs.ts` and navigation in `lib/docs-catalog.ts`.
 No engine checkout or sibling directory is required.
 
+## Reading with AI agents
+
+Agents can start at `/llms.txt`, which lists every published page with a link to
+its plain Markdown at `/markdown/<doc-slug>.md`. `/llms-full.txt` contains the
+complete documentation, including generated API references and every language
+example. These files are generated from the same catalog and sources as the
+HTML pages on each build, so they stay synchronized automatically.
+
+HTML pages advertise their Markdown alternative and the agent index in their
+head metadata, and navigation links directly to the agent index.
+`/robots.txt` permits crawling and points to `/sitemap.xml`.
+Canonical URLs use `lib/site-url.ts`; update it when hosting a fork elsewhere.
+
 Markdown was imported from `Engine/docs` at engine commit `3db0126b`.
 This audit checked behavior against engine commit `017db99b7025eeda96acf21967664153a2df0795`.
 See [maintenance](/docs/open-source/docs/maintenance) for route/link checks and source synchronization.

app/docs/[[...slug]]/page.tsx

@@ -4,6 +4,7 @@ import { DocsLanding } from "@/components/docs-landing";
 import { DocsShell } from "@/components/docs-shell";
 import { allDocs, docAliases, routeFor } from "@/lib/docs-catalog";
 import { findDoc, readDoc } from "@/lib/docs";
+import { markdownRouteFor } from "@/lib/site-url";
 
 export const dynamicParams = false;
 
@@ -15,7 +16,7 @@ export async function generateMetadata({ params }: PageProps<"/docs/[[...slug]]"
   const { slug } = await params;
   if (!slug?.length) return { title: "Welcome · Rustic Engine Docs", description: "Start building with Rustic Engine or contribute to the documentation website." };
   const doc = findDoc(slug);
-  return doc ? { title: `${doc.title} · Rustic Engine Docs`, description: doc.description, alternates: { canonical: routeFor(doc.slug) } } : {};
+  return doc ? { title: `${doc.title} · Rustic Engine Docs`, description: doc.description, alternates: { canonical: routeFor(doc.slug), types: { "text/markdown": markdownRouteFor(doc.slug), "text/plain": "/llms.txt" } } } : {};
 }
 
 export default async function DocumentationPage({ params }: PageProps<"/docs/[[...slug]]">) {

app/layout.tsx

@@ -1,6 +1,7 @@
 import type { Metadata } from "next";
 import { IBM_Plex_Mono, Manrope } from "next/font/google";
 import { SiteFrame } from "@/components/site-frame";
+import { siteUrl } from "@/lib/site-url";
 import "./globals.css";
 
 const manrope = Manrope({ subsets: ["latin"], variable: "--font-sans" });
@@ -11,9 +12,11 @@ const plexMono = IBM_Plex_Mono({
 });
 
 export const metadata: Metadata = {
+  metadataBase: new URL(siteUrl),
   title: "Rustic Engine Docs",
   description: "Guides and API reference for building games with Rustic Engine.",
   icons: { icon: "/favicon.svg" },
+  alternates: { types: { "text/plain": "/llms.txt" } },
 };
 
 export default function RootLayout({ children }: Readonly<{ children: React.ReactNode }>) {

app/llms-full.txt/route.ts

@@ -0,0 +1,7 @@
+import { buildAgentFullText } from "@/lib/agent-docs";
+
+export const dynamic = "force-static";
+
+export async function GET() {
+  return new Response(await buildAgentFullText(), { headers: { "Content-Type": "text/plain; charset=utf-8" } });
+}

app/llms.txt/route.ts

@@ -0,0 +1,7 @@
+import { buildAgentIndex } from "@/lib/agent-docs";
+
+export const dynamic = "force-static";
+
+export function GET() {
+  return new Response(buildAgentIndex(), { headers: { "Content-Type": "text/plain; charset=utf-8" } });
+}

app/markdown/[...slug]/route.ts

@@ -0,0 +1,16 @@
+import { allDocs } from "@/lib/docs-catalog";
+import { readAgentDoc } from "@/lib/agent-docs";
+
+export const dynamic = "force-static";
+export const dynamicParams = false;
+
+export function generateStaticParams() {
+  return allDocs.map((doc) => ({ slug: `${doc.slug}.md`.split("/") }));
+}
+
+export async function GET(_request: Request, { params }: { params: Promise<{ slug: string[] }> }) {
+  const { slug } = await params;
+  const doc = allDocs.find((entry) => `${entry.slug}.md` === slug.join("/"));
+  if (!doc) return new Response("Documentation not found", { status: 404 });
+  return new Response(await readAgentDoc(doc), { headers: { "Content-Type": "text/markdown; charset=utf-8" } });
+}

app/robots.ts

@@ -0,0 +1,8 @@
+import type { MetadataRoute } from "next";
+import { siteUrl } from "@/lib/site-url";
+
+export const dynamic = "force-static";
+
+export default function robots(): MetadataRoute.Robots {
+  return { rules: { userAgent: "*", allow: "/" }, sitemap: `${siteUrl}/sitemap.xml` };
+}

app/sitemap.ts

@@ -0,0 +1,9 @@
+import type { MetadataRoute } from "next";
+import { allDocs, routeFor } from "@/lib/docs-catalog";
+import { siteUrl } from "@/lib/site-url";
+
+export const dynamic = "force-static";
+
+export default function sitemap(): MetadataRoute.Sitemap {
+  return ["/", "/docs", ...allDocs.map((doc) => routeFor(doc.slug))].map((pathname) => ({ url: `${siteUrl}${pathname}` }));
+}

components/site-frame.tsx

@@ -80,6 +80,7 @@ export function SiteFrame({ children }: { children: ReactNode }) {
         <div className="sidebar-mobile-head"><span>Documentation</span><button onClick={() => setMenuOpen(false)} aria-label="Close navigation"><X /></button></div>
         <nav className="sidebar-content" aria-label="Documentation">
           <Link className="home-link" href="/" onClick={() => setMenuOpen(false)} aria-current={pathname === "/" || pathname === "/docs" ? "page" : undefined}>Welcome to Rustic</Link>
+          <a className="home-link" href="/llms.txt">Documentation for AI agents</a>
           <div className="project-switcher" aria-label="Documentation projects">{docSections.map((section) => <Link href={section.href} key={section.title} className={!isIndex && project === section.title ? "active" : ""} onClick={() => setMenuOpen(false)}>{section.title}</Link>)}</div>
           {docSections.filter((section) => isIndex || section.title === project).map((section) => <div className="nav-project" key={section.title}>
             <Link className="nav-project-title" href={section.href} onClick={() => setMenuOpen(false)}>{section.title}</Link>

lib/agent-docs.ts

@@ -0,0 +1,28 @@
+import "server-only";
+import { allDocs, docSections, routeFor } from "./docs-catalog";
+import { readDoc } from "./docs";
+import { agentMarkdown } from "./agent-markdown";
+import { markdownRouteFor, siteUrl } from "./site-url";
+
+export function buildAgentIndex() {
+  return [
+    "# Rustic Engine documentation",
+    "> Guides and API reference for building games with Rustic Engine, plus open source contributor documentation.",
+    "Each link below returns plain Markdown without JavaScript. Pages include all scripting language examples and current implementation limits. Documentation version and engine/API versions are separate; consult Feature status and compatibility before assuming support.",
+    `Full documentation: [llms-full.txt](${siteUrl}/llms-full.txt). Human-readable site: [Docs](${siteUrl}/docs).`,
+    ...docSections.flatMap((section) => section.groups.map((group) => [
+      `## ${section.title}: ${group.label}`,
+      ...group.docs.map((doc) => `- [${doc.title}](${siteUrl}${markdownRouteFor(doc.slug)}): ${doc.description}`),
+    ].join("\n"))),
+  ].join("\n\n") + "\n";
+}
+
+export async function readAgentDoc(doc: (typeof allDocs)[number]) {
+  const markdown = agentMarkdown(await readDoc(doc.source), doc.sourcePath ?? doc.source);
+  return `<!-- Page: ${siteUrl}${routeFor(doc.slug)} -->\n\n${markdown.trim()}\n`;
+}
+
+export async function buildAgentFullText() {
+  const pages = await Promise.all(allDocs.map(readAgentDoc));
+  return `${buildAgentIndex()}\n---\n\n${pages.join("\n---\n\n")}`;
+}

lib/agent-markdown.ts

@@ -0,0 +1,44 @@
+import type { RootContent } from "mdast";
+import { allDocs, resolveDocSlug } from "./docs-catalog";
+import { resolveDocHref } from "./docs-markdown";
+import { parseMarkdown } from "./markdown-model";
+import { markdownRouteFor, siteUrl } from "./site-url";
+
+const slugs = new Set(allDocs.map((doc) => doc.slug));
+
+// Edit only parsed link destinations; code fences and authored examples stay intact.
+export function agentMarkdown(source: string, sourcePath: string) {
+  const edits: { start: number; end: number; value: string }[] = [];
+  function walk(nodes: RootContent[]) {
+    for (const node of nodes) {
+      if (node.type === "link" || node.type === "definition" || node.type === "image") {
+        const start = node.position?.start.offset;
+        const end = node.position?.end.offset;
+        const href = resolveDocHref(node.url, sourcePath);
+        const [pathname, fragment] = href.split("#", 2);
+        const slug = resolveDocSlug(pathname.replace(/^\/docs\//, ""));
+        const target = pathname.startsWith("/docs/") && slugs.has(slug)
+          ? `${siteUrl}${markdownRouteFor(slug)}${fragment ? `#${fragment}` : ""}`
+          : href.startsWith("/") ? `${siteUrl}${href}` : href;
+        if (start !== undefined && end !== undefined && target !== node.url) {
+          const original = source.slice(start, end);
+          const destination = node.type === "definition" ? /^\[[^\]]+\]:\s*<?/ : /\]\(\s*<?/;
+          const prefix = destination.exec(original);
+          if (prefix) {
+            const offset = start + prefix.index + prefix[0].length;
+            // Escaped destinations need serialization; leave those authored links alone.
+            if (source.slice(offset, offset + node.url.length) === node.url) {
+              edits.push({ start: offset, end: offset + node.url.length, value: target });
+            }
+          }
+        }
+      }
+      if ("children" in node) walk(node.children as RootContent[]);
+    }
+  }
+  walk(parseMarkdown(source).children);
+  for (const edit of edits.sort((a, b) => b.start - a.start)) {
+    source = source.slice(0, edit.start) + edit.value + source.slice(edit.end);
+  }
+  return source;
+}

lib/site-url.ts

@@ -0,0 +1,5 @@
+export const siteUrl = "https://rustic-game-engine.pages.dev";
+
+export function markdownRouteFor(slug: string) {
+  return `/markdown/${slug}.md`;
+}

public/_headers

@@ -0,0 +1,8 @@
+/markdown/*
+  Content-Type: text/markdown; charset=utf-8
+
+/llms.txt
+  Content-Type: text/plain; charset=utf-8
+
+/llms-full.txt
+  Content-Type: text/plain; charset=utf-8

tests/agent-docs.test.mjs

@@ -0,0 +1,59 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import test from "node:test";
+import { agentMarkdown } from "../lib/agent-markdown.ts";
+import { load } from "../scripts/load-lib.mjs";
+
+const { allDocs } = load("docs-catalog.ts");
+const { readDoc } = load("docs.ts");
+const siteUrl = "https://rustic-game-engine.pages.dev";
+const exported = (pathname) => readFile(new URL(`../out/${pathname}`, import.meta.url), "utf8");
+
+test("agent index and full text include every published document and all code examples", async () => {
+  const index = await exported("llms.txt");
+  const fullText = await exported("llms-full.txt");
+  for (const doc of allDocs) {
+    const pathname = `markdown/${doc.slug}.md`;
+    assert.ok(index.includes(`${siteUrl}/${pathname}`), doc.slug);
+    const markdown = await exported(pathname);
+    const original = await readDoc(doc.source);
+    assert.ok(fullText.includes(markdown.trim()), doc.slug);
+    assert.deepEqual(markdown.match(/```[\s\S]*?```/g), original.match(/```[\s\S]*?```/g), doc.slug);
+  }
+});
+
+test("Markdown links resolve source-relative links and canonical aliases without editing code", () => {
+  const code = "```md\n[Lua](scriptingLua.md)\n```";
+  const source = `[Lua](scriptingLua.md)\n[Alias](/docs/website#topics)\n[Home](/docs)\n[External](https://example.com)\n[Reference][lua]\n\n[lua]: scriptingLua.md\n\n${code}`;
+  const markdown = agentMarkdown(source, "docs/Scripting/scriptingPython.md");
+  assert.ok(markdown.includes(`[Lua](${siteUrl}/markdown/scripting/lua.md)`));
+  assert.ok(markdown.includes(`[Alias](${siteUrl}/markdown/open-source/docs.md#topics)`));
+  assert.ok(markdown.includes(`[Home](${siteUrl}/docs)`));
+  assert.ok(markdown.includes("[External](https://example.com)"));
+  assert.ok(markdown.includes(`[lua]: ${siteUrl}/markdown/scripting/lua.md`));
+  assert.ok(markdown.includes(code));
+});
+
+test("every exported Markdown documentation link targets an exported file", async () => {
+  const paths = new Set(allDocs.map((doc) => `markdown/${doc.slug}.md`));
+  for (const pathname of paths) {
+    const markdown = await exported(pathname);
+    for (const match of markdown.matchAll(/https:\/\/rustic-game-engine\.pages\.dev\/(markdown\/[^\s)#]+\.md)/g)) {
+      assert.ok(paths.has(match[1]), `${pathname}: ${match[1]}`);
+    }
+  }
+});
+
+test("HTML advertises plain text and Markdown and crawlers discover canonical pages", async () => {
+  const html = await exported("docs/api/audio/stop-recording.html");
+  assert.match(html, /href="\/llms\.txt">Documentation for AI agents<\/a>/);
+  assert.match(html, /<link[^>]+type="text\/markdown"[^>]+href="https:\/\/rustic-game-engine\.pages\.dev\/markdown\/api\/audio\/stop-recording\.md"/);
+  assert.match(html, /<link[^>]+type="text\/plain"[^>]+href="https:\/\/rustic-game-engine\.pages\.dev\/llms\.txt"/);
+  const sitemap = await exported("sitemap.xml");
+  for (const doc of allDocs) assert.ok(sitemap.includes(`<loc>${siteUrl}/docs/${doc.slug}</loc>`), doc.slug);
+  const robots = await exported("robots.txt");
+  assert.match(robots, /User-Agent: \*\nAllow: \//);
+  assert.ok(robots.includes(`Sitemap: ${siteUrl}/sitemap.xml`));
+  const headers = await exported("_headers");
+  assert.match(headers, /\/markdown\/\*\n\s+Content-Type: text\/markdown; charset=utf-8/);
+});