Improve LLM agent readability of the docs site

.github/actions/build-site/action.yml

@@ -49,6 +49,11 @@ runs:
       run: npm ci
       working-directory: site
 
+    - name: Prepare agent assets
+      shell: bash
+      run: npm run prepare-agent-assets
+      working-directory: site
+
     - name: Generate static site
       shell: bash
       run: npm run generate

.gitignore

@@ -17,6 +17,8 @@ site/.data/
 site/public/api-markdown/
 site/public/api-meta/
 site/public/api-catalog.json
+site/public/docs-markdown/
+site/public/sitemap.xml
 
 ###############
 # OS / IDE

api-gen/lib/postprocess.mjs

@@ -116,6 +116,14 @@ function repairBrokenMarkdownLinks(content) {
   )
 }
 
+/** DocFX escapes >, (, ) in signatures; undo so generics read as Create<T>(). */
+export function unescapeDocFxMarkdown(content) {
+  return content
+    .replace(/\\>/g, '>')
+    .replace(/\\\(/g, '(')
+    .replace(/\\\)/g, ')')
+}
+
 function rewriteMarkdownLinks(content, packageId, versionSegment, packageDir, pageIndex) {
   function linkFor(file, hash = '') {
     const slug = String(file).replace(/\.md$/i, '').toLowerCase()
@@ -179,9 +187,11 @@ function processPackageDir(outRoot, dir, pageIndex) {
     if (!entry.isFile() || !entry.name.toLowerCase().endsWith('.md')) continue
     const filePath = path.join(dir, entry.name)
     const original = fs.readFileSync(filePath, 'utf8')
-    const updated = annotateMemberHeadingIds(
-      rewriteMarkdownLinks(original, packageId, versionSegment, dir, pageIndex),
-    ).replace(/See the \[table of contents\]\(\.\/toc\.yml\)\.\r?\n?/i, '')
+    const updated = unescapeDocFxMarkdown(
+      annotateMemberHeadingIds(
+        rewriteMarkdownLinks(original, packageId, versionSegment, dir, pageIndex),
+      ).replace(/See the \[table of contents\]\(\.\/toc\.yml\)\.\r?\n?/i, ''),
+    )
     if (updated !== original) {
       fs.writeFileSync(filePath, updated)
     }

justfile

@@ -29,8 +29,12 @@ site-install:
 site-dev:
     npm run dev --prefix site
 
-# Statically generate the site (same as CI; run prepare-api first)
-site-generate:
+# Mirror guide markdown + write sitemap (also runs via npm pregenerate)
+prepare-agent-assets:
+    npm run prepare-agent-assets --prefix site
+
+# Statically generate the site (same as CI; run prepare-api first so API routes enter the sitemap)
+site-generate: prepare-agent-assets
     npm run generate --prefix site
 
 # Prepare API docs and generate the static site (CI parity)

readme.md

@@ -19,6 +19,8 @@ Or via just: `just prepare-api`
 
 API content is generated locally and not committed. Run `prepare` before starting the dev server.
 
+For a full static build, run `prepare-api` before `site-generate` so the agent sitemap includes API routes. See [site/readme.md](./site/readme.md) for `/llms.txt`, docs-markdown mirroring, and sitemap details.
+
 Needs [DocFX](https://dotnet.github.io/docfx/), [Node.js](https://nodejs.org/), and optionally [just](https://github.com/casey/just#installation).
 
 ## CI / deploy

site/package.json

@@ -5,6 +5,8 @@
   "scripts": {
     "build": "nuxt build",
     "dev": "nuxt dev --port 3000",
+    "prepare-agent-assets": "node scripts/prepare-agent-assets.mjs",
+    "pregenerate": "node scripts/prepare-agent-assets.mjs",
     "generate": "nuxt generate",
     "preview": "nuxt preview"
   },

site/public/llms.txt

@@ -0,0 +1,69 @@
+# AutoFixture documentation
+> AutoFixture creates anonymous test data so you write less arrange code in .NET unit tests.
+
+This file helps LLM agents discover readable documentation on https://autofixture.com.
+
+Prefer markdown URLs when you need exact C# signatures (generics stay intact). HTML pages are fine for how-to reading.
+
+## Start here
+
+- https://autofixture.com/docs/get-started/introduction
+- https://autofixture.com/docs/get-started/installation
+- https://autofixture.com/docs/get-started/first-test
+- https://autofixture.com/docs/reference/cheat-sheet
+- https://autofixture.com/docs/reference/faq
+
+## Guide HTML
+
+Guides live under `/docs/{section}/{page}`.
+
+- https://autofixture.com/docs/get-started/introduction
+- https://autofixture.com/docs/fundamentals/fixture-and-create
+- https://autofixture.com/docs/fundamentals/build-dsl
+- https://autofixture.com/docs/fundamentals/customizations
+- https://autofixture.com/docs/how-to/collections
+- https://autofixture.com/docs/integrations/overview
+- https://autofixture.com/docs/integrations/xunit3
+- https://autofixture.com/docs/integrations/automoq
+- https://autofixture.com/docs/advanced/specimen-pipeline
+- https://autofixture.com/docs/reference/packages
+- https://autofixture.com/docs/reference/v4-to-v5-migration
+
+## Guide markdown (preferred for agents)
+
+Same content as HTML, as raw markdown:
+
+  https://autofixture.com/docs-markdown/{section}/{page}.md
+
+Examples:
+
+- https://autofixture.com/docs-markdown/get-started/introduction.md
+- https://autofixture.com/docs-markdown/fundamentals/customizations.md
+- https://autofixture.com/docs-markdown/reference/cheat-sheet.md
+
+## API reference
+
+Browse HTML: https://autofixture.com/api (redirects to latest AutoFixture v5)
+
+Raw API markdown (best for signatures):
+
+  https://autofixture.com/api-markdown/{packageId}/{versionSegment}/{slug}.md
+
+Example (Fixture class, v5):
+
+- https://autofixture.com/api-markdown/autofixture/5-0-0-rc-1/autofixture.fixture.md
+- HTML twin: https://autofixture.com/api/autofixture/5-0-0-rc-1/autofixture.fixture/
+
+Slugs and version segments are lowercase. Package ids match the catalog (autofixture, xunit3, automoq, idioms, …).
+
+## Machine-readable indexes
+
+- https://autofixture.com/api-catalog.json — packages and versions
+- https://autofixture.com/api-meta/routes.json — all API HTML routes
+- https://autofixture.com/api-meta/{packageId}/{versionSegment}/toc.json — package TOC
+- https://autofixture.com/api-meta/{packageId}/{versionSegment}/search.json — searchable API snippets
+- https://autofixture.com/sitemap.xml — HTML and markdown URLs
+
+## Optional
+
+- https://autofixture.com/robots.txt

site/public/robots.txt

@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://autofixture.com/sitemap.xml

site/readme.md

@@ -10,12 +10,36 @@ just prepare-api    # generate + copy API markdown into public/api-markdown
 just site-dev
 ```
 
+For a production-like static build, run `prepare-api` before `generate` so API routes are included in the sitemap:
+
+```bash
+just prepare-api
+just site-generate   # runs prepare-agent-assets, then nuxt generate
+```
+
 ## Routes
 
 | Path | Content |
 |------|---------|
 | `/` | Home |
-| `/docs/**` | Guides (Get started section; docs layout) |
+| `/docs/**` | Guides (docs layout) |
+| `/docs-markdown/**` | Same guides as raw markdown (for LLM agents; generated at build) |
 | `/api/{package}/{version}/**` | Generated API reference |
+| `/api-markdown/**` | Raw API markdown (generated; not committed) |
+| `/llms.txt` | Agent discovery index |
+| `/robots.txt` | Crawler rules + sitemap pointer |
+| `/sitemap.xml` | Generated URL list (guides, docs-markdown, API) |
 
 API markdown is generated into `public/api-markdown` (not Nuxt Content). On API pages, the header shows the package and version from the API catalog.
+
+## Agent assets
+
+`scripts/prepare-agent-assets.mjs` (also `npm run prepare-agent-assets` / `pregenerate`):
+
+1. Mirrors `content/docs/**/*.md` → `public/docs-markdown/` with Nuxt-style paths (numeric prefixes stripped).
+2. Writes `public/sitemap.xml` from those guides plus `public/api-meta/routes.json` when present.
+
+Committed: `public/llms.txt`, `public/robots.txt`.  
+Generated (gitignored): `public/docs-markdown/`, `public/sitemap.xml`.
+
+Keep `llms.txt` in sync when you add major guide sections.

site/scripts/prepare-agent-assets.mjs

@@ -0,0 +1,178 @@
+/**
+ * Prepare agent-facing static assets before `nuxt generate`:
+ * - Mirror guide markdown to public/docs-markdown/ (URL-aligned paths)
+ * - Write public/sitemap.xml (guides, docs-markdown, API routes when present)
+ *
+ * Run from site/: node scripts/prepare-agent-assets.mjs
+ * Or via npm pregenerate / just site-generate.
+ */
+import fs from 'node:fs'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const siteRoot = path.resolve(__dirname, '..')
+const contentDocs = path.join(siteRoot, 'content', 'docs')
+const publicDir = path.join(siteRoot, 'public')
+const docsMarkdownDir = path.join(publicDir, 'docs-markdown')
+const sitemapPath = path.join(publicDir, 'sitemap.xml')
+const routesJsonPath = path.join(publicDir, 'api-meta', 'routes.json')
+
+const SITE_ORIGIN = 'https://autofixture.com'
+
+/** Strip Nuxt Content-style numeric prefixes: "1.get-started" → "get-started" */
+function stripNumericPrefix(segment) {
+  return segment.replace(/^\d+\./, '')
+}
+
+/**
+ * Map content-relative path to public docs path without leading slash.
+ * e.g. "1.get-started/1.introduction.md" → "get-started/introduction.md"
+ */
+function toPublicDocsRel(relFromDocs) {
+  const parts = relFromDocs.split(/[/\\]/).filter(Boolean)
+  const mapped = parts.map((part, index) => {
+    if (index === parts.length - 1 && part.toLowerCase().endsWith('.md')) {
+      const base = part.slice(0, -3)
+      return `${stripNumericPrefix(base)}.md`
+    }
+    return stripNumericPrefix(part)
+  })
+  return mapped.join('/')
+}
+
+function walkMarkdownFiles(dir, base = dir) {
+  /** @type {string[]} */
+  const files = []
+  if (!fs.existsSync(dir)) return files
+
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name)
+    if (entry.isDirectory()) {
+      files.push(...walkMarkdownFiles(full, base))
+      continue
+    }
+    if (entry.isFile() && entry.name.toLowerCase().endsWith('.md')) {
+      files.push(path.relative(base, full))
+    }
+  }
+  return files
+}
+
+function removeIfExists(target) {
+  if (fs.existsSync(target)) {
+    fs.rmSync(target, { recursive: true, force: true })
+  }
+}
+
+function ensureDir(dir) {
+  fs.mkdirSync(dir, { recursive: true })
+}
+
+/**
+ * @returns {{ htmlPaths: string[], markdownPaths: string[] }}
+ */
+function mirrorDocsMarkdown() {
+  removeIfExists(docsMarkdownDir)
+  ensureDir(docsMarkdownDir)
+
+  /** @type {string[]} */
+  const htmlPaths = []
+  /** @type {string[]} */
+  const markdownPaths = []
+
+  const files = walkMarkdownFiles(contentDocs)
+  for (const rel of files) {
+    const publicRel = toPublicDocsRel(rel)
+    const dest = path.join(docsMarkdownDir, publicRel)
+    ensureDir(path.dirname(dest))
+    fs.copyFileSync(path.join(contentDocs, rel), dest)
+
+    const withoutExt = publicRel.replace(/\.md$/i, '')
+    htmlPaths.push(`/docs/${withoutExt}`)
+    markdownPaths.push(`/docs-markdown/${publicRel.replace(/\\/g, '/')}`)
+  }
+
+  htmlPaths.sort()
+  markdownPaths.sort()
+  return { htmlPaths, markdownPaths }
+}
+
+function readApiRoutes() {
+  if (!fs.existsSync(routesJsonPath)) return []
+  try {
+    const data = JSON.parse(fs.readFileSync(routesJsonPath, 'utf8'))
+    if (!Array.isArray(data)) return []
+    return data.filter((r) => typeof r === 'string' && r.startsWith('/'))
+  } catch {
+    return []
+  }
+}
+
+function escapeXml(value) {
+  return value
+    .replace(/&/g, '&')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+}
+
+/**
+ * @param {string[]} locs
+ */
+function writeSitemap(locs) {
+  const unique = [...new Set(locs)]
+  unique.sort()
+
+  const body = unique
+    .map(
+      (loc) => `  <url>
+    <loc>${escapeXml(`${SITE_ORIGIN}${loc}`)}</loc>
+  </url>`,
+    )
+    .join('\n')
+
+  const xml = `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${body}
+</urlset>
+`
+
+  fs.writeFileSync(sitemapPath, xml, 'utf8')
+  return unique.length
+}
+
+function main() {
+  if (!fs.existsSync(contentDocs)) {
+    console.error(`Missing content docs at ${contentDocs}`)
+    process.exit(1)
+  }
+
+  const { htmlPaths, markdownPaths } = mirrorDocsMarkdown()
+  const apiRoutes = readApiRoutes()
+
+  const locs = [
+    '/',
+    '/llms.txt',
+    '/robots.txt',
+    '/api-catalog.json',
+    ...htmlPaths,
+    ...markdownPaths,
+    ...apiRoutes,
+  ]
+
+  // Prefer catalog when prepare-api has run; omit if missing (local docs-only).
+  if (!fs.existsSync(path.join(publicDir, 'api-catalog.json'))) {
+    const i = locs.indexOf('/api-catalog.json')
+    if (i >= 0) locs.splice(i, 1)
+  }
+
+  const count = writeSitemap(locs)
+
+  console.log(
+    `prepare-agent-assets: mirrored ${htmlPaths.length} guides → docs-markdown/; sitemap ${count} URLs` +
+      (apiRoutes.length ? ` (incl. ${apiRoutes.length} API routes)` : ' (no api-meta/routes.json yet)'),
+  )
+}
+
+main()