From ef089985d9aac883adb08084beea5f14388b74ef Mon Sep 17 00:00:00 2001 From: Rafael Vuijk Date: Sat, 3 Oct 2026 06:22:47 +0000 Subject: [PATCH] An llms.txt, written beside the sitemap The library's name and description, a link to each page and to each page of the wiki, and the documentation of every type and member as one link to its index, as llmstxt.org proposes. The description is read from the pages' meta description, so it is said once. Part of #21 Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_012sonx8iAspMiwRwokT1Ura --- AGENTS.md | 4 +- src/NaiveStaticGenerator/Program.cs | 58 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3587a94a9..a1b443622 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,8 +12,8 @@ sets the working practice for both. When there is nothing to do here, it goes ba `build` runs `src/NaiveStaticGenerator`, which wraps every page in `src/content` in `src/content/_templates/top.html` and `bottom.html` and writes `.output/final`. `run` builds and opens the home page. -- `.output/final` also gets the stylesheets, `img/`, `CNAME`, `robots.txt` and a `sitemap.xml` that - the generator writes from the pages it produced. +- `.output/final` also gets the stylesheets, `img/`, `CNAME`, `robots.txt`, and a `sitemap.xml` and an + `llms.txt` that the generator writes from the pages it produced. - A pull request builds the site on Windows, Linux and macOS (`build-test.yml`). - **A merge to master is live.** `deployment.yml` builds the site and pushes `.output/final` to the `gh-pages` branch, and GitHub Pages publishes it a few minutes later. Check the live page after a diff --git a/src/NaiveStaticGenerator/Program.cs b/src/NaiveStaticGenerator/Program.cs index 46f0d2111..33704e76f 100644 --- a/src/NaiveStaticGenerator/Program.cs +++ b/src/NaiveStaticGenerator/Program.cs @@ -32,6 +32,7 @@ CopyCName(); CopyRobots(); GenerateSitemap(); +GenerateLlmsTxt(); static string GetNearestRoot(string name, string current) => Path.GetFileName(current) == name ? current : GetNearestRoot(name, Path.GetDirectoryName(current)); @@ -300,6 +301,63 @@ void GenerateSitemap() File.WriteAllText(FinalOutputP._("sitemap.xml"), sitemap.ToString()); } +// The website for language models, as llmstxt.org proposes it: the library's name and description, +// a link to each page, and the documentation of the types and members, thousands of pages, as one +// link to its index. The description is the one the pages give search engines, so it is said once. +void GenerateLlmsTxt() +{ + var host = File.ReadAllText(RootP._("CNAME")).Trim(); + var top = File.ReadAllText(ContentP._("_templates")._("top.html")); + var description = System.Text.RegularExpressions.Regex.Match(top, " + { + [""] = "Home", ["quickstart/"] = "Quickstart", ["whatsnew/"] = "What's new", + ["why/"] = "Why AngouriMath", ["research/"] = "Research", ["demo/"] = "Demo", ["wiki/"] = "Wiki", + }; + var pages = new List<(string Path, string Url)>(); + var wiki = new List(); + foreach (var page in Directory.EnumerateFiles(FinalOutputP, "*.html", SearchOption.AllDirectories).OrderBy(p => p, StringComparer.Ordinal)) + { + var relative = Path.GetRelativePath(FinalOutputP, page).Replace(Path.DirectorySeparatorChar, '/'); + if (relative.StartsWith("docs/", StringComparison.Ordinal)) + continue; + if (relative == "index.html") + relative = ""; + else if (relative.EndsWith("/index.html", StringComparison.Ordinal)) + relative = relative[..^"index.html".Length]; + var url = $"https://{host}/{string.Join("/", relative.Split('/').Select(Uri.EscapeDataString))}"; + // A page of the wiki by its name; `_Footer` and `_Sidebar` are parts of the wiki's pages. + if (relative.StartsWith("wiki/", StringComparison.Ordinal) && relative != "wiki/") + { + var name = Path.GetFileNameWithoutExtension(relative); + if (!name.StartsWith("_", StringComparison.Ordinal)) + wiki.Add($"- [{name.Replace('-', ' ')}]({url})"); + } + else + pages.Add((relative, url)); + } + var llms = new StringBuilder(); + llms.AppendLine("# AngouriMath"); + llms.AppendLine(); + llms.AppendLine($"> {System.Net.WebUtility.HtmlDecode(description)}"); + llms.AppendLine(); + llms.AppendLine("## Pages"); + llms.AppendLine(); + // The pages in the order the map names them, any other after them. + var order = named.Keys.ToList(); + foreach (var (path, url) in pages.OrderBy(page => order.IndexOf(page.Path) is var at && at >= 0 ? at : order.Count)) + llms.AppendLine($"- [{(named.TryGetValue(path, out var name) ? name : path.TrimEnd('/'))}]({url})"); + llms.AppendLine(); + llms.AppendLine("## Wiki"); + llms.AppendLine(); + wiki.ForEach(line => llms.AppendLine(line)); + llms.AppendLine(); + llms.AppendLine("## Optional"); + llms.AppendLine(); + llms.AppendLine($"- [Almanac](https://{host}/docs/namespaces.html): every namespace, type and member, from the library's XML documentation"); + File.WriteAllText(FinalOutputP._("llms.txt"), llms.ToString()); +} + public sealed class PageSaverAndCounter : IPageSave {