An llms.txt, written beside the sitemap

AGENTS.md

@@ -12,8 +12,8 @@ sets the working practice for both. When there is nothing to do here, it goes ba
   `build` runs `src/NaiveStaticGenerator`, which wraps every page in `src/content` in
   `src/content/_templates/top.html` and `bottom.html` and writes `.output/final`. `run` builds and
   opens the home page.
-- `.output/final` also gets the stylesheets, `img/`, `CNAME`, `robots.txt` and a `sitemap.xml` that
-  the generator writes from the pages it produced.
+- `.output/final` also gets the stylesheets, `img/`, `CNAME`, `robots.txt`, and a `sitemap.xml` and an
+  `llms.txt` that the generator writes from the pages it produced.
 - A pull request builds the site on Windows, Linux and macOS (`build-test.yml`).
 - **A merge to master is live.** `deployment.yml` builds the site and pushes `.output/final` to the
   `gh-pages` branch, and GitHub Pages publishes it a few minutes later. Check the live page after a

src/NaiveStaticGenerator/Program.cs

@@ -32,6 +32,7 @@
 CopyCName();
 CopyRobots();
 GenerateSitemap();
+GenerateLlmsTxt();
 
 static string GetNearestRoot(string name, string current)
     => Path.GetFileName(current) == name ? current : GetNearestRoot(name, Path.GetDirectoryName(current));
@@ -206,7 +207,7 @@
     }
 
 
     static string PathUp(string content, string relativeName, string subPath)
         => content.Replace("/" + subPath,
             string.Join("",
                 Enumerable.Range(0, relativeName.Count(c => c == '\\')
@@ -253,7 +254,7 @@
     }
 }
 
 void CopyWikiImagesToFinalWebsite()
 {
     DirectoryCopy(RootP._("wiki"), FinalOutputP._("wiki"), copySubDirs: false, _ => true);    
 }
@@ -300,6 +301,63 @@
     File.WriteAllText(FinalOutputP._("sitemap.xml"), sitemap.ToString());
 }
 
+// The website for language models, as llmstxt.org proposes it: the library's name and description,
+// a link to each page, and the documentation of the types and members, thousands of pages, as one
+// link to its index. The description is the one the pages give search engines, so it is said once.
+void GenerateLlmsTxt()
+{
+    var host = File.ReadAllText(RootP._("CNAME")).Trim();
+    var top = File.ReadAllText(ContentP._("_templates")._("top.html"));
+    var description = System.Text.RegularExpressions.Regex.Match(top, "<meta name=\"description\" content=\"([^\"]*)\"").Groups[1].Value;
+    var named = new Dictionary<string, string>
+    {
+        [""] = "Home", ["quickstart/"] = "Quickstart", ["whatsnew/"] = "What's new",
+        ["why/"] = "Why AngouriMath", ["research/"] = "Research", ["demo/"] = "Demo", ["wiki/"] = "Wiki",
+    };
+    var pages = new List<(string Path, string Url)>();
+    var wiki = new List<string>();
+    foreach (var page in Directory.EnumerateFiles(FinalOutputP, "*.html", SearchOption.AllDirectories).OrderBy(p => p, StringComparer.Ordinal))
+    {
+        var relative = Path.GetRelativePath(FinalOutputP, page).Replace(Path.DirectorySeparatorChar, '/');
+        if (relative.StartsWith("docs/", StringComparison.Ordinal))
+            continue;
+        if (relative == "index.html")
+            relative = "";
+        else if (relative.EndsWith("/index.html", StringComparison.Ordinal))
+            relative = relative[..^"index.html".Length];
+        var url = $"https://{host}/{string.Join("/", relative.Split('/').Select(Uri.EscapeDataString))}";
+        // A page of the wiki by its name; `_Footer` and `_Sidebar` are parts of the wiki's pages.
+        if (relative.StartsWith("wiki/", StringComparison.Ordinal) && relative != "wiki/")
+        {
+            var name = Path.GetFileNameWithoutExtension(relative);
+            if (!name.StartsWith("_", StringComparison.Ordinal))
+                wiki.Add($"- [{name.Replace('-', ' ')}]({url})");
+        }
+        else
+            pages.Add((relative, url));
+    }
+    var llms = new StringBuilder();
+    llms.AppendLine("# AngouriMath");
+    llms.AppendLine();
+    llms.AppendLine($"> {System.Net.WebUtility.HtmlDecode(description)}");
+    llms.AppendLine();
+    llms.AppendLine("## Pages");
+    llms.AppendLine();
+    // The pages in the order the map names them, any other after them.
+    var order = named.Keys.ToList();
+    foreach (var (path, url) in pages.OrderBy(page => order.IndexOf(page.Path) is var at && at >= 0 ? at : order.Count))
+        llms.AppendLine($"- [{(named.TryGetValue(path, out var name) ? name : path.TrimEnd('/'))}]({url})");
+    llms.AppendLine();
+    llms.AppendLine("## Wiki");
+    llms.AppendLine();
+    wiki.ForEach(line => llms.AppendLine(line));
+    llms.AppendLine();
+    llms.AppendLine("## Optional");
+    llms.AppendLine();
+    llms.AppendLine($"- [Almanac](https://{host}/docs/namespaces.html): every namespace, type and member, from the library's XML documentation");
+    File.WriteAllText(FinalOutputP._("llms.txt"), llms.ToString());
+}
+
 
 public sealed class PageSaverAndCounter : IPageSave
 {