feat: Add llm txt generator to docs

docusaurus.config.js

@@ -270,8 +270,120 @@ export default async function createConfigAsync() {
       process.env.EXCLUDE_FUSION ? null : docs_fusion,
       process.env.EXCLUDE_WAVE ? null : docs_wave,
 
-      // Disable expensive bundler options.
-      // https://github.com/facebook/docusaurus/pull/11176
+      ['docusaurus-plugin-llms', {
+        id: 'llms-enterprise',
+        docsDir: 'platform-enterprise_docs',
+        llmsTxtFilename: 'llms-enterprise.txt',
+        title: 'Seqera Platform Enterprise',
+        description: 'Documentation for Seqera Platform Enterprise.',
+        rootContent: 'This file contains links to Seqera Platform Enterprise documentation following the llmstxt.org standard.',
+        generateLLMsTxt: true,
+        generateLLMsFullTxt: false,
+        generateMarkdownFiles: true,
+        includeBlog: false,
+        excludeImports: true,
+        removeDuplicateHeadings: true,
+        ignoreFiles: ['**/tags', '**/tags/**'],
+        processingBatchSize: 50,
+      }],
+      ['docusaurus-plugin-llms', {
+        id: 'llms-cloud',
+        docsDir: 'platform-cloud/docs',
+        llmsTxtFilename: 'llms-cloud.txt',
+        title: 'Seqera Platform Cloud',
+        description: 'Documentation for Seqera Platform Cloud.',
+        rootContent: 'This file contains links to Seqera Platform Cloud documentation following the llmstxt.org standard.',
+        generateLLMsTxt: true,
+        generateLLMsFullTxt: false,
+        generateMarkdownFiles: true,
+        includeBlog: false,
+        excludeImports: true,
+        removeDuplicateHeadings: true,
+        ignoreFiles: ['**/tags', '**/tags/**'],
+        processingBatchSize: 50,
+      }],
+      ['docusaurus-plugin-llms', {
+        id: 'llms-api',
+        docsDir: 'platform-api-docs/docs',
+        llmsTxtFilename: 'llms-api.txt',
+        title: 'Seqera Platform API',
+        description: 'API reference documentation for the Seqera Platform REST API.',
+        rootContent: 'This file contains links to Seqera Platform API reference documentation following the llmstxt.org standard.',
+        generateLLMsTxt: true,
+        generateLLMsFullTxt: false,
+        generateMarkdownFiles: true,
+        includeBlog: false,
+        excludeImports: true,
+        removeDuplicateHeadings: true,
+        ignoreFiles: ['**/tags', '**/tags/**'],
+        processingBatchSize: 50,
+      }],
+      ['docusaurus-plugin-llms', {
+        id: 'llms-cli',
+        docsDir: 'platform-cli-docs/docs',
+        llmsTxtFilename: 'llms-cli.txt',
+        title: 'Seqera Platform CLI',
+        description: 'Documentation for the Seqera Platform command-line interface.',
+        rootContent: 'This file contains links to Seqera Platform CLI documentation following the llmstxt.org standard.',
+        generateLLMsTxt: true,
+        generateLLMsFullTxt: false,
+        generateMarkdownFiles: true,
+        includeBlog: false,
+        excludeImports: true,
+        removeDuplicateHeadings: true,
+        ignoreFiles: ['**/tags', '**/tags/**'],
+        processingBatchSize: 50,
+      }],
+      ['docusaurus-plugin-llms', {
+        id: 'llms-multiqc',
+        docsDir: 'multiqc_docs/multiqc_repo/docs/markdown',
+        llmsTxtFilename: 'llms-multiqc.txt',
+        title: 'MultiQC',
+        description: 'Documentation for MultiQC',
+        rootContent: 'This file contains links to MultiQC documentation following the llmstxt.org standard.',
+        generateLLMsTxt: true,
+        generateLLMsFullTxt: false,
+        generateMarkdownFiles: true,
+        includeBlog: false,
+        excludeImports: true,
+        removeDuplicateHeadings: true,
+        ignoreFiles: ['**/tags', '**/tags/**'],
+        processingBatchSize: 50,
+      }],
+      ['docusaurus-plugin-llms', {
+        id: 'llms-fusion',
+        docsDir: 'fusion_docs',
+        llmsTxtFilename: 'llms-fusion.txt',
+        title: 'Fusion',
+        description: 'Documentation for Fusion.',
+        rootContent: 'This file contains links to Fusion documentation following the llmstxt.org standard.',
+        generateLLMsTxt: true,
+        generateLLMsFullTxt: false,
+        generateMarkdownFiles: true,
+        includeBlog: false,
+        excludeImports: true,
+        removeDuplicateHeadings: true,
+        ignoreFiles: ['**/tags', '**/tags/**'],
+        processingBatchSize: 50,
+      }],
+      ['docusaurus-plugin-llms', {
+        id: 'llms-wave',
+        docsDir: 'wave_docs/wave_repo/docs',
+        llmsTxtFilename: 'llms-wave.txt',
+        title: 'Wave',
+        description: 'Documentation for Wave.',
+        rootContent: 'This file contains links to Wave documentation following the llmstxt.org standard.',
+        generateLLMsTxt: true,
+        generateLLMsFullTxt: false,
+        generateMarkdownFiles: true,
+        includeBlog: false,
+        excludeImports: true,
+        removeDuplicateHeadings: true,
+        ignoreFiles: ['**/tags', '**/tags/**'],
+        processingBatchSize: 50,
+      }],
+
+      // Disable expensive bundler options: https://github.com/facebook/docusaurus/pull/11176
       function disableExpensiveBundlerOptimizationPlugin() {
         return {
           name: 'disable-expensive-bundler-optimizations',

internal/fetch-docs-oss.mjs

@@ -54,6 +54,7 @@ async function cloneOrUpdateRepo({ name, url, path: repoPath, branch }) {
 
 // Clone/update all repositories
 async function processRepositories() {
+  console.log("fetch-docs-oss: Pulling or updating docs from os product repositories...");
   for (const repo of repositories) {
     try {
       await cloneOrUpdateRepo(repo);

internal/merge-llms-txt.mjs

@@ -0,0 +1,60 @@
+import fs from "fs-extra";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const buildDir = path.join(__dirname, "..", "build");
+const outputFile = path.join(buildDir, "llms.txt");
+
+const localFiles = [
+  "llms-enterprise.txt",
+  "llms-cloud.txt",
+  "llms-api.txt",
+  "llms-cli.txt",
+  "llms-multiqc.txt",
+  "llms-fusion.txt",
+  "llms-wave.txt",
+];
+
+const remoteFiles = [
+  { filename: "llms-nextflow.txt", url: "https://docs-migration.netlify.app/llms-nextflow.txt" },
+];
+
+const header = `# Seqera Docs
+
+> Documentation for Seqera products — Seqera Platform Cloud, Seqera Platform Enterprise, Seqera Platform API, Seqera Platform CLI, Nextflow, MultiQC, Fusion, and Wave.
+
+This file contains links to all Seqera product documentation following the llmstxt.org standard.`;
+
+const parts = [header];
+
+for (const file of localFiles) {
+  const filePath = path.join(buildDir, file);
+  if (await fs.pathExists(filePath)) {
+    const content = await fs.readFile(filePath, "utf-8");
+    parts.push(content.trim());
+    console.log(`  Merged ${file}`);
+  } else {
+    console.warn(`  Skipped ${file} (not found)`);
+  }
+}
+
+for (const { filename, url } of remoteFiles) {
+  try {
+    const response = await fetch(url);
+    if (response.ok) {
+      const content = await response.text();
+      parts.push(content.trim());
+      console.log(`  Merged ${filename} (from ${url})`);
+    } else {
+      console.warn(`  Skipped ${filename} (HTTP ${response.status})`);
+    }
+  } catch (err) {
+    console.warn(`  Skipped ${filename} (fetch failed: ${err.message})`);
+  }
+}
+
+await fs.writeFile(outputFile, parts.join("\n\n") + "\n");
+console.log(`\nWrote ${outputFile}`);