From f80b2e736dc1a456c031af1d6331e9aa72df9667 Mon Sep 17 00:00:00 2001 From: Jayden Nguyen <94533693+Jadenzzz@users.noreply.github.com> Date: Mon, 18 May 2026 12:23:22 +1000 Subject: [PATCH 01/43] feat(llms): llms.txt validator (#6) * first commit * cleanup * cleanup * more edge cases --- src/commands/import.js | 73 ++++++++++------------------------- src/utils/llms.js | 88 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 52 deletions(-) create mode 100644 src/utils/llms.js diff --git a/src/commands/import.js b/src/commands/import.js index 0fe5fb6..0b42076 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -10,6 +10,7 @@ import * as styles from '../utils/styles.js' import { syncOas } from './oas-sync.js' import OASNormalize from 'oas-normalize' import { slotOrphansPrompt, iconizeNavPrompt, organizeFromSectionsPrompt, organizeFromScratchPrompt, stripCodeFences } from '../prompts/index.js' +import { analyzeLlmsTxt } from '../utils/llms.js' export const command = 'import' export const order = 7 @@ -85,13 +86,19 @@ export async function importDocs(options) { const llmsCandidates = buildLlmsCandidates(sourceUrl) styles.info(`Checking for llms.txt (${llmsCandidates.length} candidate${llmsCandidates.length === 1 ? '' : 's'})...`) - const { llms, llmsUrl } = await timePhase('fetch llms.txt', async () => { + const { llms, llmsUrl, skippedLlms } = await timePhase('fetch llms.txt', async () => { + const skipped = [] for (const candidate of llmsCandidates) { const res = await fetchLlmsTxt(candidate) - if (res.ok) return { llms: res, llmsUrl: candidate } + if (res.ok) { + if (res.usable) return { llms: res, llmsUrl: candidate, skippedLlms: skipped } + skipped.push({ url: candidate, reason: res.reason }) + styles.info(styles.dim(` ${candidate} → skipped (${res.reason})`)) + continue + } styles.info(styles.dim(` ${candidate} → ${res.status ? `HTTP ${res.status}` : res.error || 'failed'}`)) } - return { llms: null, llmsUrl: null } + return { llms: null, llmsUrl: null, skippedLlms: skipped } }) console.log() @@ -105,7 +112,7 @@ export async function importDocs(options) { const sitemapCandidates = buildSitemapCandidates(sourceUrl) const scopePrefix = deriveSitemapScope(sourceUrl) styles.info( - `No llms.txt — checking for sitemap.xml (${sitemapCandidates.length} candidate${sitemapCandidates.length === 1 ? '' : 's'})${scopePrefix ? `, scoped to ${styles.bold(scopePrefix)}` : ''}...`, + `${skippedLlms.length > 0 ? 'No usable llms.txt' : 'No llms.txt'} — checking for sitemap.xml (${sitemapCandidates.length} candidate${sitemapCandidates.length === 1 ? '' : 's'})${scopePrefix ? `, scoped to ${styles.bold(scopePrefix)}` : ''}...`, ) const sitemapResult = await timePhase('fetch sitemap.xml', async () => { for (const candidate of sitemapCandidates) { @@ -129,7 +136,7 @@ export async function importDocs(options) { } if (debugSnapshots) { - debugSnapshots['01-llms-parsed.json'] = { llmsUrl, parsed: llms ? llms.parsed : null } + debugSnapshots['01-llms-parsed.json'] = { llmsUrl, parsed: llms ? llms.parsed : null, skipped: skippedLlms } debugSnapshots['01b-sitemap.json'] = { sitemapUrl, urls: sitemapKnownUrls } } @@ -2314,8 +2321,7 @@ function buildLlmsCandidates(sourceUrl) { } /** - * Best-effort fetch of a site's /llms.txt. Returns { ok, status, error, parsed } - * where parsed is { title, sections: [{ title, items: [{ text, url, description }] }] }. + * Best-effort fetch of a site's /llms.txt plus a simple structural usability check. */ async function fetchLlmsTxt(llmsUrl) { try { @@ -2325,56 +2331,19 @@ async function fetchLlmsTxt(llmsUrl) { }) if (!res.ok) return { ok: false, status: res.status } const text = await res.text() - return { ok: true, status: res.status, parsed: parseLlmsTxt(text) } + const analysis = analyzeLlmsTxt(text, llmsUrl) + return { + ok: true, + status: res.status, + parsed: analysis.parsed, + usable: analysis.usable, + reason: analysis.reason, + } } catch (e) { return { ok: false, error: e.message } } } -/** - * Parse the llms.txt format. `##` headings become sections; - * `- [text](url): description` bullets become items. Items before any `##` - * land in an implicit "Resources" section. - */ -function parseLlmsTxt(body) { - const lines = body.split(/\r?\n/) - let title = null - const sections = [] - let current = null - - const itemRe = /^\s*-\s*\[([^\]]*)\]\((https?:\/\/[^)\s]+)\)(?:\s*[:—–-]\s*(.+))?/ - - for (const line of lines) { - const h1 = line.match(/^#\s+(.+)$/) - if (h1 && !title) { - title = h1[1].trim() - continue - } - - const h2 = line.match(/^##\s+(.+)$/) - if (h2) { - current = { title: h2[1].trim(), items: [] } - sections.push(current) - continue - } - - const item = line.match(itemRe) - if (item) { - if (!current) { - current = { title: 'Resources', items: [] } - sections.push(current) - } - current.items.push({ - text: item[1].trim(), - url: item[2].replace(/[.,;]+$/, ''), - description: item[3] ? item[3].trim() : null, - }) - } - } - - return { title, sections } -} - // Path segments that mark a docs scope. Used to find the right "base" for // filtering sitemap URLs: walking up from the source URL, the first segment // that matches becomes the scope prefix. Hits should stay narrow — `blog` diff --git a/src/utils/llms.js b/src/utils/llms.js new file mode 100644 index 0000000..1f4155f --- /dev/null +++ b/src/utils/llms.js @@ -0,0 +1,88 @@ +const H1_RE = /^#\s+(.+)$/ +const H2_RE = /^##\s+(.+)$/ +const STANDARD_LIST_LINK_RE = /^\s*[-*+]\s+\[([^\]]+)\]\(([^)\s]+)\)(?:\s*[:—–-]\s*(.+?))?\s*$/ + +export function analyzeLlmsTxt(body, llmsUrl) { + const parsed = parseLlmsTxt(body, llmsUrl) + const reason = getSkipReason(body, parsed) + + return { + parsed, + usable: !reason, + reason, + } +} + +export function parseLlmsTxt(body, llmsUrl) { + const lines = body.split(/\r?\n/) + let title = null + const sections = [] + let current = null + let llmsOrigin = null + if (llmsUrl) { try { llmsOrigin = new URL(llmsUrl).origin } catch {} } + + for (const line of lines) { + const h1 = line.match(H1_RE) + if (h1 && !title) { + title = h1[1].trim() + continue + } + + const h2 = line.match(H2_RE) + if (h2) { + current = { title: h2[1].trim(), items: [] } + sections.push(current) + continue + } + + const item = parseListLink(line, llmsUrl, llmsOrigin) + if (!item) continue + + if (!current) { + current = { title: 'Resources', items: [] } + sections.push(current) + } + current.items.push(item) + } + + return { title, sections } +} + +function getSkipReason(body, parsed) { + if (/^---\r?\n/.test(body)) return 'starts with YAML frontmatter' + if (/^\s*(?:```|~~~)/m.test(body)) return 'contains fenced code blocks' + if (/^!\[/m.test(body)) return 'contains image markdown' + + const itemCount = parsed.sections.reduce((sum, section) => sum + section.items.length, 0) + if (itemCount === 0) return 'contains no standard llms.txt link rows' + + return null +} + +function parseListLink(line, llmsUrl, llmsOrigin) { + const match = line.match(STANDARD_LIST_LINK_RE) + if (!match) return null + + const url = normalizeUrl(match[2], llmsUrl, llmsOrigin) + if (!url) return null + + return { + text: match[1].trim(), + url, + description: match[3] ? match[3].trim() : null, + } +} + +function normalizeUrl(rawUrl, llmsUrl, llmsOrigin) { + const trimmed = String(rawUrl || '').trim().replace(/[.,;]+$/, '') + if (!trimmed || /^#/.test(trimmed) || /^(mailto|javascript):/i.test(trimmed)) return null + + try { + const url = llmsUrl ? new URL(trimmed, llmsUrl) : new URL(trimmed) + if (url.protocol !== 'http:' && url.protocol !== 'https:') return null + if (llmsOrigin && url.origin !== llmsOrigin) return null + return url.toString() + } catch { + return null + } +} From b9b63b70d052cb08fba836d0c0fcefd20eb4c826 Mon Sep 17 00:00:00 2001 From: Jadenzzz <94533693+Jadenzzz@users.noreply.github.com> Date: Mon, 18 May 2026 12:49:57 +1000 Subject: [PATCH 02/43] fix(llms-validator): remove origin check --- src/utils/llms.js | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/utils/llms.js b/src/utils/llms.js index 1f4155f..14754d6 100644 --- a/src/utils/llms.js +++ b/src/utils/llms.js @@ -18,8 +18,6 @@ export function parseLlmsTxt(body, llmsUrl) { let title = null const sections = [] let current = null - let llmsOrigin = null - if (llmsUrl) { try { llmsOrigin = new URL(llmsUrl).origin } catch {} } for (const line of lines) { const h1 = line.match(H1_RE) @@ -35,7 +33,7 @@ export function parseLlmsTxt(body, llmsUrl) { continue } - const item = parseListLink(line, llmsUrl, llmsOrigin) + const item = parseListLink(line, llmsUrl) if (!item) continue if (!current) { @@ -59,11 +57,11 @@ function getSkipReason(body, parsed) { return null } -function parseListLink(line, llmsUrl, llmsOrigin) { +function parseListLink(line, llmsUrl) { const match = line.match(STANDARD_LIST_LINK_RE) if (!match) return null - const url = normalizeUrl(match[2], llmsUrl, llmsOrigin) + const url = normalizeUrl(match[2], llmsUrl) if (!url) return null return { @@ -73,14 +71,13 @@ function parseListLink(line, llmsUrl, llmsOrigin) { } } -function normalizeUrl(rawUrl, llmsUrl, llmsOrigin) { +function normalizeUrl(rawUrl, llmsUrl) { const trimmed = String(rawUrl || '').trim().replace(/[.,;]+$/, '') if (!trimmed || /^#/.test(trimmed) || /^(mailto|javascript):/i.test(trimmed)) return null try { const url = llmsUrl ? new URL(trimmed, llmsUrl) : new URL(trimmed) if (url.protocol !== 'http:' && url.protocol !== 'https:') return null - if (llmsOrigin && url.origin !== llmsOrigin) return null return url.toString() } catch { return null From 77145b2094b723f77fc34455d5630727322523ea Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Mon, 18 May 2026 18:48:47 +1000 Subject: [PATCH 03/43] feat: better llms.txt validation, don't throw out sections based on name matching --- src/commands/import.js | 47 ++++++++++++----------------- src/utils/llms.js | 68 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 35 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 0b42076..5d3a826 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -132,7 +132,8 @@ export async function importDocs(options) { styles.warning(`No llms.txt or sitemap.xml found — falling back to sidebar discovery via scrape.`) } } else { - styles.info(styles.dim(`Using ${llmsUrl}.`)) + const s = llms.stats + styles.info(styles.dim(`Using ${llmsUrl} (${s.conforming}/${s.total} lines conforming, ratio ${s.ratio.toFixed(2)}).`)) } if (debugSnapshots) { @@ -2106,26 +2107,15 @@ async function iconizeScrapedNav(scraped, _unused, model, siteTitle) { } /** - * Sections are "usable" when the llms.txt already did the hard grouping work - * for us — meaningful titles, not too many/few, each populated. When usable we - * take a fast path that only asks Claude for icons + title polish instead of - * re-bucketing every page, which is the slow part of a full reorg. - */ -// Sections named like these are catch-all buckets — even in richly-structured -// llms.txt files (e.g. Stripe's "Docs" section is where pages go that don't -// fit into a named product tab), so always drop them rather than promote the -// grab-bag contents to a top-level sidebar category. -const GENERIC_SECTION_RE = - /^(resources?|english|root url|pages?|docs?|documentation|content|available languages.*|site|sitemap|index|home|optional|instructions?(\s|:).*|miscellaneous|misc|other)$/i - -/** - * Return the subset of llms.txt sections that carry real structural signal — - * drop catch-all buckets ("Docs", "Resources", "Optional"), empty sections, - * and oversized ones (site-dumps masquerading as sections). + * Return the subset of llms.txt sections worth sending to the fast path — + * drop only structurally bad ones (empty, or oversized site-dumps). Heading + * names are preserved verbatim: a heading like "Docs" or "Documentation" + * might be a catch-all on one site (bad) and a real top-level category on + * another (fine), and we can't tell cheaply, so we don't try. */ function usableSections(sections) { if (!sections) return [] - return sections.filter((s) => s.title && !GENERIC_SECTION_RE.test(s.title.trim()) && s.items && s.items.length > 0 && s.items.length <= 200) + return sections.filter((s) => s.title && s.items && s.items.length > 0 && s.items.length <= 200) } function sectionsLookUsable(sections) { @@ -2146,9 +2136,9 @@ async function organizeWithClaude(parsed, model) { * is O(sections), not O(pages), so this is usually ~5-15s vs. a full reorg. */ async function organizeFromSections(parsed, model) { - // Drop generic/empty/oversized sections so they don't pollute the sidebar - // (e.g. Stripe's llms.txt has a "Docs" catch-all and a 0-item "Instructions - // for Large Language Model Agents" section — neither is structural signal). + // Drop empty/oversized sections (e.g. a 0-item "Instructions for Large + // Language Model Agents" preamble, or a 500-row site-dump) — heading + // names are preserved as-is. const sections = usableSections(parsed.sections) const { systemPrompt, userPrompt } = organizeFromSectionsPrompt({ @@ -2338,6 +2328,7 @@ async function fetchLlmsTxt(llmsUrl) { parsed: analysis.parsed, usable: analysis.usable, reason: analysis.reason, + stats: analysis.stats, } } catch (e) { return { ok: false, error: e.message } @@ -2523,13 +2514,6 @@ function sitemapUrlsToKnownUrls(urls) { return out } -/** - * Write the organized hierarchy to disk as git-format markdown stubs — just - * frontmatter, no body yet. docs/ pages go under docs//.md; - * reference/recipes/custom_pages/custom_blocks get their own top-level dir - * without a category subfolder (the git-format schema doesn't nest them). - * Writes _order.yaml per directory so sidebar order matches input order. - */ /** * Recursively print the page tree. Sub-pages are indented under their parent * with no leading bullet character, to show them as children of the parent. @@ -2546,6 +2530,13 @@ function printPagesTree(pages, indentLevel) { } } +/** + * Write the organized hierarchy to disk as git-format markdown stubs — just + * frontmatter, no body yet. docs/ pages go under docs//.md; + * reference/recipes/custom_pages/custom_blocks get their own top-level dir + * without a category subfolder (the git-format schema doesn't nest them). + * Writes _order.yaml per directory so sidebar order matches input order. + */ function stageOrganized(organized, stagingDir, opts = {}) { const pickIcon = makeIconPicker() const byDir = new Map() diff --git a/src/utils/llms.js b/src/utils/llms.js index 14754d6..008c794 100644 --- a/src/utils/llms.js +++ b/src/utils/llms.js @@ -1,15 +1,30 @@ const H1_RE = /^#\s+(.+)$/ const H2_RE = /^##\s+(.+)$/ const STANDARD_LIST_LINK_RE = /^\s*[-*+]\s+\[([^\]]+)\]\(([^)\s]+)\)(?:\s*[:—–-]\s*(.+?))?\s*$/ +const BLOCKQUOTE_RE = /^\s*>/ +const FENCE_RE = /^\s*(?:```|~~~)/ + +// A llms.txt is considered usable when at least MIN_LINK_ROWS link items +// parse cleanly AND the share of "spec-shaped" lines (H1/H2/blockquote/link +// row/blank) over total lines is at least MIN_CONFORMING_RATIO. Fenced code +// blocks (including the fence delimiters) and anything else (prose +// paragraphs, image markdown, HTML, etc.) count against the ratio. Tuned to +// accept real-world files that include a short prose preamble/epilogue while +// still rejecting prose-heavy or llms-full.txt-shaped documents. +const MIN_LINK_ROWS = 10 +const MIN_CONFORMING_RATIO = 0.7 export function analyzeLlmsTxt(body, llmsUrl) { const parsed = parseLlmsTxt(body, llmsUrl) - const reason = getSkipReason(body, parsed) + const lineStats = classifyLines(body) + const linkItems = parsed.sections.reduce((sum, s) => sum + s.items.length, 0) + const reason = getSkipReason(body, linkItems, lineStats) return { parsed, usable: !reason, reason, + stats: { ...lineStats, linkItems }, } } @@ -46,15 +61,54 @@ export function parseLlmsTxt(body, llmsUrl) { return { title, sections } } -function getSkipReason(body, parsed) { +function getSkipReason(body, linkItems, lineStats) { if (/^---\r?\n/.test(body)) return 'starts with YAML frontmatter' - if (/^\s*(?:```|~~~)/m.test(body)) return 'contains fenced code blocks' - if (/^!\[/m.test(body)) return 'contains image markdown' + if (linkItems < MIN_LINK_ROWS) { + return `only ${linkItems} link item${linkItems === 1 ? '' : 's'} (need at least ${MIN_LINK_ROWS})` + } + if (lineStats.ratio < MIN_CONFORMING_RATIO) { + return `conforming-line ratio ${lineStats.ratio.toFixed(2)} below ${MIN_CONFORMING_RATIO} (${lineStats.conforming}/${lineStats.total} lines)` + } + return null +} - const itemCount = parsed.sections.reduce((sum, section) => sum + section.items.length, 0) - if (itemCount === 0) return 'contains no standard llms.txt link rows' +/** + * Walk the body line-by-line and bucket each line as conforming or not. + * + * conforming blank, H1, H2, blockquote, standard link-row + * non-conforming everything else, plus every line inside (and the delimiters + * of) a fenced code block + */ +function classifyLines(body) { + const lines = body.split(/\r?\n/) + let inFence = false + let conforming = 0 + let nonConforming = 0 - return null + for (const line of lines) { + if (FENCE_RE.test(line)) { + nonConforming++ + inFence = !inFence + continue + } + if (inFence) { + nonConforming++ + continue + } + if (line.trim() === '') { + conforming++ + continue + } + if (H1_RE.test(line) || H2_RE.test(line) || STANDARD_LIST_LINK_RE.test(line) || BLOCKQUOTE_RE.test(line)) { + conforming++ + continue + } + nonConforming++ + } + + const total = conforming + nonConforming + const ratio = total === 0 ? 0 : conforming / total + return { conforming, nonConforming, total, ratio } } function parseListLink(line, llmsUrl) { From 608aeba6c3a5231fb46bad35635251fa659307fc Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Mon, 18 May 2026 18:49:33 +1000 Subject: [PATCH 04/43] feat: use llms.txt if average section size less than 50 pages and bad firecrawl scrape --- src/commands/import.js | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 5d3a826..8f80cc6 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -322,9 +322,29 @@ export async function importDocs(options) { categories: scraped.categories.map((c) => ({ title: c.title, icon: null, pages: c.pages })), } } else if (llms) { - const fastPath = sectionsLookUsable(llms.parsed.sections) - styles.info(`Organizing with Claude (${styles.bold(options.model)}, ${fastPath ? 'fast path: icons only' : 'full reorg'})...`) - organized = await timePhase('claude organize', () => organizeWithClaude(llms.parsed, options.model)) + const usable = usableSections(llms.parsed.sections) + const avgPages = usable.length > 0 ? usable.reduce((n, s) => n + s.items.length, 0) / usable.length : 0 + if (sectionsLookUsable(llms.parsed.sections) && avgPages < 50) { + styles.info( + `Using llms.txt sections directly — ${styles.bold(String(usable.length))} sections, ${styles.bold(avgPages.toFixed(1))} avg pages/section (≤50). Skipping Claude.`, + ) + organized = { + title: llms.parsed.title || null, + categories: usable.map((s) => ({ + title: s.title, + icon: null, + pages: s.items.map((it) => ({ + title: it.text, + url: it.url, + ...(it.description ? { description: it.description } : {}), + })), + })), + } + } else { + const fastPath = sectionsLookUsable(llms.parsed.sections) + styles.info(`Organizing with Claude (${styles.bold(options.model)}, ${fastPath ? 'fast path: icons only' : 'full reorg'})...`) + organized = await timePhase('claude organize', () => organizeWithClaude(llms.parsed, options.model)) + } } else { // Sitemap-only fallback: synthesize categories by clustering the URL // paths. clusterByUrlPath returns null when the URLs don't split cleanly, From 3aa1c0641b18de89642695a03804bf0faff36805 Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Tue, 19 May 2026 11:06:36 +1000 Subject: [PATCH 05/43] feat: better parent-child nesting based on urls --- src/commands/import.js | 322 ++++++++++++++++++++++++++++++----------- 1 file changed, 235 insertions(+), 87 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 8f80cc6..d374a96 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -244,39 +244,66 @@ export async function importDocs(options) { } if (slotted.length > 0) { - // Mintlify-style docs put API endpoints in a separate tab rooted at - // /api-reference/* (or /api/*, /reference/*). Collapse remaining such - // pages into a single "API Reference" category (absorbing the flat - // category the sweep pass just built, if any) and nest it by resource - // segment so routeCategory() maps the whole thing to ReadMe's - // `reference/` top-level dir. - const apiResult = collectApiReferencePages(slotted, scraped) - const otherOrphans = apiResult.nonApiOrphans - if (apiResult.category) scraped.categories.push(apiResult.category) - - const buckets = bucketOrphansByPathType(otherOrphans, scraped) - for (const b of buckets) scraped.categories.push(b) - if (debugSnapshots) { - debugSnapshots['04-after-orphan-buckets.json'] = { - apiReferenceCollected: apiResult.category - ? { - pageCount: apiResult.category.pages.length, - mergedFromScraped: apiResult.mergedScrapedTitles, - } - : null, - buckets, - scraped: JSON.parse(JSON.stringify(scraped)), + // When orphans dwarf direct matches, the sidebar scrape was too thin + // to trust as the import's spine — keeping it would produce a small + // "real" tree plus a soup of bucketed-by-URL-type orphan categories. + // Discard the scrape and cluster every page (scrape + orphans) by + // its top URL segment instead, so each `/docs/`, `/sdk/`, `/rest-api/` + // becomes its own category. `nestByUrlHierarchy` (later) handles the + // empty-parent nesting within each category. + + // Trip when orphans are at least 2× the direct matches: the scrape + // accounts for less than a third of the known pages, so its category + // labels aren't a trustworthy spine for the remainder. + const orphansDwarfDirect = slotted.length >= directMatches * 2 + const scrapeAllPages = scraped.categories.flatMap((c) => c.pages) + const reclustered = orphansDwarfDirect ? clusterByUrlPath([...scrapeAllPages, ...slotted]) : null + if (reclustered) { + scraped.categories = reclustered.map((c) => ({ title: c.title, pages: c.pages })) + styles.info( + `Scrape covered only ${styles.bold(String(directMatches))}/${styles.bold(String(knownUrls.length))} pages — discarded as too thin and re-clustered ${styles.bold(String(scrapeAllPages.length + slotted.length))} pages by URL path into ${styles.bold(String(reclustered.length))} categor${reclustered.length === 1 ? 'y' : 'ies'}: ${reclustered.map((c) => styles.bold(c.title)).join(', ')}.`, + ) + if (debugSnapshots) { + debugSnapshots['04-after-orphan-buckets.json'] = { + mode: 'reclustered-by-url-path', + scraped: JSON.parse(JSON.stringify(scraped)), + } + } + } else { + // Mintlify-style docs put API endpoints in a separate tab rooted at + // /api-reference/* (or /api/*, /reference/*). Collapse remaining such + // pages into a single "API Reference" category (absorbing the flat + // category the sweep pass just built, if any) and nest it by resource + // segment so routeCategory() maps the whole thing to ReadMe's + // `reference/` top-level dir. + const apiResult = collectApiReferencePages(slotted, scraped) + const otherOrphans = apiResult.nonApiOrphans + if (apiResult.category) scraped.categories.push(apiResult.category) + + const buckets = bucketOrphansByPathType(otherOrphans, scraped) + for (const b of buckets) scraped.categories.push(b) + if (debugSnapshots) { + debugSnapshots['04-after-orphan-buckets.json'] = { + apiReferenceCollected: apiResult.category + ? { + pageCount: apiResult.category.pages.length, + mergedFromScraped: apiResult.mergedScrapedTitles, + } + : null, + buckets, + scraped: JSON.parse(JSON.stringify(scraped)), + } + } + const parts = [] + if (apiResult.category) { + parts.push(`${styles.bold(String(apiResult.category.pages.length))} in ${styles.bold('API Reference')}`) + } + for (const b of buckets) { + parts.push(`${styles.bold(String(b.pages.length))} in ${styles.bold(b.title)}`) + } + if (parts.length > 0) { + styles.info(`${styles.bold(String(slotted.length))} orphan page${slotted.length === 1 ? '' : 's'} bucketed by URL type: ${parts.join(', ')}.`) } - } - const parts = [] - if (apiResult.category) { - parts.push(`${styles.bold(String(apiResult.category.pages.length))} in ${styles.bold('API Reference')}`) - } - for (const b of buckets) { - parts.push(`${styles.bold(String(b.pages.length))} in ${styles.bold(b.title)}`) - } - if (parts.length > 0) { - styles.info(`${styles.bold(String(slotted.length))} orphan page${slotted.length === 1 ? '' : 's'} bucketed by URL type: ${parts.join(', ')}.`) } } } else { @@ -1648,70 +1675,164 @@ function collectFlat(page, out) { } /** - * Re-parent URL-bearing siblings by URL path so a page like `/foo/bar` - * becomes a child of `/foo` when both appear in the same list. + * Parse a URL's pathname into segments suitable for trie nesting. Strips file + * extensions (`.md`/`.html`/etc.) and drops a trailing `index` segment so + * `/foo`, `/foo/`, `/foo/index.html`, and `/foo/index.md` all collapse to the + * same logical page — the static-site-generator convention every browser and + * web server already follows. Returns null if the URL can't be parsed. + */ +function urlTrieSegs(url) { + try { + const segs = new URL(url).pathname + .split('/') + .filter(Boolean) + .map((s) => s.replace(/\.(md|mdx|html?)$/i, '')) + .filter(Boolean) + if (segs.length > 0 && segs[segs.length - 1].toLowerCase() === 'index') segs.pop() + return segs + } catch { + return null + } +} + +/** + * Re-parent URL-bearing siblings by URL path so descendants nest under their + * URL prefix even when no page exists at every intermediate segment. We build + * a trie from each page's URL path and render it back out as a page tree: + * + * - The shared top-level prefix is elided (the category folder is its + * implicit container — e.g. /docs/ai/evals collapses into "Pydantic Evals"). + * - Below the elision point, every segment becomes an entry: a real page + * where one exists, otherwise an empty-parent stub (folder-icon, no URL). + * + * `anchorSegs` is the parent page's URL segments during recursion; when set, + * we walk the trie to that exact depth instead of eliding to the natural + * branch point, so a missing intermediate like `b` in `/a → /a/b/c, /a/b/d` + * still renders as an empty parent under `/a`. * * @example * nestByUrlHierarchy([ - * { title: 'CLI', url: 'https://x.com/getting-started/cli' }, - * { title: 'Overview', url: 'https://x.com/getting-started/cli/overview' }, + * { title: 'Hello', url: 'https://x.com/main/sub1/sub2/sub3/hello' }, + * { title: 'World', url: 'https://x.com/main/sub1/sub2/sub3/world' }, + * { title: 'Sub 1', url: 'https://x.com/main/sub1' }, * ]) * // [ - * // { title: 'CLI', url: '…/cli', pages: [ - * // { title: 'Overview', url: '…/cli/overview' }, + * // { title: 'Sub 1', url: '…/main/sub1', pages: [ + * // { title: 'Sub2', _emptyParent: true, pages: [ + * // { title: 'Sub3', _emptyParent: true, pages: [ + * // { title: 'Hello', url: '…/main/sub1/sub2/sub3/hello' }, + * // { title: 'World', url: '…/main/sub1/sub2/sub3/world' }, + * // ]}, + * // ]}, * // ]}, * // ] */ -function nestByUrlHierarchy(pages) { +function nestByUrlHierarchy(pages, anchorSegs = null) { if (!pages || pages.length === 0) return pages - // Recurse first so group-only / already-nested subtrees get re-nested too. + // Recurse first so subtrees that came in pre-nested (e.g. from the scraper) + // get re-nested against their parent's URL as anchor. for (const p of pages) { - if (p.pages && p.pages.length > 0) p.pages = nestByUrlHierarchy(p.pages) - } - - // Need at least two URL-bearing siblings to form a parent/child pair. - const byPath = new Map() - for (const p of pages) { - if (!p.url) continue - const k = normalizePath(p.url) - if (!byPath.has(k)) byPath.set(k, p) + if (p.pages && p.pages.length > 0) { + let nextAnchor = anchorSegs + if (p._virtualPathSegs) nextAnchor = p._virtualPathSegs + else if (p.url) { + const s = urlTrieSegs(p.url) + if (s) nextAnchor = s + } + p.pages = nestByUrlHierarchy(p.pages, nextAnchor) + } } - if (byPath.size < 2) return pages - const result = [] + const urlEntries = [] + const groupOnly = [] for (const p of pages) { if (!p.url) { - result.push(p) - continue - } - let segs - try { - segs = new URL(p.url).pathname.split('/').filter(Boolean) - } catch { - result.push(p) + groupOnly.push(p) continue } - let parent = null - for (let depth = segs.length - 1; depth >= 1; depth--) { - const key = ('/' + segs.slice(0, depth).join('/')) - .toLowerCase() - .replace(/\.(md|mdx|html?)$/i, '') - .replace(/\/$/, '') - const candidate = byPath.get(key) - if (candidate && candidate !== p) { - parent = candidate - break + const segs = urlTrieSegs(p.url) + if (segs) urlEntries.push({ page: p, segs }) + else groupOnly.push(p) + } + + if (urlEntries.length === 0) return pages + if (urlEntries.length === 1 && groupOnly.length === 0) return pages + + // Build the URL trie. Each node tracks its own segment, the page at that + // exact path (if any), insertion-ordered children, and the segment path + // from root (used as the slug source for empty parents). + const root = { segment: '', page: null, children: new Map(), segs: [] } + for (const { page, segs } of urlEntries) { + let node = root + for (let i = 0; i < segs.length; i++) { + const segLower = segs[i].toLowerCase() + if (!node.children.has(segLower)) { + node.children.set(segLower, { segment: segs[i], page: null, children: new Map(), segs: segs.slice(0, i + 1) }) } + node = node.children.get(segLower) } - if (parent) { - if (!parent.pages) parent.pages = [] - parent.pages.push(p) + if (!node.page) node.page = page + } + + // Find the trie node that represents the "container" for this level. + // - With an anchor: that's the parent page's exact depth; we don't elide + // past it, so intermediate segments below the anchor become empties. + // - Without: walk down while there's only one no-page child (the shared + // prefix). Stop at the first node that either has a page or branches. + let container = root + if (anchorSegs && anchorSegs.length > 0) { + for (let i = 0; i < anchorSegs.length; i++) { + const seg = anchorSegs[i].toLowerCase() + if (container.children.has(seg)) container = container.children.get(seg) + else break + } + } else { + while (container.children.size === 1 && !container.page) { + container = container.children.values().next().value + } + } + + function nodeToPage(node) { + let outPage + if (node.page) { + outPage = node.page } else { - result.push(p) + const rawSeg = node.segment + const cleanedSeg = rawSeg.replace(/\.(md|mdx|html?)$/i, '').replace(/^\d+[-_.]/, '') || rawSeg + outPage = { + title: titleCase(cleanedSeg), + _emptyParent: true, + _virtualPathSegs: node.segs, + pages: [], + } + } + if (node.children.size > 0) { + const childPages = [] + for (const c of node.children.values()) childPages.push(nodeToPage(c)) + // Preserve any pre-existing nested children whose URLs aren't already + // represented in the trie (group-only containers from the scraper, etc.). + const existing = outPage.pages || [] + const trieUrls = new Set() + const collect = (p) => { + if (p.url) trieUrls.add(normalizePath(p.url)) + for (const c of p.pages || []) collect(c) + } + for (const cp of childPages) collect(cp) + const preservedExisting = existing.filter((e) => !e.url || !trieUrls.has(normalizePath(e.url))) + outPage.pages = [...childPages, ...preservedExisting] } + return outPage } - return result + + const trieResult = [] + if (container.page) { + trieResult.push(nodeToPage(container)) + } else { + for (const c of container.children.values()) trieResult.push(nodeToPage(c)) + } + + return [...trieResult, ...groupOnly] } /** @@ -2575,9 +2696,14 @@ function stageOrganized(organized, stagingDir, opts = {}) { // Slug names must be unique const slugFor = ensureUniqueSlugs(eligibleCategories) - const urlWidth = Math.max(...[...slugFor.keys()].map((p) => (p.url || '(group-only)').length)) + const labelFor = (p) => { + if (p.url) return p.url + if (p._emptyParent && p._virtualPathSegs) return `(empty) /${p._virtualPathSegs.join('/')}` + return '(group-only)' + } + const urlWidth = Math.max(...[...slugFor.keys()].map((p) => labelFor(p).length)) for (const [page, slug] of slugFor) { - console.log(`${(page.url || '(group-only)').padEnd(urlWidth)} → ${slug}`) + console.log(`${labelFor(page).padEnd(urlWidth)} → ${slug}`) } /** @@ -2593,18 +2719,27 @@ function stageOrganized(organized, stagingDir, opts = {}) { // no backing page on the source site — they're pure sidebar containers. // Skip the stub write but still recurse so their children land in the // right subdirectory. - const isGroupOnly = !page.url + const isEmptyParent = !!page._emptyParent + const isGroupOnly = !page.url && !isEmptyParent if (!isGroupOnly) { const relFilePath = `${dir}/${slug}.md` - // Sub-pages don't get icons per design decision. - const frontmatter = buildFrontmatter(topDir, page, slug, pickIcon, { skipIcon: isSubPage }) - // x-import points at the source URL for this stub. The content-import - // step reads it to fetch the page body. x-prefixed custom field is the - // git-format convention for metadata the schema doesn't know about. - frontmatter['x-import'] = toBrowsableUrl(page.url) - // hide pages that need import - frontmatter.hidden = 'true' + let frontmatter + if (isEmptyParent) { + // Synthetic folder page: covers a missing URL segment so descendants + // nest at the right depth. No source URL to import, and we want it + // visible so the folder shows up in the sidebar. + frontmatter = { title: page.title, icon: 'fa-solid fa-folder' } + } else { + // Sub-pages don't get icons per design decision. + frontmatter = buildFrontmatter(topDir, page, slug, pickIcon, { skipIcon: isSubPage }) + // x-import points at the source URL for this stub. The content-import + // step reads it to fetch the page body. x-prefixed custom field is the + // git-format convention for metadata the schema doesn't know about. + frontmatter['x-import'] = toBrowsableUrl(page.url) + // hide pages that need import + frontmatter.hidden = 'true' + } const absPath = path.join(stagingDir, relFilePath) fs.mkdirSync(path.dirname(absPath), { recursive: true }) @@ -2705,15 +2840,20 @@ function buildFrontmatter(topDir, page, slug, pickIcon, opts = {}) { * Extract URL path segments for slug planning. Strips file extensions and * leading numeric prefixes (e.g. `01-intro` → `intro`) the same way the * legacy deriveSlug did, so the depth-1 result is byte-for-byte compatible. + * Also drops a trailing `index` segment — many SSGs render `/foo/` as + * `/foo/index.html` and emit either form in their URL lists; we don't want + * every page collapsing to the same `index` base slug. */ function extractUrlPathSegments(url) { if (!url) return [] try { - return new URL(url).pathname + const segs = new URL(url).pathname .split('/') .filter(Boolean) .map((s) => s.replace(/\.(md|mdx|html?)$/i, '').replace(/^\d+[-_.]/, '')) .filter(Boolean) + if (segs.length > 0 && segs[segs.length - 1].toLowerCase() === 'index') segs.pop() + return segs } catch { return [] } @@ -2744,9 +2884,17 @@ function extractUrlPathSegments(url) { */ function ensureUniqueSlugs(categories) { const entries = [] + const segmentsFor = (p) => { + if (p._virtualPathSegs && p._virtualPathSegs.length > 0) { + return p._virtualPathSegs + .map((s) => s.replace(/\.(md|mdx|html?)$/i, '').replace(/^\d+[-_.]/, '')) + .filter(Boolean) + } + return extractUrlPathSegments(p.url) + } const walk = (pages) => { for (const p of pages || []) { - entries.push({ page: p, segments: extractUrlPathSegments(p.url), fallback: p.title, depth: 1 }) + entries.push({ page: p, segments: segmentsFor(p), fallback: p.title, depth: 1 }) if (p.pages) walk(p.pages) } } From a9cdb995c6b1e1addbf0a4a1f241cda6a8f58290 Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Tue, 19 May 2026 11:35:11 +1000 Subject: [PATCH 06/43] feat: better handle paths with no segments with % thresholding --- src/commands/import.js | 67 ++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index d374a96..60f1732 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -257,7 +257,15 @@ export async function importDocs(options) { // labels aren't a trustworthy spine for the remainder. const orphansDwarfDirect = slotted.length >= directMatches * 2 const scrapeAllPages = scraped.categories.flatMap((c) => c.pages) - const reclustered = orphansDwarfDirect ? clusterByUrlPath([...scrapeAllPages, ...slotted]) : null + let reclustered = null + if (orphansDwarfDirect) { + reclustered = clusterByUrlPath([...scrapeAllPages, ...slotted]) + } + styles.info( + styles.dim( + ` orphan triage: ${slotted.length} orphan${slotted.length === 1 ? '' : 's'} vs ${directMatches} direct match${directMatches === 1 ? '' : 'es'} (ratio ${directMatches === 0 ? '∞' : (slotted.length / directMatches).toFixed(2)}) — gate ${orphansDwarfDirect ? 'tripped' : 'NOT tripped'} (need ≥2.00); URL re-cluster ${reclustered ? `→ ${reclustered.length} categor${reclustered.length === 1 ? 'y' : 'ies'}` : 'skipped'}`, + ), + ) if (reclustered) { scraped.categories = reclustered.map((c) => ({ title: c.title, pages: c.pages })) styles.info( @@ -2049,14 +2057,29 @@ function urlNamespace(url) { function clusterByUrlPath(pages) { if (!pages || pages.length < 3) return null - const parts = pages.map((p) => { + // Pages with no path segments (e.g. an origin-only URL like + // `https://build.example.com/`) can't participate in segment-based + // clustering. A handful are fine — we'll fold them into the first cluster + // as a place to land — but if more than 20% of the input is rootless, + // the source isn't really a hierarchical doc set and clustering won't + // produce a meaningful tree. Bail in that case so the caller falls back. + const noSegPages = [] + const clusterable = [] + const parts = [] + for (const p of pages) { + let segs = [] try { - return new URL(p.url).pathname.split('/').filter(Boolean) - } catch { - return [] + segs = new URL(p.url).pathname.split('/').filter(Boolean) + } catch {} + if (segs.length === 0) { + noSegPages.push(p) + } else { + clusterable.push(p) + parts.push(segs) } - }) - if (parts.some((pp) => pp.length === 0)) return null + } + if (noSegPages.length / pages.length > 0.2) return null + if (clusterable.length < 3) return null // Longest common prefix depth. let commonDepth = 0 @@ -2069,13 +2092,13 @@ function clusterByUrlPath(pages) { // The segment right after the common base is the category key. const keyIdx = commonDepth const byKey = new Map() - for (let i = 0; i < pages.length; i++) { + for (let i = 0; i < clusterable.length; i++) { const key = parts[i][keyIdx] // Skip pages that have no segment at the cluster index (they're AT the // common base — those would become their own "index"-like category). if (!key) continue if (!byKey.has(key)) byKey.set(key, []) - byKey.get(key).push(pages[i]) + byKey.get(key).push(clusterable[i]) } // Reject weak clusterings: need at least 2 groups AND at least one group @@ -2086,7 +2109,7 @@ function clusterByUrlPath(pages) { // Preserve first-appearance order so the sidebar reflects source order. const firstSeen = new Map() - pages.forEach((p, i) => { + clusterable.forEach((p, i) => { const key = parts[i][keyIdx] if (key && !firstSeen.has(key)) firstSeen.set(key, i) }) @@ -2101,21 +2124,29 @@ function clusterByUrlPath(pages) { // a parent page with children wrapped in a pseudo-category label. Categories // are grouping labels with no content of their own; parent pages have // content AND children. Collect those singletons into a shared - // "Documentation" bucket so they're siblings at the top level, each with - // their own sub-tree intact. + // "Other Documentation" bucket at the bottom of the sidebar so they're + // siblings at the top level, each with their own sub-tree intact, without + // crowding the real categories above. const multipageClusters = rawClusters.filter((c) => c.pages.length >= 2) const singletonPages = rawClusters.filter((c) => c.pages.length === 1).flatMap((c) => c.pages) - const out = [] - if (singletonPages.length > 0) { - out.push({ title: 'Documentation', pages: singletonPages }) - } - out.push(...multipageClusters) - // If we didn't actually produce any multi-page cluster, clustering added no // value — every page was a singleton and we'd just have renamed Overview. // Tell the caller to stick with the original flat shape. if (multipageClusters.length === 0) return null + + const out = [...multipageClusters] + + // Park any zero-segment pages on the first real cluster. They don't fit + // segment-based clustering but the >20% guard above already proved they're + // a minority, so dropping them onto whatever lands first is a reasonable + // home rather than their own misfit category. + if (noSegPages.length > 0) out[0].pages.push(...noSegPages) + + // "Other Documentation" goes last so it never displaces a real category. + if (singletonPages.length > 0) { + out.push({ title: 'Other Documentation', pages: singletonPages }) + } return out } From 94d22b658c25f990305e9c53027e167e2cd8c532 Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Tue, 19 May 2026 13:13:40 +1000 Subject: [PATCH 07/43] fix: root level causing issues if only root level pages --- src/commands/import.js | 81 ++++++++++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 27 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 60f1732..e5b8b0c 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -150,19 +150,32 @@ export async function importDocs(options) { const rawKnownUrls = llms.parsed.sections.flatMap((s) => s.items.map((i) => ({ title: i.text, url: i.url, description: i.description }))) + // Drop asset URLs (openapi.json, package.json, .yaml specs, etc.) — they + // show up in some llms.txt files alongside real pages but aren't docs and + // shouldn't be staged as page stubs. + const ASSET_EXT_RE = /\.(json|ya?ml|xml|toml)$/i + const isAssetUrl = (url) => { + try { return ASSET_EXT_RE.test(new URL(url).pathname) } catch { return false } + } + const assetsDropped = rawKnownUrls.filter((p) => isAssetUrl(p.url)).length + const filtered = rawKnownUrls.filter((p) => !isAssetUrl(p.url)) + // Dedupe llms.txt entries by pathname. Some sites (zod.dev, fumadocs) list // every in-page anchor as its own llms.txt row (`/v4?id=wrapping-up`, // `/v4?id=metadata`, …) even though they all live on one rendered page. // We prefer the "cleanest" URL per path — the shortest one, which is // usually the one without a query string or hash. const byKnownPath = new Map() - for (const p of rawKnownUrls) { + for (const p of filtered) { const key = normalizePath(p.url) const prev = byKnownPath.get(key) if (!prev || p.url.length < prev.url.length) byKnownPath.set(key, p) } knownUrls = Array.from(byKnownPath.values()) - const dropped = rawKnownUrls.length - knownUrls.length + const dropped = filtered.length - knownUrls.length + if (assetsDropped > 0) { + styles.info(`${styles.dim(`Dropped ${assetsDropped} asset URL${assetsDropped === 1 ? '' : 's'} (.json/.yaml/.xml/.toml).`)}`) + } if (dropped > 0) { styles.info(`${styles.dim(`Collapsed ${dropped} anchor/query duplicates → ${knownUrls.length} unique pages.`)}`) } @@ -1760,7 +1773,12 @@ function nestByUrlHierarchy(pages, anchorSegs = null) { continue } const segs = urlTrieSegs(p.url) - if (segs) urlEntries.push({ page: p, segs }) + // Zero-segment URLs (origin-only links like `https://build.example.com/`) + // have no path to nest by — if we inserted them into the trie they'd land + // at the root node and the container walk would render them as the + // top-level entry with every other page nested beneath them. Treat them + // like group-only pages so they just sit alongside the real tree. + if (segs && segs.length > 0) urlEntries.push({ page: p, segs }) else groupOnly.push(p) } @@ -2120,32 +2138,41 @@ function clusterByUrlPath(pages) { pages: byKey.get(key), })) - // A cluster with exactly one top-level page is NOT a real category — it's - // a parent page with children wrapped in a pseudo-category label. Categories - // are grouping labels with no content of their own; parent pages have - // content AND children. Collect those singletons into a shared - // "Other Documentation" bucket at the bottom of the sidebar so they're - // siblings at the top level, each with their own sub-tree intact, without - // crowding the real categories above. - const multipageClusters = rawClusters.filter((c) => c.pages.length >= 2) - const singletonPages = rawClusters.filter((c) => c.pages.length === 1).flatMap((c) => c.pages) - - // If we didn't actually produce any multi-page cluster, clustering added no - // value — every page was a singleton and we'd just have renamed Overview. - // Tell the caller to stick with the original flat shape. - if (multipageClusters.length === 0) return null - - const out = [...multipageClusters] - - // Park any zero-segment pages on the first real cluster. They don't fit - // segment-based clustering but the >20% guard above already proved they're - // a minority, so dropping them onto whatever lands first is a reasonable - // home rather than their own misfit category. - if (noSegPages.length > 0) out[0].pages.push(...noSegPages) + // Classify each cluster as either a real category or an "Other + // Documentation" misfit. A cluster is a real category when: + // - it has 2+ pages, OR + // - its single page has at least one URL segment BEYOND the cluster key + // (so the key represents a folder-with-one-page, not a standalone page). + // A cluster with exactly one page whose path IS just the cluster key (e.g. + // `/help` alone in a "Help" cluster) is a standalone page with no folder + // semantic — that goes into the "Other Documentation" misfit bucket. + const realCategories = [] + const misfitPages = [] + for (const cluster of rawClusters) { + if (cluster.pages.length >= 2) { + realCategories.push(cluster) + continue + } + const onlyPage = cluster.pages[0] + let pageSegs = [] + try { pageSegs = new URL(onlyPage.url).pathname.split('/').filter(Boolean) } catch {} + if (pageSegs.length > keyIdx + 1) realCategories.push(cluster) + else misfitPages.push(onlyPage) + } + + // If clustering produced no real categories, it added no value — tell the + // caller to fall back rather than hand back a lone misfit bucket. + if (realCategories.length === 0) return null + + const out = [...realCategories] // "Other Documentation" goes last so it never displaces a real category. - if (singletonPages.length > 0) { - out.push({ title: 'Other Documentation', pages: singletonPages }) + // It absorbs both standalone-page singletons and zero-segment URLs (those + // can't participate in segment-based clustering at all but the >20% guard + // above already proved they're a minority). + const otherDocs = [...misfitPages, ...noSegPages] + if (otherDocs.length > 0) { + out.push({ title: 'Other Documentation', pages: otherDocs }) } return out } From 3095c96e9aca9092377dee9161e5c9d8ee2dfd9f Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Wed, 20 May 2026 00:18:41 +1000 Subject: [PATCH 08/43] feat: handle multi-llms.txt sites, other small fixes --- src/commands/import.js | 522 +++++++++++++++++++++++++++++++++++------ src/utils/llms.js | 47 ++-- 2 files changed, 477 insertions(+), 92 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index e5b8b0c..ffc3ad8 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -79,27 +79,69 @@ export async function importDocs(options) { if (!options.test) styles.info(`Output: ${styles.bold(outputZip)}`) console.log() - // Build the list of llms.txt URLs to probe, walking up the supplied path - // from most-specific to root. For `https://mintlify.com/docs/quickstart` - // we try `/docs/quickstart/llms.txt`, then `/docs/llms.txt`, then root. - // This catches sites that scope llms.txt to a docs subpath. - const llmsCandidates = buildLlmsCandidates(sourceUrl) - styles.info(`Checking for llms.txt (${llmsCandidates.length} candidate${llmsCandidates.length === 1 ? '' : 's'})...`) - - const { llms, llmsUrl, skippedLlms } = await timePhase('fetch llms.txt', async () => { - const skipped = [] - for (const candidate of llmsCandidates) { - const res = await fetchLlmsTxt(candidate) - if (res.ok) { - if (res.usable) return { llms: res, llmsUrl: candidate, skippedLlms: skipped } - skipped.push({ url: candidate, reason: res.reason }) - styles.info(styles.dim(` ${candidate} → skipped (${res.reason})`)) - continue - } - styles.info(styles.dim(` ${candidate} → ${res.status ? `HTTP ${res.status}` : res.error || 'failed'}`)) + // Discover llms.txt files by BFS walking parents AND children. Seed the + // frontier with the walk-up paths (source → root). Any "hit" (file exists + // with at least one link row) expands the frontier: parent path + distinct + // first-child segments extracted from its URL list. Invalid hits (fail the + // usability ratio) still expand — they're stepping stones — only valid + // hits contribute content to the merged result. See + // mydocs.com/section1/section2 → also tries /section1/, /, and every + // sibling section2 we discover along the way. + styles.info(`Discovering llms.txt (walking up and down from ${styles.bold(sourceUrl.toString())})...`) + + const discovery = await timePhase('discover llms.txt', () => discoverLlmsTxt(sourceUrl)) + for (const hit of discovery.hits) { + if (hit.usable) { + styles.info(styles.dim(` ${hit.llmsUrl} → valid (${hit.stats.linkItems} links)`)) + } else { + styles.info(styles.dim(` ${hit.llmsUrl} → invalid: ${hit.reason} (kept as stepping stone)`)) } - return { llms: null, llmsUrl: null, skippedLlms: skipped } - }) + } + for (const miss of discovery.misses) { + styles.info(styles.dim(` ${miss.url} → ${miss.status ? `HTTP ${miss.status}` : miss.error || 'no llms.txt'}`)) + } + let llms = mergeValidHits(discovery.hits) + let llmsUrl = llms?.llmsUrl || null + const skippedLlms = discovery.skipped + + // Drop asset/meta items (llms-full.txt, openapi.json, …) from the merged + // parsed result up-front, so every downstream consumer — knownUrls AND the + // "use llms.txt sections directly" organize path — sees a clean URL list. + if (llms) { + const dropped = dropAssetItemsFromParsed(llms.parsed) + if (dropped > 0) { + styles.info(styles.dim(`Dropped ${dropped} asset/meta URL${dropped === 1 ? '' : 's'} (.json/.yaml/.xml/.toml, llms*.txt) from llms.txt items.`)) + } + } + + if (llms) { + const narrowed = narrowToDocsSubtreeIfNeeded(llms, sourceUrl, discovery.hits) + if (narrowed) { + styles.info( + styles.dim( + `Narrowed merged result to /${narrowed.segment}/ subtree (kept ${narrowed.kept}, dropped ${narrowed.dropped} out-of-scope page${narrowed.dropped === 1 ? '' : 's'}).`, + ), + ) + } + } + + // Fall back to scrape if the merged llms.txt covers too few endpoints to + // be useful as a structural backbone. Replaces the old per-file + // MIN_LINK_ROWS=10 check (now relaxed in utils/llms.js) with a global + // post-merge floor — many tiny per-product llms.txt files can still pass + // as long as their union clears the bar. + const MIN_MERGED_LINK_COUNT = 10 + if (llms) { + const mergedLinkCount = llms.parsed.sections.reduce((n, s) => n + s.items.length, 0) + if (mergedLinkCount < MIN_MERGED_LINK_COUNT) { + styles.warning( + `Merged llms.txt has only ${styles.bold(String(mergedLinkCount))} endpoint${mergedLinkCount === 1 ? '' : 's'} (< ${MIN_MERGED_LINK_COUNT}) — falling back to scrape.`, + ) + skippedLlms.push({ url: llmsUrl, reason: `merged total only ${mergedLinkCount} link${mergedLinkCount === 1 ? '' : 's'}` }) + llms = null + llmsUrl = null + } + } console.log() let sitemapUrl = null @@ -133,7 +175,11 @@ export async function importDocs(options) { } } else { const s = llms.stats - styles.info(styles.dim(`Using ${llmsUrl} (${s.conforming}/${s.total} lines conforming, ratio ${s.ratio.toFixed(2)}).`)) + if (llms.sourceFiles.length === 1) { + styles.info(styles.dim(`Using ${llmsUrl} (${s.conforming}/${s.total} lines conforming, ratio ${s.ratio.toFixed(2)}).`)) + } else { + styles.info(styles.dim(`Merged ${llms.sourceFiles.length} llms.txt files (root: ${llmsUrl}; aggregate ratio ${s.ratio.toFixed(2)}).`)) + } } if (debugSnapshots) { @@ -144,38 +190,28 @@ export async function importDocs(options) { let knownUrls = [] if (llms) { const totalItems = llms.parsed.sections.reduce((n, s) => n + s.items.length, 0) + const srcCount = llms.sourceFiles.length + const srcLabel = srcCount === 1 ? 'llms.txt' : `${srcCount} llms.txt files (merged)` styles.ok( - `Found llms.txt — ${styles.bold(String(totalItems))} page${totalItems === 1 ? '' : 's'} across ${styles.bold(String(llms.parsed.sections.length))} section${llms.parsed.sections.length === 1 ? '' : 's'}${llms.parsed.title ? ` (${llms.parsed.title})` : ''}.`, + `Found ${srcLabel} — ${styles.bold(String(totalItems))} page${totalItems === 1 ? '' : 's'} across ${styles.bold(String(llms.parsed.sections.length))} section${llms.parsed.sections.length === 1 ? '' : 's'}${llms.parsed.title ? ` (${llms.parsed.title})` : ''}.`, ) const rawKnownUrls = llms.parsed.sections.flatMap((s) => s.items.map((i) => ({ title: i.text, url: i.url, description: i.description }))) - // Drop asset URLs (openapi.json, package.json, .yaml specs, etc.) — they - // show up in some llms.txt files alongside real pages but aren't docs and - // shouldn't be staged as page stubs. - const ASSET_EXT_RE = /\.(json|ya?ml|xml|toml)$/i - const isAssetUrl = (url) => { - try { return ASSET_EXT_RE.test(new URL(url).pathname) } catch { return false } - } - const assetsDropped = rawKnownUrls.filter((p) => isAssetUrl(p.url)).length - const filtered = rawKnownUrls.filter((p) => !isAssetUrl(p.url)) - // Dedupe llms.txt entries by pathname. Some sites (zod.dev, fumadocs) list // every in-page anchor as its own llms.txt row (`/v4?id=wrapping-up`, // `/v4?id=metadata`, …) even though they all live on one rendered page. // We prefer the "cleanest" URL per path — the shortest one, which is - // usually the one without a query string or hash. + // usually the one without a query string or hash. Asset/meta filtering + // already happened upstream on llms.parsed.sections. const byKnownPath = new Map() - for (const p of filtered) { + for (const p of rawKnownUrls) { const key = normalizePath(p.url) const prev = byKnownPath.get(key) if (!prev || p.url.length < prev.url.length) byKnownPath.set(key, p) } knownUrls = Array.from(byKnownPath.values()) - const dropped = filtered.length - knownUrls.length - if (assetsDropped > 0) { - styles.info(`${styles.dim(`Dropped ${assetsDropped} asset URL${assetsDropped === 1 ? '' : 's'} (.json/.yaml/.xml/.toml).`)}`) - } + const dropped = rawKnownUrls.length - knownUrls.length if (dropped > 0) { styles.info(`${styles.dim(`Collapsed ${dropped} anchor/query duplicates → ${knownUrls.length} unique pages.`)}`) } @@ -215,18 +251,21 @@ export async function importDocs(options) { if (debugSnapshots) { debugSnapshots['02-scraped-raw.json'] = scraped ? JSON.parse(JSON.stringify(scraped)) : null } - // Prefer llms.txt when it has strong multi-section structure and the - // scrape was a thin snapshot (common on big multi-tab docs — Stripe, AWS, - // Twilio — where each page only renders its own tab's sidebar). Without - // this override the 4-category scrape wins over a 25-section llms.txt and - // hundreds of real pages end up smeared into orphan buckets. + // If the sidebar scrape covers less than 75% of the llms.txt URLs, the + // scrape is too thin to trust as the import's spine. Common on multi-tab + // docs (Stripe, AWS, Twilio, Xata) where each page only renders its own + // tab's sidebar — the visible categories would otherwise absorb hundreds + // of orphan URLs via prefix-matching and produce a misleading tree (e.g. + // every /docs/* URL dumped under a single "Overview > Xata Documentation" + // node because that's the one /docs page the scrape saw). Discard the + // scrape and fall through to the llms.txt path, which uses URL-based + // clustering when multiple files were merged. if (scraped && llms && knownUrls.length > 0) { const scrapedPages = scraped.categories.reduce((n, c) => n + c.pages.length, 0) const coverage = scrapedPages / knownUrls.length - const llmsUsable = usableSections(llms.parsed.sections) - if (llmsUsable.length >= 5 && coverage < 0.5) { + if (coverage < 0.75) { styles.info( - `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages; preferring llms.txt's ${styles.bold(String(llmsUsable.length))} sections for structure.`, + `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages (need ≥75%) — discarding scrape and organizing from llms.txt.`, ) scraped = null } @@ -370,28 +409,55 @@ export async function importDocs(options) { categories: scraped.categories.map((c) => ({ title: c.title, icon: null, pages: c.pages })), } } else if (llms) { - const usable = usableSections(llms.parsed.sections) - const avgPages = usable.length > 0 ? usable.reduce((n, s) => n + s.items.length, 0) / usable.length : 0 - if (sectionsLookUsable(llms.parsed.sections) && avgPages < 50) { + if (llms.sourceFiles.length > 1) { + // Multi-file merge: H2 section headings across the merged files are + // heterogeneous and don't form a coherent spine (e.g. elevenlabs.io's + // root file uses `Docs`/`Products` while /docs/llms.txt uses + // `API Docs` — concatenating them produces meaningless top-level + // categories like `docs/Docs`, `docs/Products`, `docs/API Docs`). + // Throw away the section grouping and re-cluster by URL path instead. + // Per-page metadata (title, url, description) is still carried over. + const flatPages = llms.parsed.sections.flatMap((s) => + s.items.map((it) => ({ + title: it.text, + url: it.url, + ...(it.description ? { description: it.description } : {}), + })), + ) + const clustered = clusterByUrlPath(flatPages) || [{ title: 'Documentation', pages: flatPages }] styles.info( - `Using llms.txt sections directly — ${styles.bold(String(usable.length))} sections, ${styles.bold(avgPages.toFixed(1))} avg pages/section (≤50). Skipping Claude.`, + `Multi-file llms.txt merge — clustered ${styles.bold(String(flatPages.length))} pages by URL path into ${styles.bold(String(clustered.length))} categor${clustered.length === 1 ? 'y' : 'ies'} (H2 sections discarded).`, ) organized = { title: llms.parsed.title || null, - categories: usable.map((s) => ({ - title: s.title, - icon: null, - pages: s.items.map((it) => ({ - title: it.text, - url: it.url, - ...(it.description ? { description: it.description } : {}), - })), - })), + categories: clustered.map((c) => ({ title: c.title, icon: null, pages: c.pages })), } } else { - const fastPath = sectionsLookUsable(llms.parsed.sections) - styles.info(`Organizing with Claude (${styles.bold(options.model)}, ${fastPath ? 'fast path: icons only' : 'full reorg'})...`) - organized = await timePhase('claude organize', () => organizeWithClaude(llms.parsed, options.model)) + // Single-file mode: H2 structure IS the author's intended organization, + // so trust it. Use sections directly when usable; otherwise run Claude. + const usable = usableSections(llms.parsed.sections) + const avgPages = usable.length > 0 ? usable.reduce((n, s) => n + s.items.length, 0) / usable.length : 0 + if (sectionsLookUsable(llms.parsed.sections) && avgPages < 50) { + styles.info( + `Using llms.txt sections directly — ${styles.bold(String(usable.length))} sections, ${styles.bold(avgPages.toFixed(1))} avg pages/section (≤50). Skipping Claude.`, + ) + organized = { + title: llms.parsed.title || null, + categories: usable.map((s) => ({ + title: s.title, + icon: null, + pages: s.items.map((it) => ({ + title: it.text, + url: it.url, + ...(it.description ? { description: it.description } : {}), + })), + })), + } + } else { + const fastPath = sectionsLookUsable(llms.parsed.sections) + styles.info(`Organizing with Claude (${styles.bold(options.model)}, ${fastPath ? 'fast path: icons only' : 'full reorg'})...`) + organized = await timePhase('claude organize', () => organizeWithClaude(llms.parsed, options.model)) + } } } else { // Sitemap-only fallback: synthesize categories by clustering the URL @@ -2139,13 +2205,12 @@ function clusterByUrlPath(pages) { })) // Classify each cluster as either a real category or an "Other - // Documentation" misfit. A cluster is a real category when: - // - it has 2+ pages, OR - // - its single page has at least one URL segment BEYOND the cluster key - // (so the key represents a folder-with-one-page, not a standalone page). - // A cluster with exactly one page whose path IS just the cluster key (e.g. - // `/help` alone in a "Help" cluster) is a standalone page with no folder - // semantic — that goes into the "Other Documentation" misfit bucket. + // Documentation" misfit. A real category needs at least 2 pages — a + // "category" with a single file in it just adds a level of nesting + // without adding structure. Pydantic's `/docs/logfire/get-started` is + // the motivating example: as the only `/docs/logfire/*` URL it + // shouldn't get its own top-level "Logfire" folder. Lone pages land in + // Other Documentation where they sit alongside other low-volume material. const realCategories = [] const misfitPages = [] for (const cluster of rawClusters) { @@ -2153,11 +2218,7 @@ function clusterByUrlPath(pages) { realCategories.push(cluster) continue } - const onlyPage = cluster.pages[0] - let pageSegs = [] - try { pageSegs = new URL(onlyPage.url).pathname.split('/').filter(Boolean) } catch {} - if (pageSegs.length > keyIdx + 1) realCategories.push(cluster) - else misfitPages.push(onlyPage) + misfitPages.push(cluster.pages[0]) } // If clustering produced no real categories, it added no value — tell the @@ -2490,6 +2551,143 @@ async function runJsonQuery({ systemPrompt, userPrompt, model }) { * * Returns deduped URLs in probe order. */ +// URL patterns we strip from llms.txt items — they show up alongside real +// pages but shouldn't be staged as docs: +// - openapi.json, package.json, .yaml specs, etc. (machine-readable specs) +// - llms.txt, llms-full.txt, llms-ctx.txt, … (other llms.txt variants +// that point back at the same index/dump, not real pages) +const ASSET_EXT_RE = /\.(json|ya?ml|xml|toml)$/i +const LLMS_TXT_RE = /(?:^|\/)llms[^/]*\.txt$/i +function isAssetOrMetaUrl(url) { + try { + const pn = new URL(url).pathname + return ASSET_EXT_RE.test(pn) || LLMS_TXT_RE.test(pn) + } catch { + return false + } +} + +// Immediate-child segments of root that signal a docs subtree on a +// non-docs hostname. We deliberately keep the set tiny — over-matching +// here (e.g. including `api`, `learn`) would over-filter sites whose +// "docs" subtree lives at the apex. +const DOCS_LIKE_ROOT_SEGMENTS = new Set(['docs', 'doc', 'documentation', 'document']) + +/** + * Narrow a merged llms.txt result to a docs subtree when a root-level + * llms.txt pulled non-doc URLs (marketing, pricing, blog, etc.) into the + * union. + * + * Sites that publish a root llms.txt alongside per-product subtree files + * often mix marketing/pricing/blog links into the root file. The BFS merge + * happily unions all of it, and without this pass we'd stage stubs for + * those pages too. + * + * elevenlabs.io is the canonical example: its root `elevenlabs.io/llms.txt` + * lists marketing pages alongside docs, and per-product files live at + * `elevenlabs.io/docs//llms.txt`. After the BFS we have docs URLs + * + pricing/blog URLs in the same merged result. With this pass we keep + * only URLs under `/docs/`. + * + * Conditions (all must hold), chosen so we only narrow when the risk is + * real and there's a clear scope to narrow to: + * - Hostname doesn't itself contain "docs" — `docs.example.com` is + * already a docs-only host; nothing to narrow. + * - A root-level llms.txt was actually probed and hit (valid OR + * invalid). Either way it's what dragged in the wider URL space via + * the BFS expansion, so it's the trigger we care about. + * - Multiple llms.txt files contributed to the merge. A single-file + * merge is by definition scoped to that one file's intent. + * - The merged URL set has a docs-shaped first-segment child of root + * (`/docs/`, `/documentation/`, …). That child is our scope target. + * + * Mutates `llms.parsed.sections` in place. Returns `{ segment, kept, + * dropped }` when narrowing happened, or `null` when conditions weren't + * met. + */ +function narrowToDocsSubtreeIfNeeded(llms, sourceUrl, hits) { + if (!llms || llms.sourceFiles.length < 2) return null + if (sourceUrl.hostname.toLowerCase().includes('docs')) return null + const rootHit = hits.some((h) => h.path === '/' || h.path === '') + if (!rootHit) return null + + // Identify docs-shaped first segments of root that actually appear in + // the merged URLs. Walking just the first path segment matches the + // user's spec ("we only look at immediate children in this case"). + const firstSegs = new Set() + for (const section of llms.parsed.sections) { + for (const item of section.items) { + try { + const seg = new URL(item.url).pathname.split('/').filter(Boolean)[0] + if (seg) firstSegs.add(seg.toLowerCase()) + } catch { + // ignore unparseable URLs + } + } + } + let docsSegment = null + for (const seg of firstSegs) { + if (DOCS_LIKE_ROOT_SEGMENTS.has(seg)) { + docsSegment = seg + break + } + } + if (!docsSegment) return null + + const prefix = '/' + docsSegment + let kept = 0 + let dropped = 0 + const keptSections = [] + for (const section of llms.parsed.sections) { + const items = [] + for (const item of section.items) { + try { + const pn = new URL(item.url).pathname.toLowerCase() + if (pn === prefix || pn.startsWith(prefix + '/')) { + items.push(item) + kept++ + } else { + dropped++ + } + } catch { + dropped++ + } + } + if (items.length > 0) { + section.items = items + keptSections.push(section) + } + } + llms.parsed.sections = keptSections + return { segment: docsSegment, kept, dropped } +} + +/** + * Drop asset/meta items in place from a parsed llms.txt structure. Sections + * that end up empty after the drop are removed too. Returns the count of + * items dropped. + */ +function dropAssetItemsFromParsed(parsed) { + let dropped = 0 + const keptSections = [] + for (const section of parsed.sections) { + const keptItems = [] + for (const item of section.items) { + if (isAssetOrMetaUrl(item.url)) { + dropped++ + continue + } + keptItems.push(item) + } + if (keptItems.length > 0) { + section.items = keptItems + keptSections.push(section) + } + } + parsed.sections = keptSections + return dropped +} + function buildLlmsCandidates(sourceUrl) { const out = [] const seen = new Set() @@ -2509,6 +2707,182 @@ function buildLlmsCandidates(sourceUrl) { return out } +// Hard cap on total probes during BFS discovery — protects against +// pathological sites where every URL returns a "200 + a few link rows" +// response and the frontier would otherwise blow up. +const LLMS_PROBE_CAP = 30 +// Per-node cap on child segments to expand. Root-level llms.txt files can +// list hundreds of URLs; without this a single root hit could schedule +// dozens of unrelated probes (`/blog`, `/pricing`, …). +const LLMS_CHILDREN_PER_NODE = 12 + +/** + * Convert a path like `/docs/quickstart` into its llms.txt URL. + */ +function pathToLlmsUrl(origin, path) { + if (!path || path === '/') return `${origin}/llms.txt` + return `${origin}${path}/llms.txt` +} + +/** + * Given a current path + a parsed llms.txt, return the next paths to probe: + * the parent directory (UP) and distinct first-segment-children extracted + * from the URL list (DOWN). Children are scoped to URLs whose pathname + * actually starts with the current path's prefix — out-of-scope URLs (e.g. + * a `/docs/llms.txt` listing `/blog/foo`) are ignored. + */ +function expandLlmsFrontier(currentPath, parsed) { + const next = [] + const prefix = currentPath === '/' || currentPath === '' ? '/' : currentPath + '/' + + if (currentPath && currentPath !== '/') { + const parent = currentPath.replace(/\/[^/]+$/, '') || '/' + next.push(parent) + } + + const childSegs = new Set() + for (const section of parsed.sections) { + for (const item of section.items) { + try { + const u = new URL(item.url) + const pn = u.pathname.toLowerCase() + if (!pn.startsWith(prefix.toLowerCase())) continue + const rest = pn.slice(prefix.length).split('/').filter(Boolean) + if (rest.length > 0) childSegs.add(rest[0]) + } catch { + // ignore unparseable URLs + } + if (childSegs.size >= LLMS_CHILDREN_PER_NODE) break + } + if (childSegs.size >= LLMS_CHILDREN_PER_NODE) break + } + for (const seg of childSegs) { + next.push(currentPath === '/' || !currentPath ? `/${seg}` : `${currentPath}/${seg}`) + } + return next +} + +/** + * BFS-discover all llms.txt files reachable from sourceUrl by walking UP + * the path and DOWN into child segments. Invalid hits (file exists but + * fails the usability ratio) are kept as "stepping stones" — they still + * expand the frontier so we can find their valid siblings/children — but + * don't contribute content to the merged result. Returns: + * { hits, misses, skipped } + * where `hits` includes both valid (usable: true) and invalid hits. + */ +async function discoverLlmsTxt(sourceUrl) { + const tried = new Set() + const hits = [] + const misses = [] + const skipped = [] + + const startSegs = sourceUrl.pathname.split('/').filter(Boolean) + const initial = [] + for (let i = startSegs.length; i >= 0; i--) { + initial.push('/' + startSegs.slice(0, i).join('/')) + } + + let frontier = initial + while (frontier.length > 0 && tried.size < LLMS_PROBE_CAP) { + const ring = [] + for (const p of frontier) { + const norm = p === '' ? '/' : p + if (tried.has(norm)) continue + tried.add(norm) + ring.push(norm) + if (tried.size >= LLMS_PROBE_CAP) break + } + if (ring.length === 0) break + + const results = await Promise.all( + ring.map(async (path) => { + const llmsUrl = pathToLlmsUrl(sourceUrl.origin, path) + const res = await fetchLlmsTxt(llmsUrl) + return { path, llmsUrl, res } + }), + ) + + const next = [] + for (const { path, llmsUrl, res } of results) { + if (!res.ok) { + misses.push({ url: llmsUrl, status: res.status, error: res.error }) + continue + } + // Treat as a "hit" only if there's at least one link row — that's the + // minimum signal that this is an llms.txt-shaped file (and not just a + // 200 response serving the site's HTML or a stray text file). Hits + // expand the frontier regardless of whether they pass usability. + const linkItems = res.stats?.linkItems ?? 0 + if (linkItems === 0) { + misses.push({ url: llmsUrl, status: 'no link items' }) + continue + } + hits.push({ ...res, llmsUrl, path }) + if (!res.usable) skipped.push({ url: llmsUrl, reason: res.reason }) + for (const np of expandLlmsFrontier(path, res.parsed)) { + if (!tried.has(np)) next.push(np) + } + } + frontier = next + } + + return { hits, misses, skipped } +} + +/** + * Merge content from all VALID hits into one llms-result-shaped object that + * downstream code can consume the same as a single-file result. Pages are + * deduped by normalized pathname, with the deepest-path hit winning — the + * `/docs/conversational-ai/` file's section assignment beats `/docs/`'s for + * any URL under that subtree. Returns null when there are no valid hits. + */ +function mergeValidHits(hits) { + const valid = hits.filter((h) => h.usable) + if (valid.length === 0) return null + + const deepestFirst = [...valid].sort((a, b) => b.path.length - a.path.length) + const shallowestFirst = [...valid].sort((a, b) => a.path.length - b.path.length) + + const claimed = new Set() + const sections = [] + for (const hit of deepestFirst) { + for (const section of hit.parsed.sections) { + const items = [] + for (const item of section.items) { + const key = normalizePath(item.url) + if (claimed.has(key)) continue + claimed.add(key) + items.push(item) + } + if (items.length > 0) sections.push({ title: section.title, items }) + } + } + + // Prefer the root-most file's H1 title — usually the canonical site name. + const title = shallowestFirst.find((h) => h.parsed.title)?.parsed.title || null + + const stats = valid.reduce( + (acc, h) => ({ + conforming: acc.conforming + (h.stats?.conforming || 0), + nonConforming: acc.nonConforming + (h.stats?.nonConforming || 0), + total: acc.total + (h.stats?.total || 0), + linkItems: acc.linkItems + (h.stats?.linkItems || 0), + }), + { conforming: 0, nonConforming: 0, total: 0, linkItems: 0 }, + ) + stats.ratio = stats.total === 0 ? 0 : stats.conforming / stats.total + + return { + parsed: { title, sections }, + usable: true, + reason: null, + stats, + llmsUrl: shallowestFirst[0].llmsUrl, + sourceFiles: shallowestFirst.map((h) => h.llmsUrl), + } +} + /** * Best-effort fetch of a site's /llms.txt plus a simple structural usability check. */ diff --git a/src/utils/llms.js b/src/utils/llms.js index 008c794..4602ea0 100644 --- a/src/utils/llms.js +++ b/src/utils/llms.js @@ -1,17 +1,24 @@ const H1_RE = /^#\s+(.+)$/ const H2_RE = /^##\s+(.+)$/ -const STANDARD_LIST_LINK_RE = /^\s*[-*+]\s+\[([^\]]+)\]\(([^)\s]+)\)(?:\s*[:—–-]\s*(.+?))?\s*$/ +// Any line whose meaningful content is a markdown link. Accepts: +// - Standard list rows: `- [text](url)`, `* [text](url) — desc` +// - Bare link lines: `[text](url)` +// - Breadcrumb-prefixed rows used by Fern/Mintlify-style indices: +// `ElevenAgents [Agent WebSockets](url)` +// `API Reference > Agents > Branches [List branches](url)` +// Captures: [prefix, text, url, description] (prefix and description may be empty). +const LINK_LINE_RE = /^\s*(?:[-*+]\s+)?([^\[\n]*?)\s*\[([^\]]+)\]\(([^)\s]+)\)(?:\s*[:—–-]\s*(.+?))?\s*$/ const BLOCKQUOTE_RE = /^\s*>/ const FENCE_RE = /^\s*(?:```|~~~)/ -// A llms.txt is considered usable when at least MIN_LINK_ROWS link items -// parse cleanly AND the share of "spec-shaped" lines (H1/H2/blockquote/link -// row/blank) over total lines is at least MIN_CONFORMING_RATIO. Fenced code -// blocks (including the fence delimiters) and anything else (prose -// paragraphs, image markdown, HTML, etc.) count against the ratio. Tuned to -// accept real-world files that include a short prose preamble/epilogue while -// still rejecting prose-heavy or llms-full.txt-shaped documents. -const MIN_LINK_ROWS = 10 +// A llms.txt is considered usable as long as it has at least one link row. +// Beyond that we enforce a structural ratio (share of "spec-shaped" lines — +// H1/H2/blockquote/link row/blank — over total lines), but only once the +// file has enough links for the ratio to be statistically meaningful. A +// 3-link file with one prose sentence would compute a misleading ratio; we +// can't tell signal from noise at that size, so we trust it and let the +// BFS-merge dedupe handle any junk that slips through. +const RATIO_CHECK_MIN_LINKS = 10 const MIN_CONFORMING_RATIO = 0.7 export function analyzeLlmsTxt(body, llmsUrl) { @@ -63,10 +70,8 @@ export function parseLlmsTxt(body, llmsUrl) { function getSkipReason(body, linkItems, lineStats) { if (/^---\r?\n/.test(body)) return 'starts with YAML frontmatter' - if (linkItems < MIN_LINK_ROWS) { - return `only ${linkItems} link item${linkItems === 1 ? '' : 's'} (need at least ${MIN_LINK_ROWS})` - } - if (lineStats.ratio < MIN_CONFORMING_RATIO) { + if (linkItems === 0) return 'no link items' + if (linkItems >= RATIO_CHECK_MIN_LINKS && lineStats.ratio < MIN_CONFORMING_RATIO) { return `conforming-line ratio ${lineStats.ratio.toFixed(2)} below ${MIN_CONFORMING_RATIO} (${lineStats.conforming}/${lineStats.total} lines)` } return null @@ -99,7 +104,7 @@ function classifyLines(body) { conforming++ continue } - if (H1_RE.test(line) || H2_RE.test(line) || STANDARD_LIST_LINK_RE.test(line) || BLOCKQUOTE_RE.test(line)) { + if (H1_RE.test(line) || H2_RE.test(line) || LINK_LINE_RE.test(line) || BLOCKQUOTE_RE.test(line)) { conforming++ continue } @@ -112,16 +117,22 @@ function classifyLines(body) { } function parseListLink(line, llmsUrl) { - const match = line.match(STANDARD_LIST_LINK_RE) + const match = line.match(LINK_LINE_RE) if (!match) return null - const url = normalizeUrl(match[2], llmsUrl) + const [, prefix, text, rawUrl, trailingDesc] = match + const url = normalizeUrl(rawUrl, llmsUrl) if (!url) return null + // Prefer an explicit trailing description (`[text](url) — desc`); fall back + // to the breadcrumb prefix (`API Reference > Agents [text](url)`) when no + // explicit description is present. Either gives downstream extra context. + const description = trailingDesc?.trim() || prefix?.trim() || null + return { - text: match[1].trim(), + text: text.trim(), url, - description: match[3] ? match[3].trim() : null, + description: description || null, } } From 41a4fdb9cbc08eabc63bd11487f28290a7f451b7 Mon Sep 17 00:00:00 2001 From: Jadenzzz <94533693+Jadenzzz@users.noreply.github.com> Date: Wed, 20 May 2026 16:23:05 +1000 Subject: [PATCH 09/43] fix(llms-validator): allow space between description and link in standard format --- src/utils/llms.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/llms.js b/src/utils/llms.js index 4602ea0..ca60531 100644 --- a/src/utils/llms.js +++ b/src/utils/llms.js @@ -7,7 +7,7 @@ const H2_RE = /^##\s+(.+)$/ // `ElevenAgents [Agent WebSockets](url)` // `API Reference > Agents > Branches [List branches](url)` // Captures: [prefix, text, url, description] (prefix and description may be empty). -const LINK_LINE_RE = /^\s*(?:[-*+]\s+)?([^\[\n]*?)\s*\[([^\]]+)\]\(([^)\s]+)\)(?:\s*[:—–-]\s*(.+?))?\s*$/ +const LINK_LINE_RE = /^\s*(?:[-*+]\s+)?([^\[\n]*?)\s*\[([^\]]+)\] ?\(([^)\s]+)\)(?:\s*[:—–-]\s*(.+?))?\s*$/ const BLOCKQUOTE_RE = /^\s*>/ const FENCE_RE = /^\s*(?:```|~~~)/ From f753dfd707e4fe473adaf9a80eda8ed68c42548d Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Wed, 20 May 2026 18:49:04 +1000 Subject: [PATCH 10/43] feat: import can handle multiple web urls --- package.json | 2 +- src/commands/import.js | 237 +++++++++++++++++++++++++++-------------- 2 files changed, 159 insertions(+), 80 deletions(-) diff --git a/package.json b/package.json index 53a8c7c..8501e9d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@readme/cli", - "version": "0.0.27", + "version": "0.0.28", "description": "The ReadMe CLI", "type": "module", "bin": { diff --git a/src/commands/import.js b/src/commands/import.js index ffc3ad8..46c1788 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -19,7 +19,10 @@ export const hidden = true export const skipBootstrap = true export function args(cmd) { - cmd.requiredOption('--source ', 'URL to import from, or path to a local OpenAPI spec (.json/.yaml/.yml)') + cmd.requiredOption( + '--source ', + 'One or more URLs to import from (space-separated or repeated), or a single local OpenAPI spec (.json/.yaml/.yml)', + ) cmd.option('-o, --output ', 'Output zip path (defaults to -readme.zip in cwd)') cmd.addOption(new Option('-m, --model ', 'Claude model alias: haiku, sonnet, opus').choices(['haiku', 'sonnet', 'opus']).default('sonnet')) cmd.option('--firecrawl-key ', 'Firecrawl API key (or set FIRECRAWL_API_KEY env var) — enables JS-rendered sidebar scraping') @@ -39,7 +42,8 @@ export function args(cmd) { * on success. * * @param {object} options - * @param {string} options.source URL to import from, or path to a local OAS spec. + * @param {string | string[]} options.source One or more URLs to import from, or a single path to a local OAS spec. + * Multiple URLs are merged into one zip; same-titled categories merge their pages. * @param {string} [options.output] Output zip path. Defaults to `-readme.zip` in cwd. * @param {string} [options.model] Claude model alias: 'haiku' | 'sonnet' | 'opus'. Defaults to 'sonnet'. * @param {string} [options.firecrawlKey] Firecrawl API key (falls back to FIRECRAWL_API_KEY env var). @@ -60,25 +64,130 @@ export async function importDocs(options) { const debugSnapshots = options.debug ? {} : null - // Dispatch: http(s) URL → docs-site scrape flow; anything else → local OAS. - if (!/^https?:\/\//i.test(options.source)) { - return runOasImport(options.source, options, startedAt, phases, timePhase) - } + // Normalize source → array; preserve back-compat with single-string callers. + const rawSources = Array.isArray(options.source) ? options.source : [options.source] + if (rawSources.length === 0) throw new Error('At least one --source is required') - let sourceUrl - try { - sourceUrl = new URL(options.source) - } catch { - throw new Error(`Invalid --source URL: ${options.source}`) + // Dispatch: any non-URL entry triggers the local-OAS path, which is + // strictly single-source. + const hasNonUrl = rawSources.some((s) => !/^https?:\/\//i.test(s)) + if (hasNonUrl) { + if (rawSources.length > 1) { + throw new Error('OAS imports accept a single --source; URL imports can accept multiple.') + } + return runOasImport(rawSources[0], options, startedAt, phases, timePhase) } - const outputZip = path.resolve(options.output || path.join(process.cwd(), `${sourceUrl.hostname}-readme.zip`)) + const sourceUrls = rawSources.map((s) => { + try { + return new URL(s) + } catch { + throw new Error(`Invalid --source URL: ${s}`) + } + }) + + const hostnameJoined = sourceUrls.map((u) => u.hostname).join('-') + const outputZip = path.resolve(options.output || path.join(process.cwd(), `${hostnameJoined}-readme.zip`)) console.log() - styles.info(`Importing from ${styles.bold(sourceUrl.toString())}`) + if (sourceUrls.length === 1) { + styles.info(`Importing from ${styles.bold(sourceUrls[0].toString())}`) + } else { + styles.info(`Importing from ${styles.bold(String(sourceUrls.length))} sources:`) + for (const u of sourceUrls) styles.info(styles.dim(` · ${u.toString()}`)) + } if (!options.test) styles.info(`Output: ${styles.bold(outputZip)}`) console.log() + // Per-source pipeline: discovery + scrape + organize, in parallel. + // Each call returns an `organized` object with the standard shape + // `{ title, categories: [...] }`; we merge them before staging. + const perSourceOrganized = await Promise.all( + sourceUrls.map((sourceUrl) => produceOrganizedForSource(sourceUrl, options, timePhase, debugSnapshots)), + ) + + const organized = sourceUrls.length === 1 ? perSourceOrganized[0] : mergeOrganized(perSourceOrganized) + + if (debugSnapshots) { + debugSnapshots['05-organized.json'] = organized + const debugDir = path.join(os.tmpdir(), `readme-import-debug-${hostnameJoined}-${Date.now()}`) + fs.mkdirSync(debugDir, { recursive: true }) + for (const [name, data] of Object.entries(debugSnapshots)) { + fs.writeFileSync(path.join(debugDir, name), JSON.stringify(data, null, 2)) + } + styles.info(`${styles.dim(`Debug snapshots → ${debugDir}`)}`) + } + console.log() + + console.log(` ${styles.bold(organized.title || '(untitled)')}`) + for (const cat of organized.categories || []) { + console.log() + const iconLabel = cat.icon ? `${styles.brand(`[${cat.icon}]`)} ` : '' + console.log(` ${iconLabel}${styles.bold(cat.title)}`) + printPagesTree(cat.pages || [], 2) + } + + const stagingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'readme-import-')) + + let result + try { + styles.info(`Staging frontmatter stubs in ${styles.bold(stagingDir)}...`) + const stageStart = Date.now() + const staged = await timePhase('stage stubs', async () => + stageOrganized(organized, stagingDir, { skipApiReference: !!options.skipApiReference }), + ) + const landingTitle = + organized.title || + (sourceUrls.length === 1 ? sourceUrls[0].hostname : sourceUrls.map((u) => u.hostname).join(' + ')) + ensureDocsLandingPage(stagingDir, landingTitle) + styles.ok( + `Staged ${styles.bold(String(staged.fileCount))} stub${staged.fileCount === 1 ? '' : 's'} across ${styles.bold(String(staged.dirCount))} director${staged.dirCount === 1 ? 'y' : 'ies'} in ${styles.bold(formatDuration(Date.now() - stageStart))}.`, + ) + if (staged.skippedApiRef > 0) { + styles.info(`Skipped ${styles.bold(String(staged.skippedApiRef))} API reference page${staged.skippedApiRef === 1 ? '' : 's'} (--skip-api-reference)`) + } + console.log() + + if (options.test) { + styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Staged ${styles.bold(String(staged.fileCount))} files at ${styles.bold(stagingDir)}`) + console.log() + styles.info('Starting the dev server for preview...') + console.log() + await runDevPreview(stagingDir) + return { source: 'url', stagingDir, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases } + } + + if (staged.fileCount === 0) { + styles.warning('Staging directory is empty — skipping zip.') + return { source: 'url', fileCount: 0, duration: Date.now() - startedAt, phases } + } + + styles.info(`Packaging ${styles.bold(String(staged.fileCount))} files into ${styles.bold(outputZip)}...`) + await timePhase('zip', () => createZip(stagingDir, outputZip)) + + console.log() + styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Your ReadMe import is ready at ${styles.bold(outputZip)}`) + console.log(styles.dim(` ⏱ ${phases.map((p) => `${p.label} ${formatDuration(p.ms)}`).join(' · ')}`)) + result = { source: 'url', outputZip, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases } + } finally { + if (!options.test) { + fs.rmSync(stagingDir, { recursive: true, force: true }) + } + } + + return result +} + +/** + * Run the discovery + scrape + organize pipeline for ONE source URL and + * return its `organized` object (`{ title, categories: [...] }`). Pure + * per-source work — no staging, no zipping. `importDocs` runs this in + * parallel across multiple sources and merges the results. + * + * `debugSnapshots`, when provided, gets per-source keys suffixed with + * the hostname so parallel runs don't clobber each other. + */ +async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSnapshots) { // Discover llms.txt files by BFS walking parents AND children. Seed the // frontier with the walk-up paths (source → root). Any "hit" (file exists // with at least one link row) expands the frontier: parent path + distinct @@ -182,9 +291,10 @@ export async function importDocs(options) { } } + const dbgSuffix = `-${sourceUrl.hostname}` if (debugSnapshots) { - debugSnapshots['01-llms-parsed.json'] = { llmsUrl, parsed: llms ? llms.parsed : null, skipped: skippedLlms } - debugSnapshots['01b-sitemap.json'] = { sitemapUrl, urls: sitemapKnownUrls } + debugSnapshots[`01-llms-parsed${dbgSuffix}.json`] = { llmsUrl, parsed: llms ? llms.parsed : null, skipped: skippedLlms } + debugSnapshots[`01b-sitemap${dbgSuffix}.json`] = { sitemapUrl, urls: sitemapKnownUrls } } let knownUrls = [] @@ -229,7 +339,7 @@ export async function importDocs(options) { const mintlifyStart = Date.now() const mintlifyNav = await timePhase('mintlify probe', () => tryMintlifyNav(sourceUrl.toString(), knownUrls, firecrawlKey)) if (debugSnapshots) { - debugSnapshots['02a-mintlify-nav.json'] = mintlifyNav ? JSON.parse(JSON.stringify(mintlifyNav)) : null + debugSnapshots[`02a-mintlify-nav${dbgSuffix}.json`] = mintlifyNav ? JSON.parse(JSON.stringify(mintlifyNav)) : null } if (mintlifyNav) { const pageCount = mintlifyNav.categories.reduce((n, c) => n + c.pages.length, 0) @@ -249,7 +359,7 @@ export async function importDocs(options) { scraped = await timePhase('scrape nav', () => scrapeNavFromSite(sourceUrl.toString(), knownUrls, firecrawlKey)) } if (debugSnapshots) { - debugSnapshots['02-scraped-raw.json'] = scraped ? JSON.parse(JSON.stringify(scraped)) : null + debugSnapshots[`02-scraped-raw${dbgSuffix}.json`] = scraped ? JSON.parse(JSON.stringify(scraped)) : null } // If the sidebar scrape covers less than 75% of the llms.txt URLs, the // scrape is too thin to trust as the import's spine. Common on multi-tab @@ -276,7 +386,7 @@ export async function importDocs(options) { if (knownUrls.length > 0) { const slotted = slotOrphansByPath(scraped, knownUrls) if (debugSnapshots) { - debugSnapshots['03-after-slot-by-path.json'] = { + debugSnapshots[`03-after-slot-by-path${dbgSuffix}.json`] = { scraped: JSON.parse(JSON.stringify(scraped)), unslottedOrphans: slotted, } @@ -324,7 +434,7 @@ export async function importDocs(options) { `Scrape covered only ${styles.bold(String(directMatches))}/${styles.bold(String(knownUrls.length))} pages — discarded as too thin and re-clustered ${styles.bold(String(scrapeAllPages.length + slotted.length))} pages by URL path into ${styles.bold(String(reclustered.length))} categor${reclustered.length === 1 ? 'y' : 'ies'}: ${reclustered.map((c) => styles.bold(c.title)).join(', ')}.`, ) if (debugSnapshots) { - debugSnapshots['04-after-orphan-buckets.json'] = { + debugSnapshots[`04-after-orphan-buckets${dbgSuffix}.json`] = { mode: 'reclustered-by-url-path', scraped: JSON.parse(JSON.stringify(scraped)), } @@ -343,7 +453,7 @@ export async function importDocs(options) { const buckets = bucketOrphansByPathType(otherOrphans, scraped) for (const b of buckets) scraped.categories.push(b) if (debugSnapshots) { - debugSnapshots['04-after-orphan-buckets.json'] = { + debugSnapshots[`04-after-orphan-buckets${dbgSuffix}.json`] = { apiReferenceCollected: apiResult.category ? { pageCount: apiResult.category.pages.length, @@ -475,69 +585,38 @@ export async function importDocs(options) { } styles.ok(`Organized in ${styles.bold(formatDuration(Date.now() - organizeStart))}.`) - if (debugSnapshots) { - debugSnapshots['05-organized.json'] = organized - const debugDir = path.join(os.tmpdir(), `readme-import-debug-${sourceUrl.hostname}-${Date.now()}`) - fs.mkdirSync(debugDir, { recursive: true }) - for (const [name, data] of Object.entries(debugSnapshots)) { - fs.writeFileSync(path.join(debugDir, name), JSON.stringify(data, null, 2)) - } - styles.info(`${styles.dim(`Debug snapshots → ${debugDir}`)}`) - } - console.log() - - console.log(` ${styles.bold(organized.title || '(untitled)')}`) - for (const cat of organized.categories || []) { - console.log() - const iconLabel = cat.icon ? `${styles.brand(`[${cat.icon}]`)} ` : '' - console.log(` ${iconLabel}${styles.bold(cat.title)}`) - printPagesTree(cat.pages || [], 2) - } - - const stagingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'readme-import-')) - - let result - try { - styles.info(`Staging frontmatter stubs in ${styles.bold(stagingDir)}...`) - const stageStart = Date.now() - const staged = await timePhase('stage stubs', async () => stageOrganized(organized, stagingDir, { skipApiReference: !!options.skipApiReference })) - ensureDocsLandingPage(stagingDir, organized.title || sourceUrl.hostname) - styles.ok( - `Staged ${styles.bold(String(staged.fileCount))} stub${staged.fileCount === 1 ? '' : 's'} across ${styles.bold(String(staged.dirCount))} director${staged.dirCount === 1 ? 'y' : 'ies'} in ${styles.bold(formatDuration(Date.now() - stageStart))}.`, - ) - if (staged.skippedApiRef > 0) { - styles.info(`Skipped ${styles.bold(String(staged.skippedApiRef))} API reference page${staged.skippedApiRef === 1 ? '' : 's'} (--skip-api-reference)`) - } - console.log() - - if (options.test) { - styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Staged ${styles.bold(String(staged.fileCount))} files at ${styles.bold(stagingDir)}`) - console.log() - styles.info('Starting the dev server for preview...') - console.log() - await runDevPreview(stagingDir) - return { source: 'url', stagingDir, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases } - } - - if (staged.fileCount === 0) { - styles.warning('Staging directory is empty — skipping zip.') - return { source: 'url', fileCount: 0, duration: Date.now() - startedAt, phases } - } - - styles.info(`Packaging ${styles.bold(String(staged.fileCount))} files into ${styles.bold(outputZip)}...`) - await timePhase('zip', () => createZip(stagingDir, outputZip)) + return organized +} - console.log() - styles.ok(`Done in ${styles.bold(formatDuration(Date.now() - startedAt))}! Your ReadMe import is ready at ${styles.bold(outputZip)}`) - console.log(styles.dim(` ⏱ ${phases.map((p) => `${p.label} ${formatDuration(p.ms)}`).join(' · ')}`)) - result = { source: 'url', outputZip, fileCount: staged.fileCount, duration: Date.now() - startedAt, phases } - } finally { - if (!options.test) { - fs.rmSync(stagingDir, { recursive: true, force: true }) +/** + * Merge per-source `organized` results into one. Same-titled categories MERGE + * their page lists (intentional — when a user bundles related sources into + * one project, they probably want a coherent sidebar; `ensureUniqueSlugs` later + * dedupes any slug collisions globally). First non-null title wins; first + * non-null icon per merged category wins. Page order within a merged category + * is the concatenation of input sources in argument order. + */ +function mergeOrganized(perSource) { + const title = perSource.map((o) => o.title).find(Boolean) || null + + const byTitle = new Map() + const order = [] + for (const o of perSource) { + for (const cat of o.categories || []) { + const key = cat.title + const existing = byTitle.get(key) + if (existing) { + existing.pages.push(...(cat.pages || [])) + if (!existing.icon && cat.icon) existing.icon = cat.icon + } else { + const copy = { ...cat, pages: [...(cat.pages || [])] } + byTitle.set(key, copy) + order.push(copy) + } } } - return result + return { title, categories: order } } /** From 71920f6b7ae3f20d4eefdef027f886ef95233b01 Mon Sep 17 00:00:00 2001 From: Jadenzzz <94533693+Jadenzzz@users.noreply.github.com> Date: Thu, 21 May 2026 00:38:20 +1000 Subject: [PATCH 11/43] feat: Add Archbee docs tree import support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds an Archbee-specific docs navigation path to `import`, using Archbee’s embedded Next.js `_docSpace.publicDocsTree` when available. This preserves the site-authored document order and nesting instead of relying on Archbee `llms.txt` output, which can be flat or shuffled. ## Changes - Probe for Archbee document trees after Mintlify detection and before generic sidebar scraping. - Parse `__NEXT_DATA__` and normalize `_docSpace.publicDocsTree` into the existing `{ title, categories }` shape. - Preserve metadata from known `llms.txt` URLs where paths match Archbee tree nodes. --- src/commands/import.js | 121 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 116 insertions(+), 5 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 46c1788..6407307 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -342,17 +342,36 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna debugSnapshots[`02a-mintlify-nav${dbgSuffix}.json`] = mintlifyNav ? JSON.parse(JSON.stringify(mintlifyNav)) : null } if (mintlifyNav) { - const pageCount = mintlifyNav.categories.reduce((n, c) => n + c.pages.length, 0) + const pageCount = mintlifyNav.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0) styles.ok( `Found Mintlify config at ${styles.bold(mintlifyNav.source)} in ${styles.bold(formatDuration(Date.now() - mintlifyStart))} — ${styles.bold(String(mintlifyNav.categories.length))} categor${mintlifyNav.categories.length === 1 ? 'y' : 'ies'}, ${styles.bold(String(pageCount))} pages.`, ) } console.log() + // Archbee sites embed the canonical document tree in Next.js page data. + // Their llms.txt export can be a flat, shuffled list, so prefer the tree + // when present. + let archbeeNav = null + if (!mintlifyNav) { + styles.info(`Probing for Archbee document tree...`) + const archbeeStart = Date.now() + archbeeNav = await timePhase('archbee probe', () => tryArchbeeNav(sourceUrl.toString(), knownUrls, firecrawlKey)) + if (archbeeNav) { + const pageCount = archbeeNav.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0) + styles.ok( + `Found Archbee document tree in ${styles.bold(formatDuration(Date.now() - archbeeStart))} — ${styles.bold(String(pageCount))} page${pageCount === 1 ? '' : 's'}.`, + ) + } + console.log() + } + let scraped let scrapeStart = Date.now() if (mintlifyNav) { scraped = { title: mintlifyNav.title, categories: mintlifyNav.categories } + } else if (archbeeNav) { + scraped = { title: archbeeNav.title, categories: archbeeNav.categories } } else { styles.info(`Scraping sidebar nav from ${styles.bold(sourceUrl.toString())}${firecrawlKey ? ' ' + styles.dim('(via Firecrawl)') : ''}...`) scrapeStart = Date.now() @@ -371,7 +390,7 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna // scrape and fall through to the llms.txt path, which uses URL-based // clustering when multiple files were merged. if (scraped && llms && knownUrls.length > 0) { - const scrapedPages = scraped.categories.reduce((n, c) => n + c.pages.length, 0) + const scrapedPages = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0) const coverage = scrapedPages / knownUrls.length if (coverage < 0.75) { styles.info( @@ -382,7 +401,7 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna } if (scraped) { - const directMatches = scraped.categories.reduce((n, c) => n + c.pages.length, 0) + const directMatches = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0) if (knownUrls.length > 0) { const slotted = slotOrphansByPath(scraped, knownUrls) if (debugSnapshots) { @@ -391,7 +410,7 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna unslottedOrphans: slotted, } } - const totalMatched = scraped.categories.reduce((n, c) => n + c.pages.length, 0) + const totalMatched = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0) styles.ok( `Scraped nav in ${styles.bold(formatDuration(Date.now() - scrapeStart))} — ${styles.bold(String(scraped.categories.length))} categor${scraped.categories.length === 1 ? 'y' : 'ies'}, ${styles.bold(String(directMatches))} direct matches + ${styles.bold(String(totalMatched - directMatches))} slotted by path = ${styles.bold(String(totalMatched))}/${knownUrls.length}.`, ) @@ -418,7 +437,7 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna // accounts for less than a third of the known pages, so its category // labels aren't a trustworthy spine for the remainder. const orphansDwarfDirect = slotted.length >= directMatches * 2 - const scrapeAllPages = scraped.categories.flatMap((c) => c.pages) + const scrapeAllPages = scraped.categories.flatMap((c) => collectUrlPagesDeep(c.pages)) let reclustered = null if (orphansDwarfDirect) { reclustered = clusterByUrlPath([...scrapeAllPages, ...slotted]) @@ -1148,6 +1167,81 @@ function parseMintlifyConfig(config, origin, byPath) { return { title, categories } } +/** + * Archbee exposes its canonical docs tree in Next.js page data as + * `_docSpace.publicDocsTree`. That tree preserves author order and nesting, + * while Archbee's llms.txt export may be flat and shuffled. + * + * Returns { title, categories } or null if the page is not Archbee or the + * embedded tree is unavailable. + */ +async function tryArchbeeNav(sourceUrl, knownPages, firecrawlKey) { + const origin = new URL(sourceUrl).origin + const fetchHtml = firecrawlKey ? makeFirecrawlFetcher(firecrawlKey) : fetchHtmlDirect + const html = await fetchHtml(toBrowsableUrl(sourceUrl)) + if (!html || !/publicDocsTree|Archbee/i.test(html)) return null + + const nextData = extractNextData(html) + const docSpace = nextData?.props?.pageProps?._docSpace + const tree = docSpace?.publicDocsTree + if (!Array.isArray(tree) || tree.length === 0) return null + + const byPath = new Map() + for (const p of knownPages) byPath.set(normalizePath(p.url), p) + + const pageFromNode = (node) => { + if (!node || typeof node !== 'object') return null + if (node.isHidden === true) return null + + const title = String(node.name || node.title || node.urlKey || '').trim() + const children = Array.isArray(node.children) ? node.children.map(pageFromNode).filter(Boolean) : [] + if (!title && children.length === 0) return null + + const url = archbeeNodeUrl(origin, node.urlKey) + const known = url ? byPath.get(normalizePath(url)) : null + const page = { + title: known?.title || title || 'Untitled', + url, + ...(known?.description ? { description: known.description } : {}), + ...(children.length > 0 ? { pages: children } : {}), + } + + // Root docs pages have no URL path segments, but should stay in their + // authored position when the later URL-trie nesting pass runs. + if (url && extractUrlPathSegments(url).length === 0) { + page._virtualPathSegs = [kebabCase(page.title || 'home') || 'home'] + } + + return page + } + + const pages = tree.map(pageFromNode).filter(Boolean) + if (pages.length === 0) return null + + return { + title: docSpace.hostingTitle || docSpace.name || null, + categories: [{ title: 'Documentation', pages }], + } +} + +function extractNextData(html) { + const m = String(html).match(/]*\bid=["']__NEXT_DATA__["'])[^>]*>([\s\S]*?)<\/script>/i) + if (!m) return null + try { + return JSON.parse(m[1]) + } catch { + return null + } +} + +function archbeeNodeUrl(origin, urlKey) { + if (urlKey === undefined || urlKey === null) return null + const key = String(urlKey || '').trim() + if (!key || key === '/') return `${origin}/` + if (/^https?:\/\//i.test(key)) return key + return `${origin}/${key.replace(/^\/+/, '')}` +} + /** * Score a parsed nav tree for "sidebar-likeness". A real docs sidebar has * multiple section headers (hierarchy) and tens of links; secondary navs @@ -1840,6 +1934,23 @@ function collectFlat(page, out) { } } +function countUrlPagesDeep(pages) { + let n = 0 + for (const p of pages || []) { + if (p.url) n++ + if (p.pages && p.pages.length > 0) n += countUrlPagesDeep(p.pages) + } + return n +} + +function collectUrlPagesDeep(pages, out = []) { + for (const p of pages || []) { + if (p.url) out.push(p) + if (p.pages && p.pages.length > 0) collectUrlPagesDeep(p.pages, out) + } + return out +} + /** * Parse a URL's pathname into segments suitable for trie nesting. Strips file * extensions (`.md`/`.html`/etc.) and drops a trailing `index` segment so From 5c7b7fb934a9b96446e507ea53e57e8eb90ab6e8 Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Thu, 21 May 2026 20:03:41 +1000 Subject: [PATCH 12/43] chore: update to v0.28 --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index b419bbf..a994dac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@readme/cli", - "version": "0.0.27", + "version": "0.0.28", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@readme/cli", - "version": "0.0.27", + "version": "0.0.28", "license": "ISC", "dependencies": { "@anthropic-ai/claude-agent-sdk": "^0.2.114", From 2edc32e58bcf3ff9220b55fb99effb44410ae29e Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Thu, 21 May 2026 23:35:42 +1000 Subject: [PATCH 13/43] fix: get unique slugs via concatenating cat title --- src/commands/import.js | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 6407307..491538f 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -3486,8 +3486,12 @@ function extractUrlPathSegments(url) { * * Base slug is the last URL segment (kebab-cased). When two or more pages * share a base, we prepend the next-up URL segment to every member of the - * colliding group and recheck - * + * colliding group and recheck. The category title is treated as the + * outermost segment, so when URL segments alone aren't enough (e.g. a + * group-only `/v1` parent that appears in multiple categories), expansion + * can still reach for the category to disambiguate before falling back to + * a numeric `-N` suffix. + * * @example * ensureUniqueSlugs([ * { title: 'Getting Started', pages: [ @@ -3514,13 +3518,19 @@ function ensureUniqueSlugs(categories) { } return extractUrlPathSegments(p.url) } - const walk = (pages) => { + const walk = (pages, categorySeg) => { for (const p of pages || []) { - entries.push({ page: p, segments: segmentsFor(p), fallback: p.title, depth: 1 }) - if (p.pages) walk(p.pages) + const urlSegs = segmentsFor(p) + const segments = categorySeg ? [categorySeg, ...urlSegs] : urlSegs + entries.push({ page: p, segments, fallback: p.title, depth: 1 }) + if (p.pages) walk(p.pages, categorySeg) } } - for (const c of categories || []) walk(c.pages) + for (const c of categories || []) { + const rawTitle = (c?.title || '').trim() + const categorySeg = rawTitle ? kebabCase(rawTitle) : '' + walk(c.pages, categorySeg) + } const slugFor = (e) => { if (e.segments.length === 0) return kebabCase(e.fallback || 'page') || 'page' @@ -3563,7 +3573,8 @@ function ensureUniqueSlugs(categories) { slug = `${base}-${n}` styles.error( `Slug collision after segment expansion: ${base} — falling back to ${slug} for ${describe(e)}. ` + - `Pages were deduped by path before organize, so this indicates the organize step produced duplicates.`, + `Pages were deduped by path before organize and category title is already part of the slug, ` + + `so this indicates the organize step produced duplicates within a single category.`, ) } used.add(slug) From d55fd98d98f4f2d6e4dd40aeb9b3c3c7fb665f43 Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Fri, 22 May 2026 16:45:16 +1000 Subject: [PATCH 14/43] feat: hande putting changelogs separately with an arg --- src/commands/import.js | 168 +++++++++++++++++++++++++++++++++-------- 1 file changed, 136 insertions(+), 32 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 491538f..795a6a4 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -27,6 +27,10 @@ export function args(cmd) { cmd.addOption(new Option('-m, --model ', 'Claude model alias: haiku, sonnet, opus').choices(['haiku', 'sonnet', 'opus']).default('sonnet')) cmd.option('--firecrawl-key ', 'Firecrawl API key (or set FIRECRAWL_API_KEY env var) — enables JS-rendered sidebar scraping') cmd.option('--skip-api-reference', 'Drop pages routed to the API Reference / reference dir. Use when uploading the OAS spec separately.') + cmd.option( + '--separate-changelog', + 'Move changelog / release-notes pages into a top-level changelog/ directory instead of docs/Changelog/. Note: changelog/ is outside the git-format schema, so it will not lint or sync to ReadMe.', + ) // Internal dev-only flag: skip the zip, keep staging, and boot the dev server // against it for quick visual previews. Hidden from --help. cmd.addOption(new Option('--test').hideHelp()) @@ -48,6 +52,7 @@ export function args(cmd) { * @param {string} [options.model] Claude model alias: 'haiku' | 'sonnet' | 'opus'. Defaults to 'sonnet'. * @param {string} [options.firecrawlKey] Firecrawl API key (falls back to FIRECRAWL_API_KEY env var). * @param {boolean} [options.skipApiReference] Drop pages routed to the API Reference dir. + * @param {boolean} [options.separateChangelog] Move changelog pages into a top-level `changelog/` dir instead of `docs/Changelog/`. * @param {boolean} [options.test] Skip the zip, keep staging, and boot the dev server. * @param {boolean} [options.debug] Dump intermediate pipeline artifacts to a tmp dir. * @returns {Promise<{ source: 'url' | 'oas', outputZip?: string, stagingDir?: string, fileCount: number, duration: number, phases: Array<{ label: string, ms: number }> }>} @@ -136,6 +141,20 @@ export async function importDocs(options) { const staged = await timePhase('stage stubs', async () => stageOrganized(organized, stagingDir, { skipApiReference: !!options.skipApiReference }), ) + + // Changelog pages stage into docs/Changelog/ (a schema-valid docs + // category). With --separate-changelog, relocate them afterwards into a + // top-level changelog/ dir — done as a post-staging move so the staged + // tree is a valid git-format layout up to this point. + if (options.separateChangelog) { + const movedChangelog = relocateChangelogDir(stagingDir) + if (movedChangelog > 0) { + styles.info( + `Moved ${styles.bold(String(movedChangelog))} changelog page${movedChangelog === 1 ? '' : 's'} into ${styles.bold('changelog/')} (--separate-changelog).`, + ) + } + } + const landingTitle = organized.title || (sourceUrls.length === 1 ? sourceUrls[0].hostname : sourceUrls.map((u) => u.hostname).join(' + ')) @@ -424,6 +443,14 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna styles.info(`Moved ${styles.bold(String(moved))} page${moved === 1 ? '' : 's'} into ${styles.bold('API Reference')} based on URL path.`) } + // Same idea for changelog / release-notes pages — pull any whose URL + // carries a changelog-style segment into one "Changelog" category. It + // routes to docs/Changelog/ unless --separate-changelog relocates it. + const movedChangelog = reclassifyChangelogPages(scraped) + if (movedChangelog > 0) { + styles.info(`Moved ${styles.bold(String(movedChangelog))} page${movedChangelog === 1 ? '' : 's'} into ${styles.bold('Changelog')} based on URL path.`) + } + if (slotted.length > 0) { // When orphans dwarf direct matches, the sidebar scrape was too thin // to trust as the import's spine — keeping it would produce a small @@ -1754,9 +1781,9 @@ function bucketOrphansByPathType(orphans, scraped) { endpoints: 'API Reference', endpoint: 'API Reference', changelog: 'Changelog', - release: 'Release Notes', - releases: 'Release Notes', - 'release-notes': 'Release Notes', + release: 'Changelog', + releases: 'Changelog', + 'release-notes': 'Changelog', recipes: 'Recipes', recipe: 'Recipes', guides: 'Guides', @@ -1849,43 +1876,42 @@ function titleCase(s) { /** * Walk all scraped pages (including nested sub-pages) and move any whose URL - * contains a strong reference segment (`/api-reference/`, `/endpoints/`, etc.) - * into a single "API Reference" category. Some docs sites (e.g. greenflash.ai) - * spotlight a handful of endpoints under "Developers" in the sidebar while the - * bulk of endpoints live under a separate API Reference section — we favor - * the URL-path signal over the sidebar placement so all reference pages land - * together in `reference/` after staging. + * carries one of `segmentRe`'s path segments into a single category, + * flattening sub-pages as it goes. Shared backbone for the API Reference and + * Changelog sweeps: both favor the URL-path signal over sidebar placement, so + * e.g. `/api-reference/*` or `/changelog/*` pages land together in their own + * category even when a site's sidebar spotlights a few of them elsewhere. * - * Returns the number of pages relocated. + * `categoryRe` matches an existing category title to merge into (so we don't + * create a duplicate); when none matches, a fresh `{ title: defaultTitle }` + * category is appended. Returns the number of pages relocated. */ -function reclassifyReferencePages(scraped) { - const REFERENCE_SEGMENT = /^(api[-_]?reference|endpoints?)$/i +function reclassifyPagesByUrlSegment(scraped, { segmentRe, categoryRe, defaultTitle }) { const normTitle = (t) => String(t || '') .replace(INVISIBLE_CHARS, '') .trim() .toLowerCase() - const looksLikeRefUrl = (url) => { + const urlHasSegment = (url) => { try { const segs = new URL(url).pathname.split('/').filter(Boolean) - return segs.some((s) => REFERENCE_SEGMENT.test(s)) + return segs.some((s) => segmentRe.test(s)) } catch { return false } } - // Find (or create) the canonical API Reference category. Prefer an existing - // one with a reference-shaped title so we don't end up with duplicates. - let refCat = scraped.categories.find((c) => /^(api[ -]?reference|reference|api|endpoints?)$/i.test(normTitle(c.title).replace(/\s+/g, ' '))) - const existedBefore = Boolean(refCat) + // Find (or create) the canonical destination category. Prefer an existing + // one with a matching title so we don't end up with duplicates. + let destCat = scraped.categories.find((c) => categoryRe.test(normTitle(c.title).replace(/\s+/g, ' '))) const collected = [] const filterPages = (pages) => { const kept = [] for (const p of pages || []) { - if (looksLikeRefUrl(p.url)) { - // Flatten sub-pages when relocating — API Reference is a flat list. + if (urlHasSegment(p.url)) { + // Flatten sub-pages when relocating — these sections are flat lists. collectFlat(p, collected) continue } @@ -1895,32 +1921,61 @@ function reclassifyReferencePages(scraped) { return kept } - // Never pull pages out of the reference category itself. + // Never pull pages out of the destination category itself. for (const cat of scraped.categories) { - if (cat === refCat) continue + if (cat === destCat) continue cat.pages = filterPages(cat.pages) } if (collected.length === 0) return 0 - if (!refCat) { - refCat = { title: 'API Reference', pages: [] } - scraped.categories.push(refCat) + if (!destCat) { + destCat = { title: defaultTitle, pages: [] } + scraped.categories.push(destCat) } - // Dedupe against anything already in the reference category. - const seen = new Set(refCat.pages.map((p) => normalizePath(p.url))) + // Dedupe against anything already in the destination category. + const seen = new Set(destCat.pages.map((p) => normalizePath(p.url))) for (const p of collected) { const key = normalizePath(p.url) if (seen.has(key)) continue seen.add(key) - refCat.pages.push(p) + destCat.pages.push(p) } - // Drop now-empty categories (other than the reference one we may have just created). - scraped.categories = scraped.categories.filter((c) => c === refCat || (c.pages && c.pages.length > 0)) + // Drop now-empty categories (other than the one we may have just created). + scraped.categories = scraped.categories.filter((c) => c === destCat || (c.pages && c.pages.length > 0)) - // If the category existed before but the relocation was a no-op, surface 0. - return existedBefore ? collected.length : collected.length + return collected.length +} + +/** + * Sweep `/api-reference/*` and `/endpoints/*` pages into one "API Reference" + * category. Some docs sites (e.g. greenflash.ai) spotlight a handful of + * endpoints under "Developers" in the sidebar while the bulk of endpoints + * live under a separate API Reference section — this lands them all in + * `reference/` after staging. Returns the number of pages relocated. + */ +function reclassifyReferencePages(scraped) { + return reclassifyPagesByUrlSegment(scraped, { + segmentRe: /^(api[-_]?reference|endpoints?)$/i, + categoryRe: /^(api[ -]?reference|reference|api|endpoints?)$/i, + defaultTitle: 'API Reference', + }) +} + +/** + * Sweep `/changelog/*`, `/release-notes/*` and `/releases/*` pages into one + * "Changelog" category, wherever the site's sidebar happened to place them. + * That category routes to `docs/Changelog/`; `--separate-changelog` later + * relocates it to a top-level `changelog/` dir. Returns the number of pages + * relocated. + */ +function reclassifyChangelogPages(scraped) { + return reclassifyPagesByUrlSegment(scraped, { + segmentRe: /^(changelog|change[-_]?log|release[-_]?notes?|releases|whats?[-_]?new)$/i, + categoryRe: /^(changelog|change ?log|release ?notes?|releases|what'?s ?new)$/i, + defaultTitle: 'Changelog', + }) } function collectFlat(page, out) { @@ -3416,6 +3471,52 @@ function countPagesDeep(pages) { return n } +/** + * Post-staging relocation for `--separate-changelog`: move the staged + * `docs/Changelog/` category up to a top-level `changelog/` directory. + * + * Done as a filesystem move *after* staging (rather than routing there + * directly) so the staged tree is a valid git-format layout up to this point, + * and the one schema-divergent step lives in a single, clearly-named place. + * Note `changelog/` is not part of the git-format schema — it won't lint or + * sync to ReadMe until git-format adds first-class changelog support. + * + * Returns the number of changelog pages moved (0 if there were none). + */ +function relocateChangelogDir(stagingDir) { + const srcDir = path.join(stagingDir, 'docs', 'Changelog') + if (!fs.existsSync(srcDir) || !fs.statSync(srcDir).isDirectory()) return 0 + + // Count changelog pages (recursively, _order.yaml excluded) for reporting. + let pageCount = 0 + const countMd = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory()) countMd(path.join(dir, entry.name)) + else if (entry.name.endsWith('.md')) pageCount++ + } + } + countMd(srcDir) + + // changelog/ is a fresh top-level dir — move the whole folder across in one + // step. Its own _order.yaml travels with it and stays valid. + fs.renameSync(srcDir, path.join(stagingDir, 'changelog')) + + // docs/_order.yaml lists the docs category subfolders — drop the moved one + // so the docs sidebar doesn't point at a folder that's no longer there. + const docsOrderPath = path.join(stagingDir, 'docs', '_order.yaml') + if (fs.existsSync(docsOrderPath)) { + const entries = yamlRequire().load(fs.readFileSync(docsOrderPath, 'utf-8')) + const filtered = Array.isArray(entries) ? entries.filter((e) => e !== 'Changelog') : [] + if (filtered.length > 0) { + fs.writeFileSync(docsOrderPath, filtered.map((s) => `- ${yamlSafeSlug(s)}`).join('\n') + '\n') + } else { + fs.rmSync(docsOrderPath) + } + } + + return pageCount +} + /** * Map a category title to the git-format top-level directory + optional * category subdir. docs/ is the only top dir that takes a subfolder. @@ -3423,6 +3524,9 @@ function countPagesDeep(pages) { function routeCategory(title) { const t = (title || '').trim() if (/^(api[ -]?reference|reference|api|endpoints?)$/i.test(t)) return { topDir: 'reference', subDir: null } + // Changelog-ish categories normalize to one docs/Changelog/ folder so the + // --separate-changelog relocation has a single, predictable source dir. + if (/^(changelog|change[ -]?log|release[ -]?notes?|releases|what'?s[ -]?new)$/i.test(t)) return { topDir: 'docs', subDir: 'Changelog' } if (/^(recipes?|cookbook|tutorials?|how[ -]?tos?)$/i.test(t)) return { topDir: 'recipes', subDir: null } if (/^(custom[ -]?pages?|landing( page)?s?)$/i.test(t)) return { topDir: 'custom_pages', subDir: null } if (/^(custom[ -]?blocks?|snippets?|reusable( content)?)$/i.test(t)) return { topDir: 'custom_blocks', subDir: null } From 483be3073a5bc5fa992efe3a877a44117dce8e15 Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Tue, 26 May 2026 16:37:02 +1000 Subject: [PATCH 15/43] feat: flatten hierarchy to single layer (readme style changelogs) --- src/commands/import.js | 49 +++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 795a6a4..3f80436 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -3487,19 +3487,48 @@ function relocateChangelogDir(stagingDir) { const srcDir = path.join(stagingDir, 'docs', 'Changelog') if (!fs.existsSync(srcDir) || !fs.statSync(srcDir).isDirectory()) return 0 - // Count changelog pages (recursively, _order.yaml excluded) for reporting. - let pageCount = 0 - const countMd = (dir) => { + const dstDir = path.join(stagingDir, 'changelog') + fs.renameSync(srcDir, dstDir) + + // Collect every real x-import page from the (possibly nested) tree. + // Synthetic group-only stubs (no x-import — they exist only to render an + // empty parent in the sidebar) are dropped: the flat layout has no concept + // of containers, so they'd just be dead frontmatter. + const pages = [] + const walk = (dir, ancestors) => { for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { - if (entry.isDirectory()) countMd(path.join(dir, entry.name)) - else if (entry.name.endsWith('.md')) pageCount++ + const full = path.join(dir, entry.name) + if (entry.isDirectory()) { + walk(full, [...ancestors, entry.name]) + continue + } + if (!entry.name.endsWith('.md')) continue + const parsed = matter(fs.readFileSync(full, 'utf-8')) + if (!parsed.data || !parsed.data['x-import']) continue + const slug = entry.name.replace(/\.md$/, '') + pages.push({ ancestors, slug, content: parsed.content, data: parsed.data }) } } - countMd(srcDir) + walk(dstDir, []) + + // Wipe the nested tree and rewrite it flat. Filename concatenates the + // ancestor folder slugs with the leaf slug so siblings share a prefix — + // alphabetical order then naturally matches authored reading order, and + // names are unique across the (formerly nested) tree. + fs.rmSync(dstDir, { recursive: true, force: true }) + fs.mkdirSync(dstDir, { recursive: true }) + + const flatSlugs = [] + for (const p of pages) { + const flatSlug = [...p.ancestors, p.slug].join('-') + const frontmatter = { ...p.data } + delete frontmatter.icon + fs.writeFileSync(path.join(dstDir, `${flatSlug}.md`), matter.stringify(p.content, frontmatter)) + flatSlugs.push(flatSlug) + } - // changelog/ is a fresh top-level dir — move the whole folder across in one - // step. Its own _order.yaml travels with it and stays valid. - fs.renameSync(srcDir, path.join(stagingDir, 'changelog')) + flatSlugs.sort() + fs.writeFileSync(path.join(dstDir, '_order.yaml'), flatSlugs.map((s) => `- ${yamlSafeSlug(s)}`).join('\n') + '\n') // docs/_order.yaml lists the docs category subfolders — drop the moved one // so the docs sidebar doesn't point at a folder that's no longer there. @@ -3514,7 +3543,7 @@ function relocateChangelogDir(stagingDir) { } } - return pageCount + return pages.length } /** From 509c760bc6f0abe2965580e2c1e48b991c0c6af4 Mon Sep 17 00:00:00 2001 From: Bach Tran Date: Tue, 26 May 2026 19:54:54 +1000 Subject: [PATCH 16/43] fix(importer): RM-16780 stop hardcoding folder icons on empty parents Empty-parent stubs (synthetic dropdowns standing in for missing URL segments) shipped with a hardcoded fa-solid fa-folder icon, which reads as a placeholder next to the sidebar's expand chevron. Reuse the same pickIcon heuristic leaf pages already use, and filter the folder family out of every picker pass so it can't be selected anywhere. --- src/commands/import.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index ffc3ad8..322c3c4 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -3158,10 +3158,12 @@ function stageOrganized(organized, stagingDir, opts = {}) { const relFilePath = `${dir}/${slug}.md` let frontmatter if (isEmptyParent) { - // Synthetic folder page: covers a missing URL segment so descendants - // nest at the right depth. No source URL to import, and we want it - // visible so the folder shows up in the sidebar. - frontmatter = { title: page.title, icon: 'fa-solid fa-folder' } + // Synthetic parent page: covers a missing URL segment so descendants + // nest at the right depth. No source URL to import. Reuse the same + // pickIcon heuristic leaf pages use so the parent gets a topical + // icon — the sidebar already renders an expand chevron, so a + // hardcoded `fa-folder` would just look like a placeholder. + frontmatter = { title: page.title, icon: formatIconClass(pickIcon(slug, page.title)) } } else { // Sub-pages don't get icons per design decision. frontmatter = buildFrontmatter(topDir, page, slug, pickIcon, { skipIcon: isSubPage }) From 8503b3ea50e72faeb48eb3bf583dc40859246434 Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Wed, 27 May 2026 14:12:53 +1000 Subject: [PATCH 17/43] feat: handle changelog with version-xxx format --- src/commands/import.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 3f80436..9277c16 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -1972,8 +1972,8 @@ function reclassifyReferencePages(scraped) { */ function reclassifyChangelogPages(scraped) { return reclassifyPagesByUrlSegment(scraped, { - segmentRe: /^(changelog|change[-_]?log|release[-_]?notes?|releases|whats?[-_]?new)$/i, - categoryRe: /^(changelog|change ?log|release ?notes?|releases|what'?s ?new)$/i, + segmentRe: /^(changelog|change[-_]?log|release[-_]?notes?|releases?|release[-_]?v?\d[\w.-]*|whats?[-_]?new)$/i, + categoryRe: /^(changelog|change ?log|release ?notes?|releases?|what'?s ?new)$/i, defaultTitle: 'Changelog', }) } @@ -3555,7 +3555,7 @@ function routeCategory(title) { if (/^(api[ -]?reference|reference|api|endpoints?)$/i.test(t)) return { topDir: 'reference', subDir: null } // Changelog-ish categories normalize to one docs/Changelog/ folder so the // --separate-changelog relocation has a single, predictable source dir. - if (/^(changelog|change[ -]?log|release[ -]?notes?|releases|what'?s[ -]?new)$/i.test(t)) return { topDir: 'docs', subDir: 'Changelog' } + if (/^(changelog|change[ -]?log|release[ -]?notes?|releases?|what'?s[ -]?new)$/i.test(t)) return { topDir: 'docs', subDir: 'Changelog' } if (/^(recipes?|cookbook|tutorials?|how[ -]?tos?)$/i.test(t)) return { topDir: 'recipes', subDir: null } if (/^(custom[ -]?pages?|landing( page)?s?)$/i.test(t)) return { topDir: 'custom_pages', subDir: null } if (/^(custom[ -]?blocks?|snippets?|reusable( content)?)$/i.test(t)) return { topDir: 'custom_blocks', subDir: null } From 22ec9d8c79ef61839bbd8b41aa9c58e2850620b1 Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Wed, 27 May 2026 15:44:52 +1000 Subject: [PATCH 18/43] feat: changelog, release notes, whats new always go to changelogs --- src/commands/import.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/commands/import.js b/src/commands/import.js index 9277c16..5a8ca54 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -1972,7 +1972,13 @@ function reclassifyReferencePages(scraped) { */ function reclassifyChangelogPages(scraped) { return reclassifyPagesByUrlSegment(scraped, { - segmentRe: /^(changelog|change[-_]?log|release[-_]?notes?|releases?|release[-_]?v?\d[\w.-]*|whats?[-_]?new)$/i, + // Anchored half covers ambiguous bare keywords (release, releases, versioned + // release-2026-2) — those need to be the whole segment so we don't sweep + // `/release-pipeline/` etc. Unanchored half covers the unambiguous keywords + // (changelog, release-notes, whats-new), which we catch as substrings so + // slugs like `/docs/changelog-javascript-agent` and `/docs/ios-sdk-changelog` + // get pulled in too. + segmentRe: /(?:^(?:releases?|release[-_]?v?\d[\w.-]*)$)|(?:changelog|change[-_]?log|release[-_]?notes?|whats?[-_]?new)/i, categoryRe: /^(changelog|change ?log|release ?notes?|releases?|what'?s ?new)$/i, defaultTitle: 'Changelog', }) From ed41958dd86dc10ce4d86d05efa1b3d1ef4b32a8 Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Wed, 27 May 2026 15:52:33 +1000 Subject: [PATCH 19/43] feat: extract changelogs pre-ai formatting of the sidebar --- src/commands/import.js | 126 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 117 insertions(+), 9 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 5a8ca54..d0390db 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -310,10 +310,34 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna } } + // Pre-extract changelog pages so AI / URL clustering / section-direct paths + // never see them as candidates for organization. We re-attach them as a + // dedicated Changelog category once `organized` is finalized below — that + // guarantees they route to docs/Changelog/ (or to changelog/ with + // --separate-changelog) regardless of which organize path ran. Done here, + // before knownUrls is derived, so every downstream consumer operates on the + // already-changelog-free view. + let extractedChangelog = [] + if (llms) { + extractedChangelog = extractChangelogFromSections(llms.parsed.sections) + } else if (sitemapKnownUrls.length > 0) { + const { extracted, kept } = partitionChangelogFromKnownUrls(sitemapKnownUrls) + extractedChangelog = extracted + sitemapKnownUrls = kept + } + if (extractedChangelog.length > 0) { + styles.info( + styles.dim( + `Pre-extracted ${extractedChangelog.length} changelog page${extractedChangelog.length === 1 ? '' : 's'} — bypassing organization, attaching as Changelog category at the end.`, + ), + ) + } + const dbgSuffix = `-${sourceUrl.hostname}` if (debugSnapshots) { debugSnapshots[`01-llms-parsed${dbgSuffix}.json`] = { llmsUrl, parsed: llms ? llms.parsed : null, skipped: skippedLlms } debugSnapshots[`01b-sitemap${dbgSuffix}.json`] = { sitemapUrl, urls: sitemapKnownUrls } + debugSnapshots[`01c-extracted-changelog${dbgSuffix}.json`] = extractedChangelog } let knownUrls = [] @@ -625,7 +649,13 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna categories: clustered.map((c) => ({ title: c.title, icon: null, pages: c.pages })), } } - + + // Re-attach the pre-extracted changelog pages as their own Changelog + // category. Done before nestByUrlHierarchy so the injected pages get nested + // alongside the rest of the tree for consistency. Merges into a pre-existing + // Changelog category (scrape path's reclassifier may have already built one). + injectChangelogCategory(organized, extractedChangelog) + for (const cat of organized.categories || []) { cat.pages = nestByUrlHierarchy(cat.pages) } @@ -1963,6 +1993,90 @@ function reclassifyReferencePages(scraped) { }) } +// Anchored half covers ambiguous bare keywords (release, releases, versioned +// release-2026-2) — those need to be the whole segment so we don't sweep +// `/release-pipeline/` etc. Unanchored half covers the unambiguous keywords +// (changelog, release-notes, whats-new), which we catch as substrings so +// slugs like `/docs/changelog-javascript-agent` and `/docs/ios-sdk-changelog` +// get pulled in too. +const CHANGELOG_URL_SEGMENT_RE = /(?:^(?:releases?|release[-_]?v?\d[\w.-]*)$)|(?:changelog|change[-_]?log|release[-_]?notes?|whats?[-_]?new)/i +const CHANGELOG_CATEGORY_TITLE_RE = /^(changelog|change ?log|release ?notes?|releases?|what'?s ?new)$/i + +function urlIsChangelog(url) { + try { + const segs = new URL(url).pathname.split('/').filter(Boolean) + return segs.some((s) => CHANGELOG_URL_SEGMENT_RE.test(s)) + } catch { + return false + } +} + +/** + * Pull changelog items OUT of llms.parsed.sections in place. Done before the + * organize pipeline runs so AI / clustering never sees them — they get + * re-attached as a dedicated Changelog category at the end via + * injectChangelogCategory. Sections that go empty afterwards are filtered out + * naturally by `usableSections` downstream. + * + * Returns the extracted items in `{ title, url, description? }` shape. + */ +function extractChangelogFromSections(sections) { + const extracted = [] + for (const section of sections || []) { + const kept = [] + for (const item of section.items || []) { + if (urlIsChangelog(item.url)) { + extracted.push({ + title: item.text, + url: item.url, + ...(item.description ? { description: item.description } : {}), + }) + } else { + kept.push(item) + } + } + section.items = kept + } + return extracted +} + +/** + * Split a flat known-URL list into changelog vs. the rest. Used for the + * sitemap-only fallback, where there's no llms.txt section structure to mutate. + */ +function partitionChangelogFromKnownUrls(knownUrls) { + const extracted = [] + const kept = [] + for (const p of knownUrls || []) { + if (urlIsChangelog(p.url)) extracted.push(p) + else kept.push(p) + } + return { extracted, kept } +} + +/** + * Attach pre-extracted changelog items as a dedicated Changelog category on + * the final organized tree. Merges into an existing Changelog-titled category + * if one is already present (e.g. the scrape path's reclassifier already built + * one); otherwise creates a fresh one. Dedupe is by normalized URL path. + */ +function injectChangelogCategory(organized, items) { + if (!items || items.length === 0) return + organized.categories = organized.categories || [] + let dest = organized.categories.find((c) => CHANGELOG_CATEGORY_TITLE_RE.test(String(c.title || '').trim())) + if (!dest) { + dest = { title: 'Changelog', icon: null, pages: [] } + organized.categories.push(dest) + } + const seen = new Set((dest.pages || []).map((p) => normalizePath(p.url))) + for (const item of items) { + const key = normalizePath(item.url) + if (seen.has(key)) continue + seen.add(key) + dest.pages.push(item) + } +} + /** * Sweep `/changelog/*`, `/release-notes/*` and `/releases/*` pages into one * "Changelog" category, wherever the site's sidebar happened to place them. @@ -1972,14 +2086,8 @@ function reclassifyReferencePages(scraped) { */ function reclassifyChangelogPages(scraped) { return reclassifyPagesByUrlSegment(scraped, { - // Anchored half covers ambiguous bare keywords (release, releases, versioned - // release-2026-2) — those need to be the whole segment so we don't sweep - // `/release-pipeline/` etc. Unanchored half covers the unambiguous keywords - // (changelog, release-notes, whats-new), which we catch as substrings so - // slugs like `/docs/changelog-javascript-agent` and `/docs/ios-sdk-changelog` - // get pulled in too. - segmentRe: /(?:^(?:releases?|release[-_]?v?\d[\w.-]*)$)|(?:changelog|change[-_]?log|release[-_]?notes?|whats?[-_]?new)/i, - categoryRe: /^(changelog|change ?log|release ?notes?|releases?|what'?s ?new)$/i, + segmentRe: CHANGELOG_URL_SEGMENT_RE, + categoryRe: CHANGELOG_CATEGORY_TITLE_RE, defaultTitle: 'Changelog', }) } From 2ea243a7c3e14d012f2271f60a39470fefe611c7 Mon Sep 17 00:00:00 2001 From: Xavier Andueza Date: Fri, 5 Jun 2026 17:03:44 +1000 Subject: [PATCH 20/43] feat: scan found llms.txt files for reference to other llms.txt files --- src/commands/import.js | 92 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/src/commands/import.js b/src/commands/import.js index ece10fa..3c2abbd 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -3074,6 +3074,16 @@ const LLMS_PROBE_CAP = 30 // list hundreds of URLs; without this a single root hit could schedule // dozens of unrelated probes (`/blog`, `/pricing`, …). const LLMS_CHILDREN_PER_NODE = 12 +// Budget for the explicit-link phase of discovery (phase 2 in discoverLlmsTxt). +// Separate from LLMS_PROBE_CAP so following authored `llms.txt` links — a +// strong, deliberate signal, e.g. docs.snowflake.com's root index listing ~40 +// per-section files — isn't starved by the speculative slug-walk's budget. +const EXPLICIT_LLMS_PROBE_CAP = 200 + +// Matches ONLY the canonical index filename, not sibling dumps like +// `llms-full.txt` (concatenated page content, not a link index) — those would +// add no link rows and just waste a fetch. +const LLMS_INDEX_RE = /\/llms\.txt$/i /** * Convert a path like `/docs/quickstart` into its llms.txt URL. @@ -3083,6 +3093,39 @@ function pathToLlmsUrl(origin, path) { return `${origin}${path}/llms.txt` } +/** + * Derive the path a fetched llms.txt "lives at" (its pathname minus the + * trailing `/llms.txt`), so explicit-link hits carry the same `path` shape + * the slug-walk hits do — mergeValidHits sorts by it (deepest wins) and + * narrowToDocsSubtreeIfNeeded reads it. + */ +function llmsPathFromUrl(llmsUrl) { + const pn = new URL(llmsUrl).pathname.replace(/\/llms\.txt$/i, '') + return pn || '/' +} + +/** + * Pull explicit child-index links out of a parsed llms.txt — item rows whose + * URL literally points at another `llms.txt` (e.g. Snowflake's root file + * listing `…/data-integration/llms.txt`). Restricted to the same origin as the + * source: cross-origin llms.txt would drag in unrelated third-party docs, and + * the slug-walk's path math assumes one origin. Returns deduped URL strings. + */ +function extractNestedLlmsUrls(parsed, originScope) { + const out = new Set() + for (const section of parsed.sections) { + for (const item of section.items) { + try { + const u = new URL(item.url) + if (u.origin === originScope && LLMS_INDEX_RE.test(u.pathname)) out.add(u.toString()) + } catch { + // ignore unparseable URLs + } + } + } + return [...out] +} + /** * Given a current path + a parsed llms.txt, return the next paths to probe: * the parent directory (UP) and distinct first-segment-children extracted @@ -3186,6 +3229,55 @@ async function discoverLlmsTxt(sourceUrl) { frontier = next } + // Phase 2 — follow EXPLICIT llms.txt links authored inside the files phase 1 + // discovered, recursively. No slug-guessing here: we only fetch URLs that + // literally point at an llms.txt (a root index listing per-section files, + // like docs.snowflake.com). Each new file's own explicit links are followed + // in turn until none remain. Dedupes against every URL phase 1 touched + // (hits + misses) and against itself; phase 1's `tried` set is keyed by path, + // so we track full llms.txt URLs separately here and leave phase 1 untouched. + const triedUrls = new Set([...hits.map((h) => h.llmsUrl), ...misses.map((m) => m.url)]) + let explicitFrontier = [] + for (const hit of hits) { + for (const u of extractNestedLlmsUrls(hit.parsed, sourceUrl.origin)) { + if (!triedUrls.has(u)) explicitFrontier.push(u) + } + } + + let explicitProbes = 0 + while (explicitFrontier.length > 0 && explicitProbes < EXPLICIT_LLMS_PROBE_CAP) { + const ring = [] + for (const u of explicitFrontier) { + if (triedUrls.has(u)) continue + triedUrls.add(u) + ring.push(u) + explicitProbes++ + if (explicitProbes >= EXPLICIT_LLMS_PROBE_CAP) break + } + if (ring.length === 0) break + + const results = await Promise.all(ring.map(async (llmsUrl) => ({ llmsUrl, res: await fetchLlmsTxt(llmsUrl) }))) + + const next = [] + for (const { llmsUrl, res } of results) { + if (!res.ok) { + misses.push({ url: llmsUrl, status: res.status, error: res.error }) + continue + } + const linkItems = res.stats?.linkItems ?? 0 + if (linkItems === 0) { + misses.push({ url: llmsUrl, status: 'no link items' }) + continue + } + hits.push({ ...res, llmsUrl, path: llmsPathFromUrl(llmsUrl) }) + if (!res.usable) skipped.push({ url: llmsUrl, reason: res.reason }) + for (const nested of extractNestedLlmsUrls(res.parsed, sourceUrl.origin)) { + if (!triedUrls.has(nested)) next.push(nested) + } + } + explicitFrontier = next + } + return { hits, misses, skipped } } From c47c0122834d72f711226fbcf12911172ab1b659 Mon Sep 17 00:00:00 2001 From: Xavier Andueza <88118725+xavierandueza@users.noreply.github.com> Date: Thu, 11 Jun 2026 08:53:08 +1000 Subject: [PATCH 21/43] feat: extract oas files from llms.txt files (#10) Co-authored-by: Xavier Andueza --- src/commands/import.js | 153 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 147 insertions(+), 6 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 3c2abbd..65d2d9e 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -113,6 +113,19 @@ export async function importDocs(options) { const organized = sourceUrls.length === 1 ? perSourceOrganized[0] : mergeOrganized(perSourceOrganized) + // Union the candidate OpenAPI-spec URLs across all sources, deduped by URL + // (mergeOrganized intentionally drops non-category fields, so collect from + // the per-source results directly). + const oasJsonUrls = [] + const seenOasUrls = new Set() + for (const o of perSourceOrganized) { + for (const item of o.oasJsonUrls || []) { + if (seenOasUrls.has(item.url)) continue + seenOasUrls.add(item.url) + oasJsonUrls.push(item) + } + } + if (debugSnapshots) { debugSnapshots['05-organized.json'] = organized const debugDir = path.join(os.tmpdir(), `readme-import-debug-${hostnameJoined}-${Date.now()}`) @@ -165,6 +178,17 @@ export async function importDocs(options) { if (staged.skippedApiRef > 0) { styles.info(`Skipped ${styles.bold(String(staged.skippedApiRef))} API reference page${staged.skippedApiRef === 1 ? '' : 's'} (--skip-api-reference)`) } + + // Download captured `.json` specs into /oas/. Raw bytes only — + // validation + OpenAPI detection + upload all happen in the runner. Staged + // here (not in docs/ or reference/) so it rides the zip but is invisible to + // the docs build. Non-fatal: a failed download is logged and skipped. + if (oasJsonUrls.length > 0) { + const oasCount = await timePhase('download oas specs', () => downloadOasSpecs(oasJsonUrls, stagingDir)) + if (oasCount > 0) { + styles.ok(`Downloaded ${styles.bold(String(oasCount))} candidate OpenAPI spec${oasCount === 1 ? '' : 's'} into ${styles.bold('oas/')}.`) + } + } console.log() if (options.test) { @@ -232,13 +256,26 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna let llmsUrl = llms?.llmsUrl || null const skippedLlms = discovery.skipped - // Drop asset/meta items (llms-full.txt, openapi.json, …) from the merged - // parsed result up-front, so every downstream consumer — knownUrls AND the - // "use llms.txt sections directly" organize path — sees a clean URL list. + // Capture `.json` links (candidate OpenAPI specs) BEFORE anything else sees + // the parsed tree. They're removed in place here, downloaded into the staged + // `oas/` dir by importDocs, and validated + uploaded by the runner. Stub + // generation never sees them. + let oasJsonUrls = [] + if (llms) { + oasJsonUrls = extractOasJsonUrlsFromParsed(llms.parsed) + if (oasJsonUrls.length > 0) { + styles.info(styles.dim(`Captured ${oasJsonUrls.length} .json link${oasJsonUrls.length === 1 ? '' : 's'} as candidate OpenAPI spec${oasJsonUrls.length === 1 ? '' : 's'} → oas/.`)) + } + } + + // Drop the remaining asset/meta items (llms-full.txt, .yaml specs, …) from + // the merged parsed result up-front, so every downstream consumer — knownUrls + // AND the "use llms.txt sections directly" organize path — sees a clean URL + // list. `.json` items were already pulled out above. if (llms) { const dropped = dropAssetItemsFromParsed(llms.parsed) if (dropped > 0) { - styles.info(styles.dim(`Dropped ${dropped} asset/meta URL${dropped === 1 ? '' : 's'} (.json/.yaml/.xml/.toml, llms*.txt) from llms.txt items.`)) + styles.info(styles.dim(`Dropped ${dropped} asset/meta URL${dropped === 1 ? '' : 's'} (.yaml/.xml/.toml, llms*.txt) from llms.txt items.`)) } } @@ -661,6 +698,7 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna } styles.ok(`Organized in ${styles.bold(formatDuration(Date.now() - organizeStart))}.`) + organized.oasJsonUrls = oasJsonUrls return organized } @@ -2400,6 +2438,62 @@ async function fetchHtmlDirect(url) { } } +/** + * Download a `.json` URL as raw bytes. Returns a Buffer, or null on non-OK / + * network error. No parsing or validation — the runner is the OAS gate. + */ +async function fetchJsonSpec(url) { + try { + const res = await fetch(url, { + redirect: 'follow', + headers: { 'User-Agent': 'readme-cli-import', Accept: 'application/json' }, + }) + if (!res.ok) return null + return Buffer.from(await res.arrayBuffer()) + } catch { + return null + } +} + +/** + * Download the captured candidate-spec URLs into `/oas/.json`. + * Filenames come from the URL basename (collisions get a numeric suffix); the + * runner de-dups by content, so identical bytes under different names are + * handled there. Returns the count of files written. + */ +async function downloadOasSpecs(oasJsonUrls, stagingDir) { + const oasDir = path.join(stagingDir, 'oas') + fs.mkdirSync(oasDir, { recursive: true }) + const usedNames = new Set() + let written = 0 + for (const { url } of oasJsonUrls) { + const buf = await fetchJsonSpec(url) + if (!buf) { + styles.info(styles.dim(` oas: skipped ${url} (download failed)`)) + continue + } + // Derive a safe, unique *.json filename from the URL basename. + let base + try { + base = path.basename(new URL(url).pathname) || 'openapi.json' + } catch { + base = 'openapi.json' + } + base = base.replace(/[^a-zA-Z0-9._-]/g, '-') + if (!/\.json$/i.test(base)) base += '.json' + let name = base + let n = 2 + while (usedNames.has(name)) { + name = base.replace(/\.json$/i, `-${n}.json`) + n++ + } + usedNames.add(name) + fs.writeFileSync(path.join(oasDir, name), buf) + written++ + } + return written +} + /** * Firecrawl-backed HTML loader. Firecrawl runs a real browser, waits for * hydration, and returns the rendered DOM — which is what we need for sites @@ -2912,10 +3006,15 @@ async function runJsonQuery({ systemPrompt, userPrompt, model }) { */ // URL patterns we strip from llms.txt items — they show up alongside real // pages but shouldn't be staged as docs: -// - openapi.json, package.json, .yaml specs, etc. (machine-readable specs) +// - .yaml specs, package.json-ish assets, sitemaps, etc. (machine-readable) // - llms.txt, llms-full.txt, llms-ctx.txt, … (other llms.txt variants // that point back at the same index/dump, not real pages) -const ASSET_EXT_RE = /\.(json|ya?ml|xml|toml)$/i +// +// `.json` is deliberately NOT in this list: those links are captured up-front +// by extractOasJsonUrlsFromParsed (they may be OpenAPI specs we want to +// download into oas/ and upload separately) and removed from the parsed tree +// before this drop runs, so a `.json` item never reaches here. +const ASSET_EXT_RE = /\.(ya?ml|xml|toml)$/i const LLMS_TXT_RE = /(?:^|\/)llms[^/]*\.txt$/i function isAssetOrMetaUrl(url) { try { @@ -2926,6 +3025,19 @@ function isAssetOrMetaUrl(url) { } } +// A llms.txt item whose URL points at a `.json` file. These are pulled out of +// the parsed tree before organization (so stub generation never sees them) and +// downloaded into the staged `oas/` dir; the runner validates each as OpenAPI +// and uploads the valid ones. NOTE: only `.json` is captured — `.yaml`/`.yml` +// specs are intentionally left to the asset drop above. +function isOasJsonUrl(url) { + try { + return /\.json$/i.test(new URL(url).pathname) + } catch { + return false + } +} + // Immediate-child segments of root that signal a docs subtree on a // non-docs hostname. We deliberately keep the set tiny — over-matching // here (e.g. including `api`, `learn`) would over-filter sites whose @@ -3021,6 +3133,35 @@ function narrowToDocsSubtreeIfNeeded(llms, sourceUrl, hits) { return { segment: docsSegment, kept, dropped } } +/** + * Pull `.json` items out of a parsed llms.txt structure IN PLACE, returning the + * removed items as `[{ url, text }]`. Sections emptied by the removal are + * pruned too — same shape as dropAssetItemsFromParsed. Run BEFORE + * dropAssetItemsFromParsed / knownUrls / organize so the captured specs never + * enter the stub tree (skeleton has zero visibility of OAS files). The caller + * downloads the returned URLs into the staged `oas/` dir. + */ +function extractOasJsonUrlsFromParsed(parsed) { + const captured = [] + const keptSections = [] + for (const section of parsed.sections) { + const keptItems = [] + for (const item of section.items) { + if (isOasJsonUrl(item.url)) { + captured.push({ url: item.url, text: item.text || null }) + continue + } + keptItems.push(item) + } + if (keptItems.length > 0) { + section.items = keptItems + keptSections.push(section) + } + } + parsed.sections = keptSections + return captured +} + /** * Drop asset/meta items in place from a parsed llms.txt structure. Sections * that end up empty after the drop are removed too. Returns the count of From 2b8521f62bbd828a2b7b54375ac1378ad2e9b8d8 Mon Sep 17 00:00:00 2001 From: Xavier Andueza <88118725+xavierandueza@users.noreply.github.com> Date: Tue, 16 Jun 2026 10:25:35 +1000 Subject: [PATCH 22/43] fix: workaround for bad slug on single folder within folder (#11) --- src/commands/import.js | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/commands/import.js b/src/commands/import.js index 65d2d9e..ae5be3b 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -3695,6 +3695,25 @@ function printPagesTree(pages, indentLevel) { } } +/** + * Collapse redundant single-folder layers in a category/page subtree, in place. + * + * When a container (category or page) has exactly one child and that child is a + * pure folder node — no `x-import-url` gitto/readme has an issue with unique slug + * generation. + * This is a temporary workaround. + * TODO: Remove after fixing on the Gitto/ReadMe side. + */ +function collapseRedundantLayers(container) { + for (const child of container.pages || []) collapseRedundantLayers(child) + while ((container.pages || []).length === 1) { + const only = container.pages[0] + if (only.url) break // real page (gets x-import), just has children + if (!only.pages || only.pages.length === 0) break // nothing to lift up + container.pages = only.pages + } +} + /** * Write the organized hierarchy to disk as git-format markdown stubs — just * frontmatter, no body yet. docs/ pages go under docs//.md; @@ -3717,6 +3736,9 @@ function stageOrganized(organized, stagingDir, opts = {}) { return true }) + // TODO: Remove after readme/gitto handles better + for (const cat of eligibleCategories) collapseRedundantLayers(cat) + // Slug names must be unique const slugFor = ensureUniqueSlugs(eligibleCategories) From d5690b9fa839d4f8c98d35fc00ee4a2eb26a50d1 Mon Sep 17 00:00:00 2001 From: minh <150941282+minhthanhdang@users.noreply.github.com> Date: Thu, 18 Jun 2026 17:26:15 +1000 Subject: [PATCH 23/43] fix: duplicate pages produced when slug ends with -N suffix --- .claude/skills/readme-cli/SKILL.md | 2 +- src/validators/numbering.js | 134 ----------------------------- vendor/TOOLS.md | 2 +- 3 files changed, 2 insertions(+), 136 deletions(-) delete mode 100644 src/validators/numbering.js diff --git a/.claude/skills/readme-cli/SKILL.md b/.claude/skills/readme-cli/SKILL.md index bdb4dcc..7b128d7 100644 --- a/.claude/skills/readme-cli/SKILL.md +++ b/.claude/skills/readme-cli/SKILL.md @@ -17,7 +17,7 @@ npx @readme/cli lint --fix # Automatically fix common issues npx @readme/cli lint --json # Machine-readable output (good for CI) ``` -Validates: frontmatter, ordering, numbering, duplicates, OAS references, OAS schema, MDX components, and recipes. +Validates: frontmatter, ordering, duplicates, OAS references, OAS schema, MDX components, and recipes. Always try `lint --fix` before attempting manual fixes. diff --git a/src/validators/numbering.js b/src/validators/numbering.js deleted file mode 100644 index 2ac9aff..0000000 --- a/src/validators/numbering.js +++ /dev/null @@ -1,134 +0,0 @@ -import fs from 'node:fs' -import path from 'node:path' -import readline from 'node:readline' -import * as styles from '../utils/styles.js' - -export const name = 'numbering' - -const SUFFIX_RE = /-(\d+)$/ - -function prompt(question) { - const rl = readline.createInterface({ input: process.stdin, output: process.stdout }) - return new Promise((resolve) => { - rl.question(question, (answer) => { - rl.close() - resolve(answer.trim().toLowerCase()) - }) - }) -} - -function updateOrderYaml(fromPath, toPath) { - const dir = path.dirname(fromPath) - const orderFile = path.join(dir, '_order.yaml') - if (!fs.existsSync(orderFile)) return - - const oldSlug = path.basename(fromPath).replace(/\.(md|mdx)$/, '') - const newSlug = path.basename(toPath).replace(/\.(md|mdx)$/, '') - - const content = fs.readFileSync(orderFile, 'utf-8') - const updated = content.replace(new RegExp(`^(- )${oldSlug.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`, 'm'), `$1${newSlug}`) - - if (updated !== content) { - fs.writeFileSync(orderFile, updated) - } -} - -export async function validateAll(files, gitRoot, { fix, nonInteractive } = {}) { - const results = [] - const renames = [] - - // Collect all slugs (filenames without ext) and directory names across the repo. - const allSlugs = new Set() - const allDirs = new Set() - for (const relPath of files) { - allSlugs.add(path.basename(relPath).replace(/\.(md|mdx)$/, '')) - const parts = relPath.split('/') - for (let i = 0; i < parts.length - 1; i++) { - allDirs.add(parts[i]) - } - } - - // Check files: if slug ends with -N and the base slug doesn't exist anywhere, warn. - for (const relPath of files) { - const slug = path.basename(relPath).replace(/\.(md|mdx)$/, '') - const match = slug.match(SUFFIX_RE) - if (!match) continue - - const baseSlug = slug.slice(0, -match[0].length) - if (!allSlugs.has(baseSlug)) { - const ext = path.extname(relPath) - const from = path.join(gitRoot, relPath) - const toRel = path.join(path.dirname(relPath), `${baseSlug}${ext}`) - const to = path.join(gitRoot, toRel) - - results.push({ - file: relPath, - rule: name, - severity: 'warning', - fixable: true, - message: `Unnecessary suffix: "${slug}${ext}" should be renamed to "${baseSlug}${ext}"`, - }) - renames.push({ from, to, label: `${relPath} → ${toRel}` }) - } - } - - // Check directories: if a dir name ends with -N and the base dir doesn't exist, warn. - const warnedDirs = new Set() - for (const relPath of files) { - const parts = relPath.split('/') - for (let i = 0; i < parts.length - 1; i++) { - const dirName = parts[i] - if (warnedDirs.has(dirName)) continue - - const match = dirName.match(SUFFIX_RE) - if (!match) continue - - const baseName = dirName.slice(0, -match[0].length) - if (!allDirs.has(baseName)) { - warnedDirs.add(dirName) - const dirPath = parts.slice(0, i + 1).join('/') - const baseDirPath = [...parts.slice(0, i), baseName].join('/') - const from = path.join(gitRoot, dirPath) - const to = path.join(gitRoot, baseDirPath) - - results.push({ - file: dirPath, - rule: name, - severity: 'warning', - fixable: true, - message: `Unnecessary suffix: "${dirName}" folder should be renamed to "${baseName}"`, - }) - renames.push({ from, to, label: `${dirPath}/ → ${baseDirPath}/` }) - } - } - } - - // Interactive rename when --fix is passed. - if (fix && renames.length > 0) { - console.log() - console.log(` The following will be renamed:`) - for (const r of renames) { - console.log(` ${styles.dim(r.label)}`) - } - console.log() - console.log(` ${styles.warn('Note:')} Renaming changes slugs, which could break existing URLs.`) - console.log() - - // assume yes if non-interactive, otherwise prompt for confirmation - const answer = nonInteractive ? 'yes' : await prompt(` Rename ${renames.length} ${renames.length === 1 ? 'path' : 'paths'}? (y/N) `) - - if (answer === 'y' || answer === 'yes') { - // Sort longest path first so nested dirs get renamed before parents. - renames.sort((a, b) => b.from.length - a.from.length) - for (const r of renames) { - fs.renameSync(r.from, r.to) - updateOrderYaml(r.from, r.to) - } - for (const r of results) { - r.message += ' (fixed)' - } - } - } - - return results.length > 0 ? results : null -} diff --git a/vendor/TOOLS.md b/vendor/TOOLS.md index 0005eff..de2200b 100644 --- a/vendor/TOOLS.md +++ b/vendor/TOOLS.md @@ -4,7 +4,7 @@ The ReadMe CLI (`npx @readme/cli`) has several commands that can help fix issues ## `npx @readme/cli lint --fix` -Automatically fixes common linting issues: ordering problems, frontmatter cleanup, numbering suffixes, etc. Always try this first. +Automatically fixes common linting issues: ordering problems, frontmatter cleanup, etc. Always try this first. ## `npx @readme/cli oas:sync` From 901bb9ae54f8578fa5f5716d65783e10ed922abb Mon Sep 17 00:00:00 2001 From: Xavier Andueza <88118725+xavierandueza@users.noreply.github.com> Date: Tue, 23 Jun 2026 00:08:13 +1000 Subject: [PATCH 24/43] feature(import): RM-17083 if user inputs homepage then traverse well known routes to find docs sites (#14) * feat: traverse well known routes for bare domains (get docs site from marketing site) * fix: handle case of redirects better * chore: better logging for attempts * feat: record which files come from llms.txt so we know to grab the .md version --- src/commands/import.js | 175 ++++++++++++++++++++++++++++++++++------- 1 file changed, 148 insertions(+), 27 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index ae5be3b..e9ebe57 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -40,6 +40,14 @@ export function args(cmd) { cmd.addOption(new Option('--debug').hideHelp()) } +// Docs locations probed when the user hands us a bare homepage. Each is tried +// as a subdomain swap (docs.example.com) and, unless subdomain-only, a path +// prefix (example.com/docs/). Order is preference order — earlier wins ties. +const WELL_KNOWN_DOC_ROUTES = ['docs', 'doc', 'developers', 'developer', 'documentation', 'documentations', 'platform'] +// Routes probed only as a subdomain (platform.example.com). Their path form +// (example.com/platform/) is usually a marketing/product page, not docs. +const SUBDOMAIN_ONLY_DOC_ROUTES = new Set(['platform']) + /** * Run the importer programmatically. Mirrors the CLI command but throws on * fatal errors instead of calling `process.exit`, and returns a result object @@ -126,6 +134,11 @@ export async function importDocs(options) { } } + const llmsPaths = new Set() + for (const o of perSourceOrganized) { + for (const p of o.llmsPaths || []) llmsPaths.add(p) + } + if (debugSnapshots) { debugSnapshots['05-organized.json'] = organized const debugDir = path.join(os.tmpdir(), `readme-import-debug-${hostnameJoined}-${Date.now()}`) @@ -152,7 +165,7 @@ export async function importDocs(options) { styles.info(`Staging frontmatter stubs in ${styles.bold(stagingDir)}...`) const stageStart = Date.now() const staged = await timePhase('stage stubs', async () => - stageOrganized(organized, stagingDir, { skipApiReference: !!options.skipApiReference }), + stageOrganized(organized, stagingDir, { skipApiReference: !!options.skipApiReference, llmsPaths }), ) // Changelog pages stage into docs/Changelog/ (a schema-valid docs @@ -231,6 +244,16 @@ export async function importDocs(options) { * the hostname so parallel runs don't clobber each other. */ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSnapshots) { + // Probe well-known docs routes (docs./developer. subdomains and /docs/, + // … path prefixes) and adopt the first that resolves as the real base, so everything downstream runs + const docsBase = await timePhase('probe well-known docs routes', () => resolveDocsBaseUrl(sourceUrl)) + if (docsBase) { + styles.info( + `${styles.bold(sourceUrl.toString())} looks like a homepage — switching to docs route ${styles.bold(docsBase.url.toString())}${styles.dim(docsBase.hasLlms ? ' (llms.txt found)' : ' (no llms.txt; using for scrape fallback)')}.`, + ) + sourceUrl = docsBase.url + } + // Discover llms.txt files by BFS walking parents AND children. Seed the // frontier with the walk-up paths (source → root). Any "hit" (file exists // with at least one link row) expands the frontier: parent path + distinct @@ -563,9 +586,9 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna debugSnapshots[`04-after-orphan-buckets${dbgSuffix}.json`] = { apiReferenceCollected: apiResult.category ? { - pageCount: apiResult.category.pages.length, - mergedFromScraped: apiResult.mergedScrapedTitles, - } + pageCount: apiResult.category.pages.length, + mergedFromScraped: apiResult.mergedScrapedTitles, + } : null, buckets, scraped: JSON.parse(JSON.stringify(scraped)), @@ -699,6 +722,15 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna styles.ok(`Organized in ${styles.bold(formatDuration(Date.now() - organizeStart))}.`) organized.oasJsonUrls = oasJsonUrls + + // Record which page URLs came from llms.txt + const llmsPaths = new Set() + if (llms) { + for (const p of [...knownUrls, ...extractedChangelog]) { + if (p.url) llmsPaths.add(normalizePath(p.url)) + } + } + organized.llmsPaths = llmsPaths return organized } @@ -1164,7 +1196,7 @@ async function tryMintlifyNav(sourceUrl, knownPages, firecrawlKey) { function extractMintlifyConfig(body) { try { return JSON.parse(body) - } catch {} + } catch { } // Pull out the first balanced `{ ... }` that contains a "navigation" key. const navIdx = body.indexOf('"navigation"') if (navIdx === -1) return null @@ -1175,7 +1207,7 @@ function extractMintlifyConfig(body) { try { const parsed = JSON.parse(candidate) if (parsed && parsed.navigation) return parsed - } catch {} + } catch { } } start = body.lastIndexOf('{', start - 1) } @@ -1533,9 +1565,9 @@ async function scrapeNavFromSite(sourceUrl, knownPages, firecrawlKey) { const r1Urls = isDiscovery ? flattenTree(categoryOrder).slice(0, MAX_DISCOVERY_FETCHES) : categoryOrder - .map((c) => c.pages[0]) - .filter(Boolean) - .map((p) => toBrowsableUrl(p.url)) + .map((c) => c.pages[0]) + .filter(Boolean) + .map((p) => toBrowsableUrl(p.url)) const r1Start = Date.now() // Firecrawl standard-plan concurrency is 10; 5 leaves headroom for retries. // Native HTTP can run hotter since we're hitting our own loopback. @@ -1793,7 +1825,7 @@ function collectApiReferencePages(orphans, scraped) { let segs = [] try { segs = new URL(p.url).pathname.split('/').filter(Boolean) - } catch {} + } catch { } // segs[0] is the api-prefix itself; segs[1] (if any) is the resource. const resource = segs.length >= 3 ? segs[1] : null if (!resource) { @@ -2607,7 +2639,7 @@ function clusterByUrlPath(pages) { let segs = [] try { segs = new URL(p.url).pathname.split('/').filter(Boolean) - } catch {} + } catch { } if (segs.length === 0) { noSegPages.push(p) } else { @@ -2985,9 +3017,9 @@ async function runJsonQuery({ systemPrompt, userPrompt, model }) { } catch (e) { throw new Error( `Claude returned invalid JSON: ${e.message}\n` + - `Output length: ${stripped.length} chars. Likely hit the model's output limit — try --model sonnet.\n\n` + - `First 500 chars:\n${stripped.slice(0, 500)}\n\n` + - `Last 500 chars:\n${stripped.slice(-500)}`, + `Output length: ${stripped.length} chars. Likely hit the model's output limit — try --model sonnet.\n\n` + + `First 500 chars:\n${stripped.slice(0, 500)}\n\n` + + `Last 500 chars:\n${stripped.slice(-500)}`, ) } } @@ -3188,25 +3220,109 @@ function dropAssetItemsFromParsed(parsed) { return dropped } -function buildLlmsCandidates(sourceUrl) { + +/** + * Build the ordered list of well-known docs locations to probe. For each route + * we emit a subdomain candidate (stripping a leading `www.` first) and, unless + * the route is subdomain-only, a path candidate. Base URLs end in `/` so + * `${url.href}llms.txt` is well-formed. + */ +function buildWellKnownDocRoutes(sourceUrl) { const out = [] const seen = new Set() - const add = (url) => { - if (!seen.has(url)) { - seen.add(url) - out.push(url) - } + const add = (kind, href) => { + if (seen.has(href)) return + seen.add(href) + out.push({ kind, url: new URL(href) }) } - const origin = sourceUrl.origin - const segs = sourceUrl.pathname.split('/').filter(Boolean) - for (let i = segs.length; i >= 0; i--) { - const prefix = segs.slice(0, i).join('/') - add(`${origin}${prefix ? '/' + prefix : ''}/llms.txt`) + const apexHost = sourceUrl.hostname.replace(/^www\./i, '') + for (const route of WELL_KNOWN_DOC_ROUTES) { + const subHost = `${route}.${apexHost}` + if (subHost !== sourceUrl.hostname) add('subdomain', `${sourceUrl.protocol}//${subHost}/`) + if (!SUBDOMAIN_ONLY_DOC_ROUTES.has(route)) add('path', `${sourceUrl.origin}/${route}/`) } return out } +/** + * For a bare homepage, probe well-known docs routes and return the best base to + * adopt, or null to leave the source untouched. A route serving a usable + * llms.txt always beats one that merely resolves. Returns `{ url, kind, hasLlms }`. + */ +async function resolveDocsBaseUrl(sourceUrl) { + if (sourceUrl.pathname && sourceUrl.pathname !== '/') return null + if (WELL_KNOWN_DOC_ROUTES.includes(sourceUrl.hostname.split('.')[0].toLowerCase())) return null + + const candidates = buildWellKnownDocRoutes(sourceUrl) + if (candidates.length === 0) return null + + styles.info(`Probing ${styles.bold(String(candidates.length))} well-known docs routes from ${styles.bold(sourceUrl.toString())}...`) + + const llmsHits = await Promise.all(candidates.map(async (c) => ({ c, res: await fetchLlmsTxt(`${c.url.href}llms.txt`) }))) + for (const { c, res } of llmsHits) { + const url = `${c.url.href}llms.txt` + if (res.ok && res.usable && (res.stats?.linkItems ?? 0) > 0) { + if (inCandidateScope(c, res.finalUrl)) { + styles.info(styles.dim(` ${url} → valid llms.txt (${res.stats.linkItems} links)`)) + return { url: c.url, kind: c.kind, hasLlms: true } + } + styles.info(styles.dim(` ${url} → llms.txt redirects to ${new URL(res.finalUrl).host} (alias) — skipping`)) + } else if (res.ok) { + styles.info(styles.dim(` ${url} → no usable llms.txt`)) + } else { + styles.info(styles.dim(` ${url} → ${res.status ? `HTTP ${res.status}` : res.error || 'no llms.txt'}`)) + } + } + + const existence = await Promise.all( + llmsHits.map(async ({ c, res }) => { + if (res.error && c.kind === 'subdomain') return { c, exists: false } + return { c, exists: await docsRouteResolves(c) } + }), + ) + for (const { c, exists } of existence) { + if (exists) { + styles.info(styles.dim(` ${c.url.href} → resolves (no llms.txt)`)) + return { url: c.url, kind: c.kind, hasLlms: false } + } + } + + styles.info(styles.dim(` no well-known docs route matched — keeping ${sourceUrl.toString()}`)) + return null +} + +/** + * Is `finalUrl` (where a fetch landed after following redirects) still within + * the candidate's scope? Subdomain candidates must keep their host; path + * candidates must keep their path prefix. A cross-scope redirect means the + * route is just an alias to somewhere else (usually the marketing apex). + */ +function inCandidateScope(candidate, finalUrl) { + let final + try { + final = new URL(finalUrl || candidate.url.href) + } catch { + return false + } + if (candidate.kind === 'subdomain') return final.hostname === candidate.url.hostname + return final.pathname.toLowerCase().startsWith(candidate.url.pathname.toLowerCase()) +} + +/** + * Does a well-known docs route exist and stay in scope? Follows redirects and + * rejects anything that lands outside the candidate (a `docs.` subdomain that + * bounces to marketing, a `/docs/` that 302s home). DNS failures → false. + */ +async function docsRouteResolves(candidate) { + try { + const res = await fetch(candidate.url.href, { redirect: 'follow', headers: { 'User-Agent': 'readme-cli-import' } }) + return res.ok && inCandidateScope(candidate, res.url) + } catch { + return false + } +} + // Hard cap on total probes during BFS discovery — protects against // pathological sites where every URL returns a "200 + a few link rows" // response and the frontier would otherwise blow up. @@ -3490,6 +3606,7 @@ async function fetchLlmsTxt(llmsUrl) { return { ok: true, status: res.status, + finalUrl: res.url, parsed: analysis.parsed, usable: analysis.usable, reason: analysis.reason, @@ -3727,6 +3844,7 @@ function stageOrganized(organized, stagingDir, opts = {}) { const subDirsByTopDir = new Map() const counts = { fileCount: 0, skippedApiRef: 0 } const skipApiReference = !!opts.skipApiReference + const llmsPaths = opts.llmsPaths || new Set() const eligibleCategories = (organized.categories || []).filter((cat) => { if (skipApiReference && routeCategory(cat.title).topDir === 'reference') { @@ -3785,6 +3903,9 @@ function stageOrganized(organized, stagingDir, opts = {}) { // step reads it to fetch the page body. x-prefixed custom field is the // git-format convention for metadata the schema doesn't know about. frontmatter['x-import'] = toBrowsableUrl(page.url) + // Pages sourced from llms.txt serve raw markdown at `.md`; flag them + // so the runner knows it can append `.md` when fetching the body. + if (llmsPaths.has(normalizePath(page.url))) frontmatter['x-from-llms'] = 'true' // hide pages that need import frontmatter.hidden = 'true' } @@ -4077,8 +4198,8 @@ function ensureUniqueSlugs(categories) { slug = `${base}-${n}` styles.error( `Slug collision after segment expansion: ${base} — falling back to ${slug} for ${describe(e)}. ` + - `Pages were deduped by path before organize and category title is already part of the slug, ` + - `so this indicates the organize step produced duplicates within a single category.`, + `Pages were deduped by path before organize and category title is already part of the slug, ` + + `so this indicates the organize step produced duplicates within a single category.`, ) } used.add(slug) From 26d0316ca57c45faab2ca4d3160868039d0db5e0 Mon Sep 17 00:00:00 2001 From: minh <150941282+minhthanhdang@users.noreply.github.com> Date: Wed, 24 Jun 2026 20:23:39 +1000 Subject: [PATCH 25/43] fix (#16) --- src/utils/llms.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/utils/llms.js b/src/utils/llms.js index ca60531..a3f889b 100644 --- a/src/utils/llms.js +++ b/src/utils/llms.js @@ -6,8 +6,11 @@ const H2_RE = /^##\s+(.+)$/ // - Breadcrumb-prefixed rows used by Fern/Mintlify-style indices: // `ElevenAgents [Agent WebSockets](url)` // `API Reference > Agents > Branches [List branches](url)` +// - Trailing-parenthetical-breadcrumb rows (AssemblyAI/Fern variant) — the +// breadcrumb sits between the url and the description and is matched-and-discarded: +// `[text](url) (Breadcrumb > path): description` // Captures: [prefix, text, url, description] (prefix and description may be empty). -const LINK_LINE_RE = /^\s*(?:[-*+]\s+)?([^\[\n]*?)\s*\[([^\]]+)\] ?\(([^)\s]+)\)(?:\s*[:—–-]\s*(.+?))?\s*$/ +const LINK_LINE_RE = /^\s*(?:[-*+]\s+)?([^\[\n]*?)\s*\[([^\]]+)\] ?\(([^)\s]+)\)(?:\s+\([^)]*\))?(?:\s*[:—–-]\s*(.+?))?\s*$/ const BLOCKQUOTE_RE = /^\s*>/ const FENCE_RE = /^\s*(?:```|~~~)/ From 8c1cbb1cd272a0bd8c7be0eacb1ea53165aa3b52 Mon Sep 17 00:00:00 2001 From: minh <150941282+minhthanhdang@users.noreply.github.com> Date: Wed, 24 Jun 2026 20:24:07 +1000 Subject: [PATCH 26/43] fix (#17) --- src/commands/import.js | 106 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/src/commands/import.js b/src/commands/import.js index e9ebe57..957cf45 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -47,6 +47,9 @@ const WELL_KNOWN_DOC_ROUTES = ['docs', 'doc', 'developers', 'developer', 'docume // Routes probed only as a subdomain (platform.example.com). Their path form // (example.com/platform/) is usually a marketing/product page, not docs. const SUBDOMAIN_ONLY_DOC_ROUTES = new Set(['platform']) +// Max concurrent HTTP probes when checking whether a section's landing URL +// (e.g. /docs/cli) really exists before importing it as a real overview page. +const LANDING_PROBE_CAP = 8 /** * Run the importer programmatically. Mirrors the CLI command but throws on @@ -716,6 +719,12 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna // Changelog category (scrape path's reclassifier may have already built one). injectChangelogCategory(organized, extractedChangelog) + // Import section landing pages (e.g. /docs/cli) that llms.txt models as a + // bare heading + child links and so never lists as a page of their own. + // Runs before nesting so the injected page lands at its trie node and + // nestByUrlHierarchy renders it as the real parent of its children. + await injectSectionLandingPages(organized, sourceUrl) + for (const cat of organized.categories || []) { cat.pages = nestByUrlHierarchy(cat.pages) } @@ -2723,6 +2732,96 @@ function clusterByUrlPath(pages) { return out } +/** + * Import section landing pages that the source models structurally rather than + * as a link. In llms.txt a section like the Vercel CLI is a bare `## CLI` + * heading followed by 68 `/docs/cli/` links — the importer turns the + * heading into a category but `/docs/cli` itself is never listed, so its real, + * content-bearing overview page is dropped. ReadMe then redirects the + * content-less category to its first child (a broken landing page). + * + * For each category we compute the common URL prefix of its children (the + * section's landing URL), HTTP-probe it, and — when it really exists and stays + * in scope — prepend it as a real page so nestByUrlHierarchy renders it as the + * parent holding the children. Mutates `organized` in place. + */ +async function injectSectionLandingPages(organized, sourceUrl) { + const cats = organized?.categories || [] + if (cats.length === 0) return + + // Dedup set: every page URL already in the tree, by normalized path. + const known = new Set() + for (const c of cats) { + for (const p of collectUrlPagesDeep(c.pages || [])) known.add(normalizePath(p.url)) + } + + // Base path of the import itself — landing candidates must be strictly + // deeper than this so we never probe `/docs` (the base) as a "section". + const baseDepth = (urlTrieSegs(sourceUrl) || []).length + const origin = new URL(sourceUrl).origin + + // Build one landing candidate per eligible category (cheap gate first). + const candidates = [] + for (const cat of cats) { + const kids = collectUrlPagesDeep(cat.pages || []) + if (kids.length < 2) continue + + // Longest common path prefix across the children — same loop as + // clusterByUrlPath's common-depth scan, over urlTrieSegs. + const parts = kids.map((p) => urlTrieSegs(p.url)).filter(Boolean) + if (parts.length < 2) continue + let commonDepth = 0 + while (commonDepth < parts[0].length) { + const seg = parts[0][commonDepth] + if (!parts.every((pp) => pp[commonDepth] === seg)) break + commonDepth++ + } + const commonSegs = parts[0].slice(0, commonDepth) + + // Must be strictly deeper than the import base, and not already a page. + if (commonSegs.length <= baseDepth) continue + const landingUrl = `${origin}/${commonSegs.join('/')}` + if (known.has(normalizePath(landingUrl))) continue + + candidates.push({ cat, landingUrl }) + } + if (candidates.length === 0) return + + // Probe surviving candidates in parallel. Accept only a real, in-scope page + // that doesn't redirect onto an already-known child. + const accepted = [] + await visitAllInParallel( + candidates, + async ({ cat, landingUrl }) => { + try { + const res = await fetch(landingUrl, { + redirect: 'follow', + headers: { 'User-Agent': 'readme-cli-import' }, + }) + if (!res.ok) return + const candPath = new URL(landingUrl).pathname.toLowerCase() + const finalPath = new URL(res.url).pathname.toLowerCase() + if (!finalPath.startsWith(candPath)) return + if (known.has(normalizePath(res.url))) return + accepted.push({ cat, landingUrl }) + } catch { + // Network error / unparseable — treat as "no landing page". + } + }, + LANDING_PROBE_CAP, + ) + + for (const { cat, landingUrl } of accepted) { + // Don't set x-from-llms: the page isn't in llmsPaths, so the runner fetches + // its rendered HTML via the content cascade (there's no .md mirror). + cat.pages.unshift({ title: cat.title, url: landingUrl }) + known.add(normalizePath(landingUrl)) + } + if (accepted.length > 0) { + styles.info(`Imported ${styles.bold(String(accepted.length))} section landing page(s).`) + } +} + /** * Used by discovery-mode scraping (no llms.txt) to decide whether a nav * link is worth importing as a doc page. Filters out cross-origin links, @@ -4147,7 +4246,12 @@ function ensureUniqueSlugs(categories) { for (const p of pages || []) { const urlSegs = segmentsFor(p) const segments = categorySeg ? [categorySeg, ...urlSegs] : urlSegs - entries.push({ page: p, segments, fallback: p.title, depth: 1 }) + // Synthetic/empty-parent nodes have no content. Never let one win a bare + // single-segment slug (e.g. a placeholder `…/vercel-flags/cli` claiming + // `cli`), or it squats the slug a real overview page wants and ReadMe + // redirects the content-less parent to its first child. Floor their slug + // at 2 trailing segments so they namespace to `vercel-flags-cli` instead. + entries.push({ page: p, segments, fallback: p.title, depth: p._emptyParent ? 2 : 1 }) if (p.pages) walk(p.pages, categorySeg) } } From 1de0cbbaa1e883b79f82a9b6776b69efff4e6535 Mon Sep 17 00:00:00 2001 From: Xavier Andueza <88118725+xavierandueza@users.noreply.github.com> Date: Wed, 24 Jun 2026 20:33:10 +1000 Subject: [PATCH 27/43] fix: strip double extensions (.html.md) from URL slugs (#18) --- package.json | 2 +- src/commands/import.js | 63 ++-------------- src/utils/url-segs.js | 72 ++++++++++++++++++ src/utils/url-segs.test.js | 148 +++++++++++++++++++++++++++++++++++++ 4 files changed, 227 insertions(+), 58 deletions(-) create mode 100644 src/utils/url-segs.js create mode 100644 src/utils/url-segs.test.js diff --git a/package.json b/package.json index 8501e9d..6ef5d72 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ ], "scripts": { "start": "node bin/readme.js", - "test": "echo \"No tests yet\"" + "test": "node --test src/**/*.test.js" }, "engines": { "node": ">=18" diff --git a/src/commands/import.js b/src/commands/import.js index 957cf45..cc47958 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -11,6 +11,7 @@ import { syncOas } from './oas-sync.js' import OASNormalize from 'oas-normalize' import { slotOrphansPrompt, iconizeNavPrompt, organizeFromSectionsPrompt, organizeFromScratchPrompt, stripCodeFences } from '../prompts/index.js' import { analyzeLlmsTxt } from '../utils/llms.js' +import { urlTrieSegs, extractUrlPathSegments, normalizePath, stripSegmentExtensions } from '../utils/url-segs.js' export const command = 'import' export const order = 7 @@ -2199,26 +2200,7 @@ function collectUrlPagesDeep(pages, out = []) { return out } -/** - * Parse a URL's pathname into segments suitable for trie nesting. Strips file - * extensions (`.md`/`.html`/etc.) and drops a trailing `index` segment so - * `/foo`, `/foo/`, `/foo/index.html`, and `/foo/index.md` all collapse to the - * same logical page — the static-site-generator convention every browser and - * web server already follows. Returns null if the URL can't be parsed. - */ -function urlTrieSegs(url) { - try { - const segs = new URL(url).pathname - .split('/') - .filter(Boolean) - .map((s) => s.replace(/\.(md|mdx|html?)$/i, '')) - .filter(Boolean) - if (segs.length > 0 && segs[segs.length - 1].toLowerCase() === 'index') segs.pop() - return segs - } catch { - return null - } -} +// urlTrieSegs is imported from ../utils/url-segs.js /** * Re-parent URL-bearing siblings by URL path so descendants nest under their @@ -2329,7 +2311,7 @@ function nestByUrlHierarchy(pages, anchorSegs = null) { outPage = node.page } else { const rawSeg = node.segment - const cleanedSeg = rawSeg.replace(/\.(md|mdx|html?)$/i, '').replace(/^\d+[-_.]/, '') || rawSeg + const cleanedSeg = stripSegmentExtensions(rawSeg).replace(/^\d+[-_.]/, '') || rawSeg outPage = { title: titleCase(cleanedSeg), _emptyParent: true, @@ -2881,20 +2863,7 @@ function decodeEntities(s) { .replace(/”/g, '”') } -/** - * Reduce a URL to a comparable pathname: lowercase host, strip trailing slash - * and common suffixes (.md, .html) so `/foo/bar.md` and `/foo/bar` match. - */ -function normalizePath(url) { - try { - const u = new URL(url) - let p = u.pathname.replace(/\/$/, '').toLowerCase() - p = p.replace(/\.(md|mdx|html?)$/i, '') - return p - } catch { - return String(url).toLowerCase() - } -} +// normalizePath is imported from ../utils/url-segs.js /** * Given a set of scraped categories + orphan pages that didn't match the nav @@ -4183,27 +4152,7 @@ function buildFrontmatter(topDir, page, slug, pickIcon, opts = {}) { } /** - * Extract URL path segments for slug planning. Strips file extensions and - * leading numeric prefixes (e.g. `01-intro` → `intro`) the same way the - * legacy deriveSlug did, so the depth-1 result is byte-for-byte compatible. - * Also drops a trailing `index` segment — many SSGs render `/foo/` as - * `/foo/index.html` and emit either form in their URL lists; we don't want - * every page collapsing to the same `index` base slug. - */ -function extractUrlPathSegments(url) { - if (!url) return [] - try { - const segs = new URL(url).pathname - .split('/') - .filter(Boolean) - .map((s) => s.replace(/\.(md|mdx|html?)$/i, '').replace(/^\d+[-_.]/, '')) - .filter(Boolean) - if (segs.length > 0 && segs[segs.length - 1].toLowerCase() === 'index') segs.pop() - return segs - } catch { - return [] - } -} +// extractUrlPathSegments is imported from ../utils/url-segs.js /** * Compute a globally-unique slug for every page in the tree. @@ -4237,7 +4186,7 @@ function ensureUniqueSlugs(categories) { const segmentsFor = (p) => { if (p._virtualPathSegs && p._virtualPathSegs.length > 0) { return p._virtualPathSegs - .map((s) => s.replace(/\.(md|mdx|html?)$/i, '').replace(/^\d+[-_.]/, '')) + .map((s) => stripSegmentExtensions(s).replace(/^\d+[-_.]/, '')) .filter(Boolean) } return extractUrlPathSegments(p.url) diff --git a/src/utils/url-segs.js b/src/utils/url-segs.js new file mode 100644 index 0000000..3bd82cf --- /dev/null +++ b/src/utils/url-segs.js @@ -0,0 +1,72 @@ +/** + * URL segment parsing utilities shared across import.js pipeline stages. + * Centralised here so the single-extension-strip bug only lives in one place. + */ + +/** + * Strip all chained doc-file extensions (.html.md, .html, .md, .mdx, .htm) + * from a single URL path segment. Handles the double-extension pattern used + * by some SSGs (e.g. plaid.com's llms.txt links end with `.html.md`). + */ +export function stripSegmentExtensions(s) { + return s.replace(/(\.(md|mdx|html?))+$/i, '') +} + +/** + * Parse a URL's pathname into segments suitable for trie nesting. Strips file + * extensions (including double-extensions like `.html.md`) and drops a trailing + * `index` segment so `/foo`, `/foo/`, `/foo/index.html`, `/foo/index.html.md`, + * and `/foo/index.md` all collapse to the same logical page. + * Returns null if the URL can't be parsed. + */ +export function urlTrieSegs(url) { + try { + const segs = new URL(url).pathname + .split('/') + .filter(Boolean) + .map((s) => stripSegmentExtensions(s)) + .filter(Boolean) + if (segs.length > 0 && segs[segs.length - 1].toLowerCase() === 'index') segs.pop() + return segs + } catch { + return null + } +} + +/** + * Extract URL path segments for slug planning. Strips file extensions + * (including double-extensions like `.html.md`) and leading numeric prefixes + * (e.g. `01-intro` → `intro`). Also drops a trailing `index` segment so + * `/foo/index.html.md` and `/foo/` both collapse to the same slug base. + */ +export function extractUrlPathSegments(url) { + if (!url) return [] + try { + const segs = new URL(url).pathname + .split('/') + .filter(Boolean) + .map((s) => stripSegmentExtensions(s).replace(/^\d+[-_.]/, '')) + .filter(Boolean) + if (segs.length > 0 && segs[segs.length - 1].toLowerCase() === 'index') segs.pop() + return segs + } catch { + return [] + } +} + +/** + * Normalise a URL to a canonical path key for deduplication. Strips trailing + * slash, strips all chained doc-file extensions from the path tail, and + * lowercases — so `/foo/bar.html.md`, `/foo/bar.html`, and `/foo/bar` all + * resolve to the same key `/foo/bar`. + */ +export function normalizePath(url) { + try { + const u = new URL(url) + let p = u.pathname.replace(/\/$/, '').toLowerCase() + p = p.replace(/(\.(md|mdx|html?))+$/i, '') + return p + } catch { + return String(url).toLowerCase() + } +} diff --git a/src/utils/url-segs.test.js b/src/utils/url-segs.test.js new file mode 100644 index 0000000..6126cec --- /dev/null +++ b/src/utils/url-segs.test.js @@ -0,0 +1,148 @@ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { stripSegmentExtensions, urlTrieSegs, extractUrlPathSegments, normalizePath } from './url-segs.js' + +// --------------------------------------------------------------------------- +// stripSegmentExtensions +// --------------------------------------------------------------------------- + +test('stripSegmentExtensions: strips single .md', () => { + assert.equal(stripSegmentExtensions('index.md'), 'index') +}) + +test('stripSegmentExtensions: strips single .html', () => { + assert.equal(stripSegmentExtensions('index.html'), 'index') +}) + +test('stripSegmentExtensions: strips single .htm', () => { + assert.equal(stripSegmentExtensions('index.htm'), 'index') +}) + +test('stripSegmentExtensions: strips double .html.md (plaid pattern)', () => { + assert.equal(stripSegmentExtensions('index.html.md'), 'index') +}) + +test('stripSegmentExtensions: strips .mdx', () => { + assert.equal(stripSegmentExtensions('page.mdx'), 'page') +}) + +test('stripSegmentExtensions: leaves plain segments untouched', () => { + assert.equal(stripSegmentExtensions('quickstart'), 'quickstart') +}) + +test('stripSegmentExtensions: leaves kebab segments untouched', () => { + assert.equal(stripSegmentExtensions('getting-started'), 'getting-started') +}) + +// --------------------------------------------------------------------------- +// urlTrieSegs — the double-extension plaid.com pattern +// --------------------------------------------------------------------------- + +test('urlTrieSegs: .html.md index collapses to parent path (plaid pattern)', () => { + assert.deepEqual( + urlTrieSegs('https://plaid.com/docs/account/index.html.md'), + ['docs', 'account'], + ) +}) + +test('urlTrieSegs: .html index collapses to parent path', () => { + assert.deepEqual( + urlTrieSegs('https://plaid.com/docs/account/index.html'), + ['docs', 'account'], + ) +}) + +test('urlTrieSegs: trailing slash collapses correctly', () => { + assert.deepEqual( + urlTrieSegs('https://plaid.com/docs/account/'), + ['docs', 'account'], + ) +}) + +test('urlTrieSegs: non-index .html.md page keeps its segment', () => { + assert.deepEqual( + urlTrieSegs('https://plaid.com/docs/account/billing/index.html.md'), + ['docs', 'account', 'billing'], + ) +}) + +test('urlTrieSegs: orval.dev docs/index.html.md collapses correctly', () => { + assert.deepEqual( + urlTrieSegs('https://orval.dev/docs/index.html.md'), + ['docs'], + ) +}) + +test('urlTrieSegs: plain URL without extension passes through', () => { + assert.deepEqual( + urlTrieSegs('https://example.com/docs/quickstart'), + ['docs', 'quickstart'], + ) +}) + +test('urlTrieSegs: returns null on unparseable input', () => { + assert.equal(urlTrieSegs('not-a-url'), null) +}) + +// --------------------------------------------------------------------------- +// extractUrlPathSegments — slug source +// --------------------------------------------------------------------------- + +test('extractUrlPathSegments: .html.md index → parent segments only (plaid pattern)', () => { + assert.deepEqual( + extractUrlPathSegments('https://plaid.com/docs/account/index.html.md'), + ['docs', 'account'], + ) +}) + +test('extractUrlPathSegments: slug from deepened path', () => { + assert.deepEqual( + extractUrlPathSegments('https://plaid.com/docs/account/billing/index.html.md'), + ['docs', 'account', 'billing'], + ) +}) + +test('extractUrlPathSegments: strips leading numeric prefix', () => { + assert.deepEqual( + extractUrlPathSegments('https://example.com/docs/01-intro.md'), + ['docs', 'intro'], + ) +}) + +test('extractUrlPathSegments: plain .md page', () => { + assert.deepEqual( + extractUrlPathSegments('https://example.com/docs/quickstart.md'), + ['docs', 'quickstart'], + ) +}) + +test('extractUrlPathSegments: returns [] for empty/null input', () => { + assert.deepEqual(extractUrlPathSegments(''), []) + assert.deepEqual(extractUrlPathSegments(null), []) +}) + +// --------------------------------------------------------------------------- +// normalizePath — deduplication keys +// --------------------------------------------------------------------------- + +test('normalizePath: .html.md and .html and bare path all produce same key', () => { + const a = normalizePath('https://plaid.com/docs/account/index.html.md') + const b = normalizePath('https://plaid.com/docs/account/index.html') + const c = normalizePath('https://plaid.com/docs/account/index') + assert.equal(a, b) + assert.equal(b, c) +}) + +test('normalizePath: trailing slash and bare path produce same key', () => { + const a = normalizePath('https://example.com/docs/foo/') + const b = normalizePath('https://example.com/docs/foo') + assert.equal(a, b) +}) + +test('normalizePath: lowercases result', () => { + assert.equal(normalizePath('https://example.com/Docs/QuickStart'), '/docs/quickstart') +}) + +test('normalizePath: falls back for unparseable input', () => { + assert.equal(normalizePath('not-a-url'), 'not-a-url') +}) From 792cd71f474e13349024ac0ccd10a87303983018 Mon Sep 17 00:00:00 2001 From: Xavier Andueza <88118725+xavierandueza@users.noreply.github.com> Date: Thu, 25 Jun 2026 10:45:57 +1000 Subject: [PATCH 28/43] feat: after claude organizing, remap reference nav pages into "Api Reference" folder (#19) --- src/commands/import.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/commands/import.js b/src/commands/import.js index cc47958..e8c5411 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -701,6 +701,10 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna const fastPath = sectionsLookUsable(llms.parsed.sections) styles.info(`Organizing with Claude (${styles.bold(options.model)}, ${fastPath ? 'fast path: icons only' : 'full reorg'})...`) organized = await timePhase('claude organize', () => organizeWithClaude(llms.parsed, options.model)) + const movedRef = reclassifyReferencePages(organized) + if (movedRef > 0) { + styles.info(`Moved ${styles.bold(String(movedRef))} page${movedRef === 1 ? '' : 's'} into ${styles.bold('API Reference')} based on URL path.`) + } } } } else { From 4e2e5425443e4756bb7ce4e5aa3e3df9836334ff Mon Sep 17 00:00:00 2001 From: Xavier Andueza <88118725+xavierandueza@users.noreply.github.com> Date: Tue, 30 Jun 2026 11:22:31 +1000 Subject: [PATCH 29/43] fix: strip .txt endings from slugs (#20) --- src/utils/url-segs.js | 12 ++++++++---- src/utils/url-segs.test.js | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/utils/url-segs.js b/src/utils/url-segs.js index 3bd82cf..f9be138 100644 --- a/src/utils/url-segs.js +++ b/src/utils/url-segs.js @@ -3,13 +3,17 @@ * Centralised here so the single-extension-strip bug only lives in one place. */ +// Extensions that are implementation artifacts, not meaningful path segments. +const DOC_FILE_EXTENSIONS = ['md', 'mdx', 'html', 'htm', 'txt'] +const DOC_EXT_RE = new RegExp(`(\\.(${DOC_FILE_EXTENSIONS.join('|')}))+$`, 'i') + /** - * Strip all chained doc-file extensions (.html.md, .html, .md, .mdx, .htm) + * Strip all chained doc-file extensions (.html.md, .html, .md, .mdx, .htm, .txt) * from a single URL path segment. Handles the double-extension pattern used - * by some SSGs (e.g. plaid.com's llms.txt links end with `.html.md`). + * by some SSGs (e.g. `.html.md`). */ export function stripSegmentExtensions(s) { - return s.replace(/(\.(md|mdx|html?))+$/i, '') + return s.replace(DOC_EXT_RE, '') } /** @@ -64,7 +68,7 @@ export function normalizePath(url) { try { const u = new URL(url) let p = u.pathname.replace(/\/$/, '').toLowerCase() - p = p.replace(/(\.(md|mdx|html?))+$/i, '') + p = p.replace(DOC_EXT_RE, '') return p } catch { return String(url).toLowerCase() diff --git a/src/utils/url-segs.test.js b/src/utils/url-segs.test.js index 6126cec..31f123e 100644 --- a/src/utils/url-segs.test.js +++ b/src/utils/url-segs.test.js @@ -26,6 +26,14 @@ test('stripSegmentExtensions: strips .mdx', () => { assert.equal(stripSegmentExtensions('page.mdx'), 'page') }) +test('stripSegmentExtensions: strips .txt', () => { + assert.equal(stripSegmentExtensions('terminal.txt'), 'terminal') +}) + +test('stripSegmentExtensions: leaves bare txt segment untouched (no dot prefix)', () => { + assert.equal(stripSegmentExtensions('txt'), 'txt') +}) + test('stripSegmentExtensions: leaves plain segments untouched', () => { assert.equal(stripSegmentExtensions('quickstart'), 'quickstart') }) @@ -84,6 +92,13 @@ test('urlTrieSegs: returns null on unparseable input', () => { assert.equal(urlTrieSegs('not-a-url'), null) }) +test('urlTrieSegs: .txt segment strips extension (warp.dev pattern)', () => { + assert.deepEqual( + urlTrieSegs('https://docs.warp.dev/_llms-txt/terminal.txt'), + ['_llms-txt', 'terminal'], + ) +}) + // --------------------------------------------------------------------------- // extractUrlPathSegments — slug source // --------------------------------------------------------------------------- @@ -146,3 +161,10 @@ test('normalizePath: lowercases result', () => { test('normalizePath: falls back for unparseable input', () => { assert.equal(normalizePath('not-a-url'), 'not-a-url') }) + +test('normalizePath: strips .txt extension (warp.dev pattern)', () => { + assert.equal( + normalizePath('https://docs.warp.dev/_llms-txt/terminal.txt'), + '/_llms-txt/terminal', + ) +}) From b90e61c9de6a947b55735ceb83b05f835546d332 Mon Sep 17 00:00:00 2001 From: Xavier Andueza <88118725+xavierandueza@users.noreply.github.com> Date: Tue, 30 Jun 2026 11:23:35 +1000 Subject: [PATCH 30/43] fix(import): RM-17159 AI Importer couchbase sidebar pages duplicated (#21) * fix: strip .txt endings from slugs * fix: on input de-duplicate urls, on output from claude reorganize dedupe urls * feat: better logging when we can't auto-format, fallback to h3 for weird llms.txt --- src/commands/import.js | 91 ++++++++++++++++++++--- src/utils/llms.js | 29 +++++++- src/utils/llms.test.js | 160 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 268 insertions(+), 12 deletions(-) create mode 100644 src/utils/llms.test.js diff --git a/src/commands/import.js b/src/commands/import.js index e8c5411..488ccb3 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -372,6 +372,9 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna } else { styles.info(styles.dim(`Merged ${llms.sourceFiles.length} llms.txt files (root: ${llmsUrl}; aggregate ratio ${s.ratio.toFixed(2)}).`)) } + if (llms.parsed.h3Fallback) { + styles.info(styles.dim(`H2 sections oversized — re-parsed using H3 (###) headings as section boundaries.`)) + } } // Pre-extract changelog pages so AI / URL clustering / section-direct paths @@ -397,6 +400,29 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna ) } + // Dedupe items across llms.parsed.sections by URL (first occurrence wins). + // Some llms.txt files cross-reference the same page under multiple headings — + // feeding duplicate URLs into the organize step causes the same page to appear + // in multiple sidebar sections regardless of which path (direct, icons, full + // reorg) runs downstream. + if (llms) { + const seenItemUrls = new Set() + let deduped = 0 + for (const section of llms.parsed.sections) { + const before = section.items.length + section.items = section.items.filter((item) => { + const key = normalizePath(item.url) + if (seenItemUrls.has(key)) return false + seenItemUrls.add(key) + return true + }) + deduped += before - section.items.length + } + if (deduped > 0) { + styles.info(styles.dim(`Deduped ${deduped} cross-section duplicate URL${deduped === 1 ? '' : 's'} from llms.txt sections.`)) + } + } + const dbgSuffix = `-${sourceUrl.hostname}` if (debugSnapshots) { debugSnapshots[`01-llms-parsed${dbgSuffix}.json`] = { llmsUrl, parsed: llms ? llms.parsed : null, skipped: skippedLlms } @@ -475,6 +501,7 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna let scraped let scrapeStart = Date.now() + const scrapeDiagnostics = {} if (mintlifyNav) { scraped = { title: mintlifyNav.title, categories: mintlifyNav.categories } } else if (archbeeNav) { @@ -482,7 +509,8 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna } else { styles.info(`Scraping sidebar nav from ${styles.bold(sourceUrl.toString())}${firecrawlKey ? ' ' + styles.dim('(via Firecrawl)') : ''}...`) scrapeStart = Date.now() - scraped = await timePhase('scrape nav', () => scrapeNavFromSite(sourceUrl.toString(), knownUrls, firecrawlKey)) + scraped = await timePhase('scrape nav', () => scrapeNavFromSite(sourceUrl.toString(), knownUrls, firecrawlKey, scrapeDiagnostics)) + if (!scraped) scrapeDiagnostics.reason ??= 'unknown' } if (debugSnapshots) { debugSnapshots[`02-scraped-raw${dbgSuffix}.json`] = scraped ? JSON.parse(JSON.stringify(scraped)) : null @@ -496,6 +524,7 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna // node because that's the one /docs page the scrape saw). Discard the // scrape and fall through to the llms.txt path, which uses URL-based // clustering when multiple files were merged. + let scrapeDiscardedForCoverage = false if (scraped && llms && knownUrls.length > 0) { const scrapedPages = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0) const coverage = scrapedPages / knownUrls.length @@ -504,6 +533,7 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages (need ≥75%) — discarding scrape and organizing from llms.txt.`, ) scraped = null + scrapeDiscardedForCoverage = true } } @@ -637,9 +667,31 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna } else if (!llms && knownUrls.length === 0) { throw new Error(`No llms.txt or sitemap.xml and the sidebar scrape found no usable structure — can't import ${sourceUrl.toString()}.`) } else if (llms) { - styles.warning(`Couldn't extract a useful nav — falling back to llms.txt-based organization.`) + if (!scrapeDiscardedForCoverage) { + const d = scrapeDiagnostics || {} + if (d.reason === 'no-categories-round0') { + styles.warning(`Sidebar scrape found no nav structure on the round-0 fetch — falling back to llms.txt organization.`) + } else if (d.reason === 'below-threshold') { + styles.warning( + `Sidebar scrape below acceptance threshold (got ${d.categories} categor${d.categories === 1 ? 'y' : 'ies'}, ${d.matched} matched pages — need ${d.need}) — falling back to llms.txt organization.`, + ) + } else { + styles.warning(`Sidebar scrape returned no usable nav — falling back to llms.txt organization.`) + } + } } else { - styles.warning(`Couldn't extract a useful nav — falling back to sitemap URL clustering.`) + if (!scrapeDiscardedForCoverage) { + const d = scrapeDiagnostics || {} + if (d.reason === 'no-categories-round0') { + styles.warning(`Sidebar scrape found no nav structure on the round-0 fetch — falling back to sitemap URL clustering.`) + } else if (d.reason === 'below-threshold') { + styles.warning( + `Sidebar scrape below acceptance threshold (got ${d.categories} categor${d.categories === 1 ? 'y' : 'ies'}, ${d.matched} matched pages — need ${d.need}) — falling back to sitemap URL clustering.`, + ) + } else { + styles.warning(`Sidebar scrape returned no usable nav — falling back to sitemap URL clustering.`) + } + } } console.log() @@ -1426,7 +1478,7 @@ function isMonotonicAlpha(titles) { * renders its sidebar server-side as