import { getBibField, type BibEntry, type BibFile } from '@learnitive/bibliography'; import { CANONICAL_LATEX_HARD_BREAK, CANONICAL_LATEX_PARAGRAPH_BREAK, canonicalEmptySourceAnchor, canEditLatexMathWithMathLive, findLatexMathRegions, latexMathRegionToMathLiveValue, parseLatexTabular, } from '@learnitive/latex-source'; import { projectFilePath, type ProjectFilePath, } from '@learnitive/project-storage'; export type SourceRange = { from: number; to: number }; export type SemanticKind = | 'part' | 'chapter' | 'section' | 'subsection' | 'subsubsection' | 'paragraph-heading' | 'subparagraph-heading' | 'label' | 'reference' | 'citation' | 'package' | 'documentclass' | 'input' | 'include' | 'asset' | 'caption' | 'footnote' | 'bibliography' | 'environment'; export type SemanticNode = { id: string; kind: SemanticKind; file: ProjectFilePath; range: SourceRange; line: number; name?: string; target?: string; title?: string; level?: number; bodyRange?: SourceRange; options?: string; optionsBodyRange?: SourceRange; starred?: boolean; }; export type VisualInline = | { id: string; kind: 'text'; range: SourceRange; bodyRange: SourceRange; text: string; } | { id: string; kind: 'hard-break'; range: SourceRange; sourceText: typeof CANONICAL_LATEX_HARD_BREAK | '\\newline' | '\\\\'; } | { id: string; kind: 'citation'; range: SourceRange; command: string; targets: readonly string[]; } | { id: string; kind: 'reference'; range: SourceRange; command: string; target: string; } | { id: string; kind: 'footnote'; range: SourceRange; bodyRange: SourceRange; text: string; editable: boolean; } | { id: string; kind: 'math'; range: SourceRange; bodyRange: SourceRange; text: string; mathLiveValue: string; mathKind: 'inline' | 'display'; delimiter: '$' | '$$' | '\\(' | '\\[' | 'environment'; environment?: string; editable: boolean; } | { id: string; kind: 'styled-text'; range: SourceRange; bodyRange: SourceRange; text: string; command: string; style: 'bold' | 'italic' | 'underline' | 'monospace' | 'link'; href?: string; } | { id: string; kind: 'raw'; range: SourceRange; text: string; reason?: string; }; export type VisualTableCell = { id: string; row: number; column: number; range: SourceRange; bodyRange: SourceRange; text: string; inlines: readonly VisualInline[]; }; export type VisualBlock = | { id: string; kind: 'heading'; file: ProjectFilePath; range: SourceRange; bodyRange: SourceRange; text: string; level?: number; command?: string; starred?: boolean; /** Source-only label/comments immediately owned by this heading. */ suffix?: string; } | { id: string; kind: 'paragraph'; file: ProjectFilePath; range: SourceRange; bodyRange: SourceRange; text: string; } | { id: string; kind: 'rich-paragraph'; file: ProjectFilePath; range: SourceRange; text: string; inlines: readonly VisualInline[]; } | { id: string; kind: 'text-environment'; file: ProjectFilePath; range: SourceRange; bodyRange: SourceRange; text: string; environment: string; prefix: string; suffix: string; inlines: readonly VisualInline[]; } | { id: string; kind: 'figure'; file: ProjectFilePath; range: SourceRange; text: string; /** First graphics target retained for backward-compatible consumers. */ asset?: string; assetCommandRange?: SourceRange; assetBodyRange?: SourceRange; graphicsOptions?: string; graphicsOptionsBodyRange?: SourceRange; /** Every includegraphics target in source order. Complex figures remain protected. */ assets?: readonly { target: string; commandRange: SourceRange; bodyRange?: SourceRange; options?: string; optionsBodyRange?: SourceRange; }[]; complex?: boolean; caption?: string; captionBodyRange?: SourceRange; label?: string; labelBodyRange?: SourceRange; } | { id: string; kind: 'table'; file: ProjectFilePath; range: SourceRange; text: string; caption?: string; captionBodyRange?: SourceRange; label?: string; labelBodyRange?: SourceRange; columns?: string; cells?: readonly VisualTableCell[]; rowCount?: number; columnCount?: number; editable: boolean; } | { id: string; kind: 'list'; file: ProjectFilePath; range: SourceRange; text: string; environment: 'itemize' | 'enumerate'; ordered: boolean; items: readonly { id: string; range: SourceRange; bodyRange: SourceRange; inlines: readonly VisualInline[]; }[]; editable: boolean; } | { id: string; kind: 'theorem'; file: ProjectFilePath; range: SourceRange; bodyRange: SourceRange; text: string; environment: string; title?: string; editable: boolean; } | { id: string; kind: 'raw'; file: ProjectFilePath; range: SourceRange; text: string; reason?: string; }; export type FileSemanticIndex = { file: ProjectFilePath; sourceLength: number; nodes: SemanticNode[]; visualBlocks: VisualBlock[]; }; export type ProjectBibliographyEntry = BibEntry & { file: ProjectFilePath }; export type ProjectSemanticDiagnostic = { kind: | 'missing-input' | 'duplicate-label' | 'missing-reference' | 'missing-asset' | 'missing-citation' | 'duplicate-citation-key' | 'duplicate-doi' | 'duplicate-bibliography-work' | 'bibliography-title-mismatch' | 'bibliography-author-year-mismatch' | 'possible-renamed-citation-key' | 'unused-bibliography-entry' | 'malformed-bibliography'; severity: 'error' | 'warning'; file?: ProjectFilePath; message: string; target?: string; }; export type ProjectSemanticSnapshot = { rootFile: ProjectFilePath; files: Map; outline: SemanticNode[]; labels: SemanticNode[]; references: SemanticNode[]; citations: SemanticNode[]; packages: SemanticNode[]; assets: SemanticNode[]; inputs: SemanticNode[]; environments: SemanticNode[]; bibliographyEntries: ProjectBibliographyEntry[]; bibliographyFiles: ProjectFilePath[]; diagnostics: ProjectSemanticDiagnostic[]; }; const HEADING_LEVEL: Record = { part: 1, chapter: 1, section: 2, subsection: 3, subsubsection: 4, paragraph: 5, subparagraph: 6, }; const VERBATIM_ENVIRONMENTS = [ 'verbatim', 'Verbatim', 'lstlisting', 'minted', 'comment', ]; const REF_COMMANDS = new Set(['ref', 'eqref', 'pageref', 'autoref', 'cref', 'Cref']); const INPUT_COMMANDS = new Set(['input', 'include', 'subfile']); const BIB_COMMANDS = new Set(['bibliography', 'addbibresource']); const CITATION_COMMAND = /^(?:cite|citep|citet|citealp|citeauthor|citeyear|autocite|parencite|textcite|footcite|supercite)$/; const THEOREM_ENVIRONMENTS = new Set([ 'theorem', 'lemma', 'proof', 'definition', 'proposition', 'corollary', 'remark', 'example', 'claim', ]); const VISUAL_MATH_ENVIRONMENTS = new Set([ 'equation', 'equation*', 'align', 'align*', 'aligned', 'aligned*', 'gather', 'gather*', 'multline', 'multline*', 'split', 'cases', 'array', 'matrix', 'pmatrix', 'bmatrix', 'Bmatrix', 'vmatrix', 'Vmatrix', ]); const LIST_ENVIRONMENTS = new Set(['itemize', 'enumerate']); const STRUCTURED_ENVIRONMENTS = new Set([ 'figure', 'figure*', 'table', 'table*', 'tabular', ...LIST_ENVIRONMENTS, ...THEOREM_ENVIRONMENTS, ]); const VISUAL_INLINE_STYLE_COMMANDS = new Map([ ['textbf', 'bold'], ['textit', 'italic'], ['emph', 'italic'], ['underline', 'underline'], ['texttt', 'monospace'], ] as const); type EnvironmentRegion = { name: string; from: number; to: number; bodyFrom: number; bodyTo: number; }; function lineAt(source: string, offset: number): number { let line = 1; for (let index = 0; index < offset; index += 1) { if (source.charCodeAt(index) === 10) line += 1; } return line; } function isEscaped(source: string, offset: number): boolean { let slashes = 0; for (let index = offset - 1; index >= 0 && source[index] === '\\'; index -= 1) slashes += 1; return slashes % 2 === 1; } function maskForScan(source: string): string { const chars = source.split(''); for (let i = 0; i < source.length; i += 1) { if (source[i] === '%' && !isEscaped(source, i)) { while (i < source.length && source[i] !== '\n') { chars[i] = ' '; i += 1; } } } let masked = chars.join(''); for (const env of VERBATIM_ENVIRONMENTS) { const startPattern = new RegExp(`\\\\begin\\s*\\{${env}\\}`, 'g'); while (true) { const match = startPattern.exec(masked); if (!match) break; const endToken = `\\end{${env}}`; const end = masked.indexOf(endToken, match.index + match[0].length); const to = end < 0 ? masked.length : end + endToken.length; const replacement = masked.slice(match.index, to).replace(/[^\n]/g, ' '); masked = masked.slice(0, match.index) + replacement + masked.slice(to); startPattern.lastIndex = to; } } return masked; } function skipWhitespace(source: string, offset: number): number { let index = offset; while (index < source.length && /\s/.test(source[index] ?? '')) index += 1; return index; } function readBalanced( source: string, offset: number, open: '{' | '[', close: '}' | ']', ): { content: string; range: SourceRange; bodyRange: SourceRange } | undefined { const start = skipWhitespace(source, offset); if (source[start] !== open) return undefined; let depth = 0; for (let index = start; index < source.length; index += 1) { const char = source[index]; if (char === open && !isEscaped(source, index)) depth += 1; if (char === close && !isEscaped(source, index)) { depth -= 1; if (depth === 0) { return { content: source.slice(start + 1, index), range: { from: start, to: index + 1 }, bodyRange: { from: start + 1, to: index }, }; } } } return undefined; } function readCommandArguments(source: string, commandEnd: number) { let cursor = commandEnd; const optional: NonNullable>[] = []; while (true) { const candidate = readBalanced(source, cursor, '[', ']'); if (!candidate) break; optional.push(candidate); cursor = candidate.range.to; } const required = readBalanced(source, cursor, '{', '}'); return { optional, required }; } function cleanTitle(value: string): string { return value .replace(/\\(?:textbf|textit|emph|texorpdfstring)\s*\{/g, '') .replace(/[{}]/g, '') .trim(); } function commandNode( file: ProjectFilePath, source: string, kind: SemanticKind, from: number, to: number, extra: Omit = {}, ): SemanticNode { return { id: `${file}:${kind}:${from}:${to}`, kind, file, range: { from, to }, line: lineAt(source, from), ...extra, }; } function scanEnvironmentRegions(masked: string): EnvironmentRegion[] { const regions: EnvironmentRegion[] = []; const stack: { name: string; from: number; bodyFrom: number }[] = []; const pattern = /\\(begin|end)\s*\{([^{}]+)\}/g; while (true) { const match = pattern.exec(masked); if (!match) break; const mode = match[1]; const name = (match[2] ?? '').trim(); if (mode === 'begin') { stack.push({ name, from: match.index, bodyFrom: match.index + match[0].length }); continue; } let index = stack.length - 1; while (index >= 0 && stack[index]?.name !== name) index -= 1; if (index < 0) continue; const begin = stack[index]!; stack.splice(index, 1); regions.push({ name, from: begin.from, to: match.index + match[0].length, bodyFrom: begin.bodyFrom, bodyTo: match.index, }); } return regions.sort((a, b) => a.from - b.from || b.to - a.to); } function trimRange(source: string, from: number, to: number): SourceRange { let start = from; let end = to; while (start < end && /\s/.test(source[start] ?? '')) start += 1; while (end > start && /\s/.test(source[end - 1] ?? '')) end -= 1; return { from: start, to: end }; } /** * Unknown environments are editable only when their body is structurally * text-like. This is deliberately capability-based: Learnitive never keys Visual * behavior to a document class/template name. Nested/layout/alignment/code * environments stay protected, while simple custom prose environments can be * edited without rewriting their begin/end wrappers. */ function isSafeUnknownTextEnvironment( source: string, region: EnvironmentRegion, regions: readonly EnvironmentRegion[], ): boolean { const bodyRange = trimRange(source, region.bodyFrom, region.bodyTo); if (bodyRange.to <= bodyRange.from) return false; if (regions.some( (candidate) => candidate !== region && candidate.from >= region.bodyFrom && candidate.to <= region.bodyTo, )) { return false; } const body = source.slice(bodyRange.from, bodyRange.to); // Environment arguments immediately after \begin{...} and alignment-like // separators are structural source, not a prose body. if (/^[{[]/u.test(body) || /(^|[^\\])&/u.test(body)) return false; // A control sequence that owns a whole source line usually indicates a // layout/code/custom structural environment. Inline commands inside prose // remain supported/protected individually by richInlinesForSegment(). if (/(?:^|\r?\n)[ \t]*\\[A-Za-z@]+\*?/u.test(body)) return false; // TeX display row syntax is structural enough that a generic environment // must remain Source-only unless a dedicated deterministic parser owns it. if (/\\\\(?:\[[^\]]*\])?(?:[ \t]*(?:\r?\n|$))/u.test(body)) return false; return /[\p{L}\p{N}]/u.test(body); } function safeUnknownEnvironmentBlock( file: ProjectFilePath, source: string, nodes: SemanticNode[], region: EnvironmentRegion, regions: readonly EnvironmentRegion[], ): VisualBlock | undefined { if (!isSafeUnknownTextEnvironment(source, region, regions)) return undefined; const bodyRange = trimRange(source, region.bodyFrom, region.bodyTo); const bodyInlines = richInlinesForSegment(file, source, nodes, bodyRange) ?? [ { id: `${file}:text:${bodyRange.from}:${bodyRange.to}`, kind: 'text' as const, range: bodyRange, bodyRange, text: source.slice(bodyRange.from, bodyRange.to), }, ]; const prefixRange = { from: region.from, to: bodyRange.from }; const suffixRange = { from: bodyRange.to, to: region.to }; return { id: `${file}:text-environment:${region.from}:${region.to}`, kind: 'text-environment', file, range: { from: region.from, to: region.to }, bodyRange, text: source.slice(bodyRange.from, bodyRange.to), environment: region.name, prefix: source.slice(prefixRange.from, prefixRange.to), suffix: source.slice(suffixRange.from, suffixRange.to), inlines: bodyInlines, }; } function readFollowingRequiredArguments( source: string, start: number, limit = 4, ): NonNullable>[] { const result: NonNullable>[] = []; let cursor = start; while (result.length < limit) { while (cursor < source.length && /\s/.test(source[cursor] ?? '')) cursor += 1; const next = readBalanced(source, cursor, '{', '}'); if (!next) break; result.push(next); cursor = next.range.to; } return result; } function rawCommandRange(source: string, commandFrom: number, commandEnd: number): SourceRange { let cursor = commandEnd; let end = commandEnd; for (let count = 0; count < 5; count += 1) { while (cursor < source.length && /[ \t]/.test(source[cursor] ?? '')) cursor += 1; const optional = readBalanced(source, cursor, '[', ']'); if (optional) { end = optional.range.to; cursor = optional.range.to; continue; } const required = readBalanced(source, cursor, '{', '}'); if (required) { end = required.range.to; cursor = required.range.to; continue; } break; } return { from: commandFrom, to: end }; } const CANONICAL_EDITABLE_LITERAL_COMMANDS = [ '\\textbackslash{}', '\\textasciitilde{}', '\\textasciicircum{}', ] as const; function canonicalEditableLiteralCommandAt(source: string, from: number): string | undefined { return CANONICAL_EDITABLE_LITERAL_COMMANDS.find((candidate) => source.startsWith(candidate, from)); } function isCanonicalEditableEscapedSymbol(value: string): boolean { return /^\\[%&#_$\{\}]$/u.test(value); } function stripCanonicalEditableLiteralSource(value: string): string { return value .replaceAll('\\textbackslash{}', '') .replaceAll('\\textasciitilde{}', '') .replaceAll('\\textasciicircum{}', '') .replace(/\\[%&#_$\{\}]/gu, ''); } function simpleEditableInlineBody(source: string, range: SourceRange): boolean { const body = source.slice(range.from, range.to); const residual = stripCanonicalEditableLiteralSource(body); // Only ordinary text plus Learnitive's canonical literal escapes is safe to // unwrap into a Visual formatting mark. Nested commands/math/groups remain // protected rather than being flattened and reserialized with new meaning. return !/[\\%$&{}#_^~]/u.test(residual); } function richInlinesForSegment( file: ProjectFilePath, source: string, nodes: readonly SemanticNode[], range: SourceRange, ): VisualInline[] | undefined { const segment = source.slice(range.from, range.to); // Comments have the highest lexical precedence in ordinary LaTeX source. // Commands/math-looking text inside a `% ...` comment must never be claimed // as visible semantics first (for example `\\vspace` or `\\centerline` // in template instructions). Claim complete comment ranges up front so later // scanners cannot split them into visible text plus protected command chips. const commentRanges: SourceRange[] = []; for (let index = 0; index < segment.length; index += 1) { if (segment[index] !== '%') continue; // An escaped percent (\%) is literal text, not the start of a comment. let slashCount = 0; for (let scan = index - 1; scan >= 0 && segment[scan] === '\\'; scan -= 1) slashCount += 1; if (slashCount % 2 === 1) continue; let end = index + 1; while (end < segment.length && segment[end] !== '\n' && segment[end] !== '\r') end += 1; commentRanges.push({ from: range.from + index, to: range.from + end }); index = Math.max(index, end - 1); } const overlapsComment = (from: number, to: number) => commentRanges.some((comment) => from < comment.to && to > comment.from); const mathRegions = findLatexMathRegions(source).filter( (region) => region.from >= range.from && region.to <= range.to && !overlapsComment(region.from, region.to), ); const semanticGroups = new Map(); for (const node of nodes) { if (node.range.from < range.from || node.range.to > range.to) continue; if (node.kind !== 'citation' && node.kind !== 'reference' && node.kind !== 'footnote') continue; if (overlapsComment(node.range.from, node.range.to)) continue; if (mathRegions.some((math) => node.range.from >= math.from && node.range.to <= math.to)) continue; const key = `${node.range.from}:${node.range.to}`; const list = semanticGroups.get(key) ?? []; list.push(node); semanticGroups.set(key, list); } type RichUnit = | { kind: 'semantic'; from: number; to: number; group: SemanticNode[] } | { kind: 'math'; from: number; to: number; region: ReturnType[number] } | { kind: 'hard-break'; from: number; to: number; sourceText: typeof CANONICAL_LATEX_HARD_BREAK | '\\newline' | '\\\\'; } | { kind: 'styled'; from: number; to: number; bodyRange: SourceRange; command: string; style: 'bold' | 'italic' | 'underline' | 'monospace' | 'link'; href?: string } | { kind: 'raw'; from: number; to: number; reason: string }; const units: RichUnit[] = [ ...commentRanges.map((comment) => ({ kind: 'raw' as const, from: comment.from, to: comment.to, reason: 'LaTeX comment preserved as invisible source trivia.', })), ...[...semanticGroups.values()].map((group) => ({ kind: 'semantic' as const, from: group[0]!.range.from, to: group[0]!.range.to, group, })), ...mathRegions.map((region) => ({ kind: 'math' as const, from: region.from, to: region.to, region, })), ]; const overlapsClaimed = (from: number, to: number) => units.some((unit) => from < unit.to && to > unit.from); // R14.23.2: treat both explicit \newline and the traditional \\ as // first-class visual hard breaks instead of raw/source-only islands. // Add them before generic command/raw scanning so \newline is not consumed // as an unsupported command and \\ is not consumed by protectPattern. const hardBreakPattern = /\\newline\b(?:\{\})?|\\\\/g; while (true) { const hardBreak = hardBreakPattern.exec(segment); if (!hardBreak) break; const from = range.from + hardBreak.index; const to = from + hardBreak[0].length; if (!overlapsClaimed(from, to)) { units.push({ kind: 'hard-break', from, to, sourceText: hardBreak[0] === CANONICAL_LATEX_HARD_BREAK ? CANONICAL_LATEX_HARD_BREAK : hardBreak[0] === '\\newline' ? '\\newline' : '\\\\', }); } } const commandPattern = /\\([A-Za-z@]+)(?:\*)?/g; while (true) { const commandMatch = commandPattern.exec(segment); if (!commandMatch) break; const commandFrom = range.from + commandMatch.index; const commandEnd = commandFrom + commandMatch[0].length; if (units.some((unit) => commandFrom >= unit.from && commandFrom < unit.to)) continue; // These exact spellings are the canonical literal encodings emitted by the // Visual serializer for ordinary text. Leave them inside the surrounding // text unit so the Visual projector can decode them back to editable prose. // This is intentionally exact rather than treating arbitrary commands as // text: unknown control sequences still fail closed as source-only LaTeX. if (canonicalEditableLiteralCommandAt(source, commandFrom)) continue; const command = commandMatch[1] ?? ''; const style = VISUAL_INLINE_STYLE_COMMANDS.get(command as 'textbf' | 'textit' | 'emph' | 'underline' | 'texttt'); if (style) { const required = readFollowingRequiredArguments(source, commandEnd, 1)[0]; if (required && simpleEditableInlineBody(source, required.bodyRange)) { const candidate: RichUnit = { kind: 'styled', from: commandFrom, to: required.range.to, bodyRange: required.bodyRange, command, style, }; if (!overlapsClaimed(candidate.from, candidate.to)) units.push(candidate); continue; } } if (command === 'url') { const required = readFollowingRequiredArguments(source, commandEnd, 1)[0]; if (required && simpleEditableInlineBody(source, required.bodyRange)) { const candidate: RichUnit = { kind: 'styled', from: commandFrom, to: required.range.to, bodyRange: required.bodyRange, command, style: 'link', href: source.slice(required.bodyRange.from, required.bodyRange.to), }; if (!overlapsClaimed(candidate.from, candidate.to)) units.push(candidate); continue; } } if (command === 'href') { const required = readFollowingRequiredArguments(source, commandEnd, 2); const target = required[0]; const label = required[1]; if (target && label && simpleEditableInlineBody(source, label.bodyRange)) { const candidate: RichUnit = { kind: 'styled', from: commandFrom, to: label.range.to, bodyRange: label.bodyRange, command, style: 'link', href: source.slice(target.bodyRange.from, target.bodyRange.to), }; if (!overlapsClaimed(candidate.from, candidate.to)) units.push(candidate); continue; } } const raw = rawCommandRange(source, commandFrom, commandEnd); const crossing = units.some( (unit) => raw.from < unit.to && raw.to > unit.from && !(unit.from >= raw.from && unit.to <= raw.to), ); if (!crossing) { for (let index = units.length - 1; index >= 0; index -= 1) { const unit = units[index]!; if (unit.from >= raw.from && unit.to <= raw.to) units.splice(index, 1); } units.push({ kind: 'raw', from: raw.from, to: raw.to, reason: `\\${command} is preserved as source-only LaTeX.`, }); } } // Protect TeX line breaks and escaped symbols as small islands rather than // forcing the entire paragraph into Raw LaTeX. Comments were claimed first // above so commands contained inside them can never leak into manuscript UI. const protectPattern = /\\\\|\\[^A-Za-z@\s]/g; while (true) { const protect = protectPattern.exec(segment); if (!protect) break; const from = range.from + protect.index; const to = from + protect[0].length; // Learnitive serializes ordinary literal %, &, #, _, $, { and } using these // exact two-byte escapes. They are text semantics, not protected syntax. if (isCanonicalEditableEscapedSymbol(protect[0])) continue; if (!overlapsClaimed(from, to)) { units.push({ kind: 'raw', from, to, reason: 'Protected LaTeX syntax.', }); } } if (units.length === 0) return undefined; units.sort((a, b) => a.from - b.from || b.to - a.to); const nonOverlapping: RichUnit[] = []; for (const unit of units) { const previous = nonOverlapping.at(-1); if (previous && unit.from < previous.to) continue; nonOverlapping.push(unit); } const inlines: VisualInline[] = []; let cursor = range.from; const pushText = (from: number, to: number) => { if (to <= from) return; inlines.push({ id: `${file}:text:${from}:${to}`, kind: 'text', range: { from, to }, bodyRange: { from, to }, text: source.slice(from, to), }); }; for (const unit of nonOverlapping) { pushText(cursor, unit.from); if (unit.kind === 'hard-break') { inlines.push({ id: `${file}:hard-break:${unit.from}:${unit.to}`, kind: 'hard-break', range: { from: unit.from, to: unit.to }, sourceText: unit.sourceText, }); } else if (unit.kind === 'math') { const region = unit.region; inlines.push({ id: `${file}:math:${region.from}:${region.to}`, kind: 'math', range: { from: region.from, to: region.to }, bodyRange: { from: region.contentFrom, to: region.contentTo }, text: region.content, mathLiveValue: latexMathRegionToMathLiveValue(region), mathKind: region.kind, delimiter: region.delimiter, ...(region.environment ? { environment: region.environment } : {}), editable: canEditLatexMathWithMathLive(region), }); } else if (unit.kind === 'styled') { inlines.push({ id: `${file}:styled:${unit.from}:${unit.to}`, kind: 'styled-text', range: { from: unit.from, to: unit.to }, bodyRange: unit.bodyRange, text: source.slice(unit.bodyRange.from, unit.bodyRange.to), command: unit.command, style: unit.style, ...(unit.href ? { href: unit.href } : {}), }); } else if (unit.kind === 'raw') { inlines.push({ id: `${file}:raw-inline:${unit.from}:${unit.to}`, kind: 'raw', range: { from: unit.from, to: unit.to }, text: source.slice(unit.from, unit.to), reason: unit.reason, }); } else { const first = unit.group[0]!; if (first.kind === 'citation') { inlines.push({ id: `${file}:citation:${first.range.from}:${first.range.to}`, kind: 'citation', range: first.range, command: first.name ?? 'cite', targets: unit.group.map((item) => item.target).filter((target): target is string => Boolean(target)), }); } else if (first.kind === 'reference') { inlines.push({ id: `${file}:reference:${first.range.from}:${first.range.to}`, kind: 'reference', range: first.range, command: first.name ?? 'ref', target: first.target ?? '', }); } else if (first.kind === 'footnote' && first.bodyRange) { const text = source.slice(first.bodyRange.from, first.bodyRange.to); inlines.push({ id: `${file}:footnote:${first.range.from}:${first.range.to}`, kind: 'footnote', range: first.range, bodyRange: first.bodyRange, text, editable: !/[\\$]/.test(text), }); } } cursor = unit.to; } pushText(cursor, range.to); // A hard break at the end of a rich paragraph/list/cell still owns a real // canonical source boundary immediately after its TeX token. Project a // zero-length editable text host there so Shift+Enter and immediate typing // never need a synthetic ProseMirror-only caret. if (inlines.at(-1)?.kind === 'hard-break' && cursor === range.to) { inlines.push({ id: `${file}:text-empty:${range.to}`, kind: 'text', range: { from: range.to, to: range.to }, bodyRange: { from: range.to, to: range.to }, text: '', }); } return inlines; } function generatedShortstackBodyRange(source: string, range: SourceRange): SourceRange | undefined { const segment = source.slice(range.from, range.to); const prefix = /^\\shortstack(?:\s*\[[lcr]\])?\s*/u.exec(segment); if (!prefix) return undefined; const body = readBalanced(source, range.from + prefix[0].length, '{', '}'); if (!body || body.range.to !== range.to) return undefined; return body.bodyRange; } function parseSimpleTabularCells( file: ProjectFilePath, source: string, region: EnvironmentRegion, nodes: readonly SemanticNode[], ): { cells: VisualTableCell[]; rowCount: number; columnCount: number } | undefined { const model = parseLatexTabular(source, { from: region.from, to: region.to }); if (!model || model.complex || model.rows.length === 0) return undefined; if (model.rows.some((row) => row.cells.length !== model.columns.length)) return undefined; const cells: VisualTableCell[] = []; for (const row of model.rows) { for (const cell of row.cells) { const range = { from: cell.from, to: cell.to }; const bodyRange = generatedShortstackBodyRange(source, range) ?? range; const text = source.slice(bodyRange.from, bodyRange.to); if (text.includes('$')) return undefined; const rich = richInlinesForSegment(file, source, nodes, bodyRange); if ( rich?.some( (inline) => inline.kind !== 'text' && inline.kind !== 'styled-text' && inline.kind !== 'hard-break', ) ) { return undefined; } const inlines: readonly VisualInline[] = rich ?? [ { id: `${file}:text:${bodyRange.from}:${bodyRange.to}`, kind: 'text', range: bodyRange, bodyRange, text, }, ]; cells.push({ id: `${file}:table-cell:${range.from}:${range.to}`, row: row.row, column: cell.column, range, bodyRange, text, inlines, }); } } return { cells, rowCount: model.rows.length, columnCount: model.columns.length, }; } function parseVisualListItems( file: ProjectFilePath, source: string, nodes: readonly SemanticNode[], region: EnvironmentRegion, regions: readonly EnvironmentRegion[], ): Extract['items'] | undefined { const nested = regions.filter( (candidate) => candidate !== region && candidate.from >= region.bodyFrom && candidate.to <= region.bodyTo, ); if (nested.length > 0) return undefined; const body = source.slice(region.bodyFrom, region.bodyTo); const itemPattern = /\\item(?![A-Za-z@])(?:\s*\[([^\]]*)\])?/g; const starts: { from: number; bodyFrom: number }[] = []; while (true) { const match = itemPattern.exec(body); if (!match) break; const from = region.bodyFrom + match.index; if (nested.some((candidate) => from >= candidate.from && from < candidate.to)) continue; starts.push({ from, bodyFrom: from + match[0].length }); } if (starts.length === 0) return undefined; return starts.map((item, index) => { const next = starts[index + 1]; const range = { from: item.from, to: next?.from ?? region.bodyTo }; const trimmedBodyRange = trimRange(source, item.bodyFrom, range.to); const emptyAnchor = trimmedBodyRange.from === trimmedBodyRange.to ? canonicalEmptySourceAnchor(source, item.bodyFrom, range.to) : undefined; const bodyRange = emptyAnchor === undefined ? trimmedBodyRange : { from: emptyAnchor, to: emptyAnchor }; const rich = richInlinesForSegment(file, source, nodes, bodyRange); const inlines: VisualInline[] = rich ?? [ { id: `${file}:list-text:${bodyRange.from}:${bodyRange.to}`, kind: 'text', range: bodyRange, bodyRange, text: source.slice(bodyRange.from, bodyRange.to), }, ]; return { id: `${file}:list-item:${item.from}:${range.to}`, range, bodyRange, inlines, }; }); } function buildVisualBlocks( file: ProjectFilePath, source: string, masked: string, nodes: SemanticNode[], ): VisualBlock[] { const begin = /\\begin\s*\{document\}/.exec(masked); const endMatches = [...masked.matchAll(/\\end\s*\{document\}/g)]; const end = endMatches.at(-1); const bodyFrom = begin ? begin.index + begin[0].length : 0; const bodyTo = end?.index ?? source.length; const regions = scanEnvironmentRegions(masked); const structured = regions.filter( (region) => STRUCTURED_ENVIRONMENTS.has(region.name) && region.from >= bodyFrom && region.to <= bodyTo && !regions.some( (parent) => parent !== region && STRUCTURED_ENVIRONMENTS.has(parent.name) && parent.from <= region.from && parent.to >= region.to && (parent.from < region.from || parent.to > region.to), ), ); const headings = nodes .filter( (node) => node.level && node.bodyRange && node.range.from >= bodyFrom && node.range.from < bodyTo && !structured.some((region) => node.range.from >= region.from && node.range.to <= region.to), ) .sort((a, b) => a.range.from - b.range.from); const headingOwnedSuffix = (heading: SemanticNode): { to: number; suffix: string } => { let to = heading.range.to; const labels = nodes .filter((node) => node.kind === 'label' && node.range.from >= to && node.range.from < bodyTo) .sort((left, right) => left.range.from - right.range.from); for (const label of labels) { const gap = source.slice(to, label.range.from); if (!/^(?:(?:[ \t\r\n]+)|(?:%[^\r\n]*(?:\r?\n|$)))*$/u.test(gap)) break; to = label.range.to; // A section normally owns one immediate label. Multiple consecutive // labels are still safe and preserved in the same invisible suffix. } return { to, suffix: source.slice(heading.range.to, to) }; }; const items: { from: number; to: number; block: () => VisualBlock }[] = []; for (const heading of headings) { const owned = headingOwnedSuffix(heading); items.push({ from: heading.range.from, to: owned.to, block: () => ({ id: heading.id, kind: 'heading', file, range: { from: heading.range.from, to: owned.to }, bodyRange: heading.bodyRange!, text: heading.title ?? '', level: heading.level, command: heading.name, starred: Boolean(heading.starred), ...(owned.suffix ? { suffix: owned.suffix } : {}), }), }); } for (const region of structured) { const regionNodes = nodes.filter( (node) => node.range.from >= region.from && node.range.to <= region.to, ); if (region.name === 'figure' || region.name === 'figure*') { const assetNodes = regionNodes.filter((node) => node.kind === 'asset' && Boolean(node.target)); const asset = assetNodes[0]; const caption = regionNodes.find((node) => node.kind === 'caption'); const label = regionNodes.find((node) => node.kind === 'label'); const nestedEnvironments = regions.filter( (candidate) => candidate !== region && candidate.from >= region.bodyFrom && candidate.to <= region.bodyTo, ); const rawFigure = source.slice(region.from, region.to); const complexFigure = assetNodes.length !== 1 || nestedEnvironments.length > 0 || /\\(?:centerline|hfill|vfill|raisebox|subfigure|subcaptionbox|parbox)\b/u.test(rawFigure); items.push({ from: region.from, to: region.to, block: () => ({ id: `${file}:figure:${region.from}:${region.to}`, kind: 'figure', file, range: { from: region.from, to: region.to }, text: rawFigure, ...(assetNodes.length ? { assets: assetNodes.map((node) => ({ target: node.target!, commandRange: node.range, ...(node.bodyRange ? { bodyRange: node.bodyRange } : {}), ...(node.options ? { options: node.options } : {}), ...(node.optionsBodyRange ? { optionsBodyRange: node.optionsBodyRange } : {}), })), } : {}), ...(asset?.target ? { asset: asset.target, assetCommandRange: asset.range, ...(asset.bodyRange ? { assetBodyRange: asset.bodyRange } : {}), ...(asset.options ? { graphicsOptions: asset.options } : {}), ...(asset.optionsBodyRange ? { graphicsOptionsBodyRange: asset.optionsBodyRange } : {}), } : {}), complex: complexFigure, ...(caption?.bodyRange ? { caption: source.slice(caption.bodyRange.from, caption.bodyRange.to), captionBodyRange: caption.bodyRange, } : {}), ...(label?.bodyRange ? { label: source.slice(label.bodyRange.from, label.bodyRange.to), labelBodyRange: label.bodyRange, } : {}), }), }); continue; } if (region.name === 'table' || region.name === 'table*' || region.name === 'tabular') { const caption = regionNodes.find((node) => node.kind === 'caption'); const label = regionNodes.find((node) => node.kind === 'label'); const tabular = region.name === 'tabular' ? region : regions.find( (candidate) => candidate.name === 'tabular' && candidate.from >= region.from && candidate.to <= region.to, ); let columns: string | undefined; if (tabular) { const beginToken = /\\begin\s*\{tabular\}/.exec(source.slice(tabular.from, tabular.bodyFrom)); if (beginToken) { const spec = readBalanced(source, tabular.bodyFrom, '{', '}'); columns = spec?.content; if (spec) tabular.bodyFrom = spec.range.to; } } const parsed = tabular ? parseSimpleTabularCells(file, source, tabular, regionNodes) : undefined; items.push({ from: region.from, to: region.to, block: () => ({ id: `${file}:table:${region.from}:${region.to}`, kind: 'table', file, range: { from: region.from, to: region.to }, text: source.slice(region.from, region.to), ...(caption?.bodyRange ? { caption: source.slice(caption.bodyRange.from, caption.bodyRange.to), captionBodyRange: caption.bodyRange } : {}), ...(label?.bodyRange ? { label: source.slice(label.bodyRange.from, label.bodyRange.to), labelBodyRange: label.bodyRange } : {}), ...(columns ? { columns } : {}), ...(parsed ? { cells: parsed.cells, rowCount: parsed.rowCount, columnCount: parsed.columnCount, } : {}), editable: Boolean(parsed), }), }); continue; } if (LIST_ENVIRONMENTS.has(region.name)) { const listItems = parseVisualListItems(file, source, nodes, region, regions); items.push({ from: region.from, to: region.to, block: () => ({ id: `${file}:list:${region.from}:${region.to}`, kind: 'list', file, range: { from: region.from, to: region.to }, text: source.slice(region.from, region.to), environment: region.name as 'itemize' | 'enumerate', ordered: region.name === 'enumerate', items: listItems ?? [], editable: Boolean(listItems), }), }); continue; } if (THEOREM_ENVIRONMENTS.has(region.name)) { const optionalTitle = readBalanced(source, region.bodyFrom, '[', ']'); const theoremBodyFrom = optionalTitle?.range.to ?? region.bodyFrom; const bodyRange = trimRange(source, theoremBodyFrom, region.bodyTo); const bodyText = source.slice(bodyRange.from, bodyRange.to); items.push({ from: region.from, to: region.to, block: () => ({ id: `${file}:theorem:${region.from}:${region.to}`, kind: 'theorem', file, range: { from: region.from, to: region.to }, bodyRange, text: bodyText, environment: region.name, ...(optionalTitle ? { title: optionalTitle.content } : {}), editable: !/[\\$]/.test(bodyText), }), }); } } // Unknown environments are protected at environment granularity. This keeps // custom/TikZ/verbatim-like source lossless without turning the paragraphs // before and after the environment into one giant Raw LaTeX block. const unsupportedRegions = regions.filter( (region) => region.name !== 'document' && !STRUCTURED_ENVIRONMENTS.has(region.name) && !VISUAL_MATH_ENVIRONMENTS.has(region.name) && region.from >= bodyFrom && region.to <= bodyTo && !regions.some( (parent) => parent !== region && parent.name !== 'document' && parent.from >= bodyFrom && parent.to <= bodyTo && parent.from <= region.from && parent.to >= region.to && (parent.from < region.from || parent.to > region.to), ), ); for (const region of unsupportedRegions) { const editable = safeUnknownEnvironmentBlock(file, source, nodes, region, regions); items.push({ from: region.from, to: region.to, block: () => editable ?? ({ id: `${file}:raw-environment:${region.from}:${region.to}`, kind: 'raw', file, range: { from: region.from, to: region.to }, text: source.slice(region.from, region.to), reason: `The ${region.name} environment is preserved as source-only LaTeX.`, }), }); } // Bibliography setup is document-level control, not manuscript prose. Keep // adjacent bibliography commands together as one exact source-backed block // so they cannot be swallowed into a rich paragraph at the end of a paper. // This is generic for BibTeX/biblatex commands and intentionally independent // of any document class or template. const bibliographyRanges = [...new Map( nodes .filter( (node) => node.kind === 'bibliography' && node.range.from >= bodyFrom && node.range.to <= bodyTo && !regions.some( (region) => region.name !== 'document' && region.from <= node.range.from && region.to >= node.range.to, ), ) .map((node) => [`${node.range.from}:${node.range.to}`, node.range] as const), ).values()].sort((left, right) => left.from - right.from || left.to - right.to); const bibliographyGroups: SourceRange[] = []; for (const range of bibliographyRanges) { const previous = bibliographyGroups.at(-1); if ( previous && /^(?:(?:[ \t\r\n]+)|(?:%[^\r\n]*(?:\r?\n|$)))*$/u.test( source.slice(previous.to, range.from), ) ) { previous.to = Math.max(previous.to, range.to); continue; } bibliographyGroups.push({ ...range }); } for (const range of bibliographyGroups) { items.push({ from: range.from, to: range.to, block: () => ({ id: `${file}:bibliography:${range.from}:${range.to}`, kind: 'raw', file, range, text: source.slice(range.from, range.to), reason: 'Bibliography setup is preserved as source-only LaTeX.', }), }); } // Graphics commands are meaningful manuscript content even when they are not // wrapped in a standard figure environment. Project them as an immutable // figure proxy instead of allowing an unsupported includegraphics command to // degrade into generic inline Raw LaTeX or disappear. A simple surrounding // TeX group ({\includegraphics...}) is owned by the proxy as well. for (const assetNode of nodes.filter((node) => node.kind === 'asset' && Boolean(node.target))) { if (assetNode.range.from < bodyFrom || assetNode.range.to > bodyTo) continue; if (items.some((item) => item.from <= assetNode.range.from && item.to >= assetNode.range.to)) continue; if (regions.some((region) => region.name !== 'document' && region.from <= assetNode.range.from && region.to >= assetNode.range.to)) continue; let from = assetNode.range.from; let to = assetNode.range.to; let before = from - 1; while (before >= bodyFrom && /[ \t]/u.test(source[before] ?? '')) before -= 1; let after = to; while (after < bodyTo && /[ \t]/u.test(source[after] ?? '')) after += 1; if (source[before] === '{' && source[after] === '}') { const beforeBody = source.slice(before + 1, from); const afterBody = source.slice(to, after); if (!beforeBody.trim() && !afterBody.trim()) { from = before; to = after + 1; } } const target = assetNode.target!; items.push({ from, to, block: () => ({ id: `${file}:standalone-figure:${from}:${to}`, kind: 'figure', file, range: { from, to }, text: source.slice(from, to), asset: target, assetCommandRange: assetNode.range, ...(assetNode.bodyRange ? { assetBodyRange: assetNode.bodyRange } : {}), ...(assetNode.options ? { graphicsOptions: assetNode.options } : {}), ...(assetNode.optionsBodyRange ? { graphicsOptionsBodyRange: assetNode.optionsBodyRange } : {}), assets: [{ target, commandRange: assetNode.range, ...(assetNode.bodyRange ? { bodyRange: assetNode.bodyRange } : {}), ...(assetNode.options ? { options: assetNode.options } : {}), ...(assetNode.optionsBodyRange ? { optionsBodyRange: assetNode.optionsBodyRange } : {}), }], complex: true, }), }); } items.sort((a, b) => a.from - b.from || b.to - a.to); const blocks: VisualBlock[] = []; let cursor = bodyFrom; const paragraphContentRanges = (from: number, to: number): SourceRange[] => { const ranges: SourceRange[] = []; let start = from; let cursor = from; let inComment = false; let previousDelimiterWasExplicitPar = false; const hasContent = (rangeFrom: number, rangeTo: number) => { const trimmed = trimRange(source, rangeFrom, rangeTo); return trimmed.to > trimmed.from; }; const pushContent = (end: number) => { if (end > start && hasContent(start, end)) ranges.push({ from: start, to: end }); }; const pushEmptyAnchor = (offset: number) => { const previous = ranges.at(-1); if (previous?.from === offset && previous.to === offset) return; ranges.push({ from: offset, to: offset }); }; while (cursor < to) { const char = source[cursor]; if (inComment) { if (char === '\n') inComment = false; cursor += 1; continue; } if (char === '%' && source[cursor - 1] !== '\\') { inComment = true; cursor += 1; continue; } // Imported/source-authored explicit \par remains fully supported, but // Visual mode no longer generates it for normal paragraph Enter. if ( char === '\\' && source[cursor - 1] !== '\\' && source.startsWith(CANONICAL_LATEX_PARAGRAPH_BREAK, cursor) ) { const after = cursor + CANONICAL_LATEX_PARAGRAPH_BREAK.length; const following = source[after]; if (!following || !/[A-Za-z@]/u.test(following)) { const contentBeforeDelimiter = hasContent(start, cursor); if (contentBeforeDelimiter) { ranges.push({ from: start, to: cursor }); } else if (previousDelimiterWasExplicitPar) { pushEmptyAnchor(cursor); } cursor = after; start = after; previousDelimiterWasExplicitPar = true; continue; } } if (char === '\n') { // A LaTeX paragraph boundary is one or more blank source lines. Keep // ordinary two-newline separation semantic-only, but preserve any // additional blank rows as real zero-length source anchors. At the end // of a segment, the final blank-line boundary itself is the next // typing-ready Visual paragraph. let whitespaceEnd = cursor; const newlineEnds: number[] = []; while (whitespaceEnd < to && /[ \t\r\n]/.test(source[whitespaceEnd] ?? '')) { if (source[whitespaceEnd] === '\n') newlineEnds.push(whitespaceEnd + 1); whitespaceEnd += 1; } if (newlineEnds.length >= 2) { // Blank source lines separate semantic blocks, but they are not // themselves visible manuscript rows. ProseMirror provides a single // trailing/interaction paragraph when the user actually needs a // typing surface. Keeping source-only blank anchors out of the // projection prevents the unexplained horizontal empty rows that // complex imported documents previously accumulated in Visual mode. pushContent(cursor); cursor = whitespaceEnd; start = whitespaceEnd; previousDelimiterWasExplicitPar = false; continue; } } cursor += 1; } if (hasContent(start, to)) { ranges.push({ from: start, to }); } else if (previousDelimiterWasExplicitPar) { pushEmptyAnchor(to); } return ranges; }; const pushSegment = (from: number, to: number) => { if (to <= from) return; for (const part of paragraphContentRanges(from, to)) { if (part.from === part.to) { blocks.push({ id: `${file}:paragraph-empty:${part.from}`, kind: 'paragraph', file, range: part, bodyRange: part, text: '', }); continue; } const trimmed = trimRange(source, part.from, part.to); if (trimmed.to <= trimmed.from) continue; const text = source.slice(trimmed.from, trimmed.to); const rich = richInlinesForSegment(file, source, nodes, trimmed); if (rich) { blocks.push({ id: `${file}:rich-paragraph:${trimmed.from}:${trimmed.to}`, kind: 'rich-paragraph', file, range: trimmed, text, inlines: rich, }); continue; } blocks.push({ id: `${file}:paragraph:${trimmed.from}:${trimmed.to}`, kind: 'paragraph', file, range: trimmed, bodyRange: trimmed, text, }); } }; for (const item of items) { if (item.to <= cursor) continue; if (item.from < cursor) continue; pushSegment(cursor, item.from); blocks.push(item.block()); cursor = item.to; } pushSegment(cursor, bodyTo); return blocks; } export function parseLatexFile( file: ProjectFilePath, source: string, ): FileSemanticIndex { const masked = maskForScan(source); const nodes: SemanticNode[] = []; const commandPattern = /\\([A-Za-z@]+)(\*)?/g; while (true) { const match = commandPattern.exec(masked); if (!match) break; const name = match[1] ?? ''; const commandEnd = match.index + match[0].length; const args = readCommandArguments(masked, commandEnd); const required = args.required; if (!required) continue; const rawRequired = source.slice(required.bodyRange.from, required.bodyRange.to); const commandTo = required.range.to; const headingLevel = HEADING_LEVEL[name]; if (headingLevel) { const kind = name === 'paragraph' ? 'paragraph-heading' : name === 'subparagraph' ? 'subparagraph-heading' : (name as SemanticKind); nodes.push( commandNode(file, source, kind, match.index, commandTo, { name, title: cleanTitle(rawRequired), level: headingLevel, bodyRange: required.bodyRange, starred: Boolean(match[2]), }), ); continue; } if (name === 'label') { nodes.push( commandNode(file, source, 'label', match.index, commandTo, { target: rawRequired.trim(), bodyRange: required.bodyRange, }), ); continue; } if (REF_COMMANDS.has(name)) { nodes.push( commandNode(file, source, 'reference', match.index, commandTo, { name, target: rawRequired.trim(), bodyRange: required.bodyRange, }), ); continue; } if (CITATION_COMMAND.test(name)) { for (const target of rawRequired .split(',') .map((item) => item.trim()) .filter(Boolean)) { nodes.push( commandNode(file, source, 'citation', match.index, commandTo, { name, target, bodyRange: required.bodyRange, }), ); } continue; } if (name === 'usepackage') { for (const target of rawRequired .split(',') .map((item) => item.trim()) .filter(Boolean)) { nodes.push(commandNode(file, source, 'package', match.index, commandTo, { target })); } continue; } if (name === 'documentclass') { nodes.push(commandNode(file, source, 'documentclass', match.index, commandTo, { target: rawRequired.trim() })); continue; } if (INPUT_COMMANDS.has(name)) { nodes.push( commandNode(file, source, name === 'include' ? 'include' : 'input', match.index, commandTo, { name, target: rawRequired.trim(), bodyRange: required.bodyRange, }), ); continue; } if (name === 'includegraphics') { const graphicsOptions = args.optional[0]; nodes.push( commandNode(file, source, 'asset', match.index, commandTo, { target: rawRequired.trim(), bodyRange: required.bodyRange, ...(graphicsOptions ? { options: source.slice( graphicsOptions.bodyRange.from, graphicsOptions.bodyRange.to, ), optionsBodyRange: graphicsOptions.bodyRange, } : {}), }), ); continue; } if (name === 'caption') { nodes.push(commandNode(file, source, 'caption', match.index, commandTo, { title: rawRequired, bodyRange: required.bodyRange })); continue; } if (name === 'footnote') { nodes.push(commandNode(file, source, 'footnote', match.index, commandTo, { title: rawRequired, bodyRange: required.bodyRange })); continue; } if (BIB_COMMANDS.has(name)) { const targets = name === 'bibliography' ? rawRequired.split(',') : [rawRequired]; for (const target of targets.map((item) => item.trim()).filter(Boolean)) { nodes.push(commandNode(file, source, 'bibliography', match.index, commandTo, { name, target, bodyRange: required.bodyRange })); } continue; } if (name === 'begin') { nodes.push(commandNode(file, source, 'environment', match.index, commandTo, { target: rawRequired.trim() })); } } nodes.sort((a, b) => a.range.from - b.range.from || a.range.to - b.range.to); return { file, sourceLength: source.length, nodes, visualBlocks: buildVisualBlocks(file, source, masked, nodes), }; } function directoryOf(path: string): string { const slash = path.lastIndexOf('/'); return slash < 0 ? '' : path.slice(0, slash + 1); } function resolveRelativeReference(from: ProjectFilePath, target: string): string | undefined { if (!target || target.startsWith('/') || target.includes('\\') || target.includes('#')) return undefined; const parts = `${directoryOf(from)}${target}`.split('/'); const normalized: string[] = []; for (const part of parts) { if (!part || part === '.') continue; if (part === '..') { if (normalized.length === 0) return undefined; normalized.pop(); continue; } normalized.push(part); } return normalized.join('/'); } export function resolveInputTarget(from: ProjectFilePath, target: string): ProjectFilePath | undefined { const base = resolveRelativeReference(from, target); if (!base) return undefined; const withExtension = /\.[A-Za-z0-9]+$/.test(base) ? base : `${base}.tex`; try { return projectFilePath(withExtension); } catch { return undefined; } } export function resolveAssetTarget(from: ProjectFilePath, target: string): ProjectFilePath | undefined { const resolved = resolveRelativeReference(from, target); if (!resolved) return undefined; try { return projectFilePath(resolved); } catch { return undefined; } } /** * Match an includegraphics-style reference against a concrete project file. * LaTeX permits the extension to be omitted, so callers that rename assets or * diagnose unused files must use the same rule as the missing-asset checker. */ export function assetReferenceMatchesFile( from: ProjectFilePath, target: string, file: ProjectFilePath, ): boolean { const resolved = resolveAssetTarget(from, target); if (!resolved) return false; if (resolved === file) return true; return !/\.[A-Za-z0-9]+$/.test(resolved) && file.startsWith(`${resolved}.`); } export function resolveBibliographyTarget(from: ProjectFilePath, target: string): ProjectFilePath | undefined { const raw = resolveRelativeReference(from, target); if (!raw) return undefined; const path = raw.toLowerCase().endsWith('.bib') ? raw : `${raw}.bib`; try { return projectFilePath(path); } catch { return undefined; } } function normalizeBibliographyText(value: string | undefined): string { return (value ?? '') .toLowerCase() .normalize('NFKD') .replace(/[{}]/g, '') .replace(/[^a-z0-9]+/g, ' ') .replace(/\s+/g, ' ') .trim(); } function bibliographyDoi(entry: BibEntry): string | undefined { const value = getBibField(entry, 'doi')?.value.trim(); if (!value) return undefined; return value .replace(/^https?:\/\/(?:dx\.)?doi\.org\//i, '') .replace(/^doi:\s*/i, '') .trim() .toLowerCase() || undefined; } function bibliographyFirstAuthor(entry: BibEntry): string { const raw = getBibField(entry, 'author')?.value.split(/\s+and\s+/i)[0]?.replace(/[{}]/g, '').trim() ?? ''; if (!raw) return ''; if (raw.includes(',')) return normalizeBibliographyText(raw.split(',')[0]); return normalizeBibliographyText(raw.split(/\s+/).at(-1)); } function bibliographyWorkIdentity(entry: BibEntry): string | undefined { const title = normalizeBibliographyText(getBibField(entry, 'title')?.value); if (!title) return undefined; const author = bibliographyFirstAuthor(entry); const year = normalizeBibliographyText(getBibField(entry, 'year')?.value); if (!author || !year) return undefined; return `${title}\u0000${author}\u0000${year}`; } function editDistance(left: string, right: string): number { if (left === right) return 0; if (!left.length) return right.length; if (!right.length) return left.length; let previous = Array.from({ length: right.length + 1 }, (_, index) => index); for (let i = 1; i <= left.length; i += 1) { const current = [i]; for (let j = 1; j <= right.length; j += 1) { current[j] = Math.min( current[j - 1]! + 1, previous[j]! + 1, previous[j - 1]! + (left[i - 1] === right[j - 1] ? 0 : 1), ); } previous = current; } return previous[right.length]!; } function probableRenamedCitationKey(target: string, keys: readonly string[]): string | undefined { const normalized = target.toLowerCase(); const candidates = keys .map((key) => ({ key, distance: editDistance(normalized, key.toLowerCase()) })) .filter((candidate) => candidate.distance <= Math.max(1, Math.floor(normalized.length * 0.2))) .sort((a, b) => a.distance - b.distance || a.key.localeCompare(b.key)); if (!candidates.length) return undefined; if (candidates[1] && candidates[1].distance === candidates[0]!.distance) return undefined; return candidates[0]!.key; } export function buildProjectSemanticIndex( sources: ReadonlyMap, rootFile: ProjectFilePath, allProjectFiles: ReadonlySet = new Set(sources.keys()), bibliographyFiles: ReadonlyMap = new Map(), ): ProjectSemanticSnapshot { const files = new Map(); for (const [path, source] of sources) files.set(path, parseLatexFile(path, source)); const allNodes = [...files.values()].flatMap((file) => file.nodes); const orderedFiles: ProjectFilePath[] = []; const visited = new Set(); const visit = (file: ProjectFilePath) => { if (visited.has(file) || !files.has(file)) return; visited.add(file); orderedFiles.push(file); const fileIndex = files.get(file); for (const input of fileIndex?.nodes ?? []) { if (input.kind !== 'input' && input.kind !== 'include') continue; const resolved = input.target ? resolveInputTarget(file, input.target) : undefined; if (resolved) visit(resolved); } }; visit(rootFile); for (const file of [...files.keys()].sort()) visit(file); const fileOrder = new Map(orderedFiles.map((file, index) => [file, index])); const outline = allNodes .filter((node) => node.level && /\.(?:tex|ltx)$/i.test(node.file)) .sort( (a, b) => (fileOrder.get(a.file) ?? Number.MAX_SAFE_INTEGER) - (fileOrder.get(b.file) ?? Number.MAX_SAFE_INTEGER) || a.range.from - b.range.from, ); const labels = allNodes.filter((node) => node.kind === 'label'); const references = allNodes.filter((node) => node.kind === 'reference'); const citations = allNodes.filter((node) => node.kind === 'citation'); const packages = allNodes.filter((node) => node.kind === 'package'); const assets = allNodes.filter((node) => node.kind === 'asset'); const inputs = allNodes.filter((node) => node.kind === 'input' || node.kind === 'include'); const environments = allNodes.filter((node) => node.kind === 'environment'); const diagnostics: ProjectSemanticDiagnostic[] = []; const bibliographyEntries: ProjectBibliographyEntry[] = []; for (const [file, bibliography] of bibliographyFiles) { bibliographyEntries.push(...bibliography.entries.map((entry) => ({ ...entry, file }))); for (const diagnostic of bibliography.diagnostics) { if (diagnostic.kind === 'duplicate-key') continue; diagnostics.push({ kind: 'malformed-bibliography', severity: 'warning', file, target: diagnostic.key, message: diagnostic.message, }); } } const labelsByTarget = new Map(); for (const label of labels) { if (!label.target) continue; const list = labelsByTarget.get(label.target) ?? []; list.push(label); labelsByTarget.set(label.target, list); } for (const [target, matches] of labelsByTarget) { if (matches.length > 1) diagnostics.push({ kind: 'duplicate-label', severity: 'error', target, message: `Duplicate label: ${target}` }); } for (const ref of references) { if (ref.target && !labelsByTarget.has(ref.target)) diagnostics.push({ kind: 'missing-reference', severity: 'warning', file: ref.file, target: ref.target, message: `Reference has no matching label: ${ref.target}` }); } for (const input of inputs) { const resolved = input.target ? resolveInputTarget(input.file, input.target) : undefined; if (!resolved || !allProjectFiles.has(resolved)) diagnostics.push({ kind: 'missing-input', severity: 'error', file: input.file, target: input.target, message: `Included source file was not found: ${input.target ?? ''}` }); } for (const asset of assets) { const resolved = asset.target ? resolveAssetTarget(asset.file, asset.target) : undefined; if (!resolved) continue; const hasExact = allProjectFiles.has(resolved); const hasExtensionVariant = !/\.[A-Za-z0-9]+$/.test(resolved) && [...allProjectFiles].some((path) => path.startsWith(`${resolved}.`)); if (!hasExact && !hasExtensionVariant) diagnostics.push({ kind: 'missing-asset', severity: 'warning', file: asset.file, target: asset.target, message: `Image asset was not found: ${asset.target ?? ''}` }); } const bibliographyByKey = new Map(); for (const entry of bibliographyEntries) { const list = bibliographyByKey.get(entry.key) ?? []; list.push(entry); bibliographyByKey.set(entry.key, list); } for (const [key, matches] of bibliographyByKey) { if (matches.length > 1) { diagnostics.push({ kind: 'duplicate-citation-key', severity: 'error', target: key, message: `Duplicate bibliography key: ${key}`, }); } } const bibliographyByDoi = new Map(); const bibliographyByWork = new Map(); for (const entry of bibliographyEntries) { const doi = bibliographyDoi(entry); if (doi) { const list = bibliographyByDoi.get(doi) ?? []; list.push(entry); bibliographyByDoi.set(doi, list); } const identity = bibliographyWorkIdentity(entry); if (identity) { const list = bibliographyByWork.get(identity) ?? []; list.push(entry); bibliographyByWork.set(identity, list); } } for (const [doi, matches] of bibliographyByDoi) { if (matches.length <= 1) continue; diagnostics.push({ kind: 'duplicate-doi', severity: 'warning', target: doi, message: `Multiple bibliography entries use DOI ${doi}: ${matches.map((entry) => entry.key).join(', ')}`, }); const titles = new Set(matches.map((entry) => normalizeBibliographyText(getBibField(entry, 'title')?.value)).filter(Boolean)); if (titles.size > 1) { diagnostics.push({ kind: 'bibliography-title-mismatch', severity: 'warning', target: doi, message: `Bibliography entries for DOI ${doi} disagree on title metadata.`, }); } const authorYears = new Set(matches.map((entry) => `${bibliographyFirstAuthor(entry)}\u0000${normalizeBibliographyText(getBibField(entry, 'year')?.value)}`)); if (authorYears.size > 1) { diagnostics.push({ kind: 'bibliography-author-year-mismatch', severity: 'warning', target: doi, message: `Bibliography entries for DOI ${doi} disagree on first-author/year metadata.`, }); } } for (const matches of bibliographyByWork.values()) { const distinctKeys = [...new Set(matches.map((entry) => entry.key))]; if (distinctKeys.length <= 1) continue; diagnostics.push({ kind: 'duplicate-bibliography-work', severity: 'warning', target: distinctKeys[0], message: `Probable duplicate work appears under multiple citation keys: ${distinctKeys.join(', ')}`, }); } const usedCitationKeys = new Set(citations.map((citation) => citation.target).filter((target): target is string => Boolean(target))); const knownBibliographyKeys = [...bibliographyByKey.keys()]; for (const citation of citations) { if (citation.target && !bibliographyByKey.has(citation.target)) { diagnostics.push({ kind: 'missing-citation', severity: 'warning', file: citation.file, target: citation.target, message: `Citation has no matching bibliography entry: ${citation.target}` }); const renamed = probableRenamedCitationKey(citation.target, knownBibliographyKeys); if (renamed) diagnostics.push({ kind: 'possible-renamed-citation-key', severity: 'warning', file: citation.file, target: citation.target, message: `Citation key ${citation.target} may have been renamed to ${renamed}. Review before replacing it.`, }); } } for (const entry of bibliographyEntries) { if (!usedCitationKeys.has(entry.key)) diagnostics.push({ kind: 'unused-bibliography-entry', severity: 'warning', file: entry.file, target: entry.key, message: `Bibliography entry is not cited: ${entry.key}` }); } const declaredBibliographies = allNodes.filter((node) => node.kind === 'bibliography'); for (const node of declaredBibliographies) { const resolved = node.target ? resolveBibliographyTarget(node.file, node.target) : undefined; if (resolved && !allProjectFiles.has(resolved)) { diagnostics.push({ kind: 'malformed-bibliography', severity: 'error', file: node.file, target: node.target, message: `Bibliography file was not found: ${node.target}` }); } } return { rootFile, files, outline, labels, references, citations, packages, assets, inputs, environments, bibliographyEntries, bibliographyFiles: [...bibliographyFiles.keys()].sort(), diagnostics, }; } export type ProjectSemanticSearchKind = | 'outline' | 'label' | 'reference' | 'citation' | 'asset' | 'package' | 'input' | 'environment' | 'bibliography'; export type ProjectSemanticSearchResult = { kind: ProjectSemanticSearchKind; file: ProjectFilePath; line: number; label: string; detail?: string; target?: string; range?: SourceRange; }; function includesNeedle(values: readonly (string | undefined)[], needle: string): boolean { return values.some((value) => value?.toLowerCase().includes(needle)); } /** Search the already-built project meaning without reparsing source. */ export function searchProjectSemanticIndex( snapshot: ProjectSemanticSnapshot, query: string, ): ProjectSemanticSearchResult[] { const needle = query.trim().toLowerCase(); if (!needle) return []; const results: ProjectSemanticSearchResult[] = []; const pushNode = ( kind: ProjectSemanticSearchKind, node: SemanticNode, label: string, detail?: string, ) => { if (!includesNeedle([label, detail, node.target, node.title, node.file], needle)) return; results.push({ kind, file: node.file, line: node.line, label, ...(detail ? { detail } : {}), ...(node.target ? { target: node.target } : {}), range: node.range, }); }; for (const node of snapshot.outline) { pushNode('outline', node, node.title || node.target || node.kind, node.kind); } for (const node of snapshot.labels) pushNode('label', node, node.target || 'label', 'label'); for (const node of snapshot.references) pushNode('reference', node, node.target || 'reference', node.name || 'reference'); for (const node of snapshot.citations) pushNode('citation', node, node.target || 'citation', node.name || 'citation'); for (const node of snapshot.assets) pushNode('asset', node, node.target || 'asset', 'image'); for (const node of snapshot.packages) pushNode('package', node, node.target || 'package', 'package'); for (const node of snapshot.inputs) pushNode('input', node, node.target || 'input', node.name || node.kind); for (const node of snapshot.environments) pushNode('environment', node, node.target || 'environment', 'environment'); for (const entry of snapshot.bibliographyEntries) { const title = entry.fields.find((field) => field.name === 'title')?.value; const author = entry.fields.find((field) => field.name === 'author')?.value; const year = entry.fields.find((field) => field.name === 'year')?.value; if (!includesNeedle([entry.key, title, author, year, entry.file], needle)) continue; results.push({ kind: 'bibliography', file: entry.file, line: entry.line, label: title?.trim() || entry.key, detail: [author?.trim(), year?.trim()].filter(Boolean).join(' ยท ') || entry.key, target: entry.key, range: entry.range, }); } return results.slice(0, 250); } export type ProjectDoctorCategory = | 'structure' | 'references' | 'bibliography' | 'assets' | 'compilation'; export type ProjectDoctorIssue = { id: string; category: ProjectDoctorCategory; severity: 'error' | 'warning' | 'info'; message: string; file?: ProjectFilePath; line?: number; target?: string; }; export type ProjectDoctorReport = { score: number; errors: number; warnings: number; issues: readonly ProjectDoctorIssue[]; }; function doctorCategory(kind: ProjectSemanticDiagnostic['kind']): ProjectDoctorCategory { if (kind === 'missing-input') return 'structure'; if (kind === 'duplicate-label' || kind === 'missing-reference') return 'references'; if (kind === 'missing-asset') return 'assets'; return 'bibliography'; } /** Convert semantic diagnostics into the stable Doctor issue contract. */ export function createSemanticDoctorReport( snapshot: ProjectSemanticSnapshot, ): ProjectDoctorReport { const issues: ProjectDoctorIssue[] = snapshot.diagnostics.map((diagnostic, index) => { const node = diagnostic.file ? snapshot.files .get(diagnostic.file) ?.nodes.find((candidate) => candidate.target === diagnostic.target) : undefined; return { id: `semantic:${diagnostic.kind}:${index}:${diagnostic.file ?? ''}:${diagnostic.target ?? ''}`, category: doctorCategory(diagnostic.kind), severity: diagnostic.severity, message: diagnostic.message, ...(diagnostic.file ? { file: diagnostic.file } : {}), ...(node ? { line: node.line } : {}), ...(diagnostic.target ? { target: diagnostic.target } : {}), }; }); const errors = issues.filter((issue) => issue.severity === 'error').length; const warnings = issues.filter((issue) => issue.severity === 'warning').length; const score = Math.max(0, 100 - errors * 15 - warnings * 4); return { score, errors, warnings, issues }; }