Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 47 additions & 27 deletions core/utils/streamMarkdownUtils.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { LineStream } from "../diff/util";
import { isMarkdownFile, MarkdownBlockStateTracker } from "./markdownUtils";
import type { MarkdownBlockStateTracker } from "./markdownUtils";
import { isMarkdownFile } from "./markdownUtils";

/**
* Determines if we should stop at a markdown block based on nested markdown logic.
Expand Down Expand Up @@ -50,48 +51,67 @@ export async function* stopAtLinesWithMarkdownSupport(
return;
}

// Collect all lines from the LLM stream first
const allLines: string[] = [];
for await (const line of lines) {
allLines.push(line);
}

const source = allLines.join("\n");
if (!source.match(/```(\w*|.*)(md|markdown|gfm|github-markdown)/)) {
// No nested markdown blocks detected, check for simple ``` stopping condition
let foundStandaloneBackticks = false;
for (let i = 0; i < allLines.length; i++) {
if (allLines[i].trim() === "```") {
// Found standalone backticks, yield lines up to this point
// The LLM reply starts *inside* the outer fence (the prompt prefills the opening fence).
// Inner blocks in the markdown body open with a fence line; while one is open, everything
// except a valid closer is content (per CommonMark, fences do not nest). A fence line
// needs 3+ backticks and at most 3 spaces of indentation; a closer must use at least as
// many backticks as its opener and carry no info string.
// A bare top-level fence is ambiguous (plain inner opener vs. the outer closer): treat it
// as the outer closer only when no fence lines follow it.
const FENCE_RE = /^ {0,3}(`{3,})(.*)$/;
const fenceIndices: number[] = [];
for (let i = 0; i < allLines.length; i++) {
if (FENCE_RE.test(allLines[i])) {
fenceIndices.push(i);
}
}
const lastFenceLine = fenceIndices[fenceIndices.length - 1] ?? -1;

let innerOpener: number | null = null; // Backtick count of the open inner fence

for (const i of fenceIndices) {
const fenceMatch = allLines[i].match(FENCE_RE)!;
const backtickCount = fenceMatch[1].length;
const infoString = fenceMatch[2].trim();

if (innerOpener === null) {
if (infoString.length === 0 && i === lastFenceLine) {
// Bare fence at the top level with nothing after it: the outer closing fence.
// Stop before it so the wrapper never leaks into the applied file.
for (let j = 0; j < i; j++) {
yield allLines[j];
}
foundStandaloneBackticks = true;
return;
}
// Opening fence (with or without an info string) — enter an inner block
innerOpener = backtickCount;
} else if (infoString.length === 0 && backtickCount >= innerOpener) {
// Valid closer for the open inner block
innerOpener = null;
}

// No standalone backticks found, yield all lines
if (!foundStandaloneBackticks) {
for (const line of allLines) {
yield line;
}
}
return;
// Any other backtick line inside an open inner block is content
}

// Use optimized state tracker for markdown block analysis
const stateTracker = new MarkdownBlockStateTracker(allLines);

for (let i = 0; i < allLines.length; i++) {
if (stateTracker.shouldStopAtPosition(i)) {
for (let j = 0; j < i; j++) {
yield allLines[j];
}
return;
// No outer closing fence found. If the stream ends with an unclosed inner fence
// (malformed markdown) or its final line is a fence, prefer treating that last fence
// as the outer closer so the wrapper delimiter never leaks into the edit output.
if (
lastFenceLine >= 0 &&
(innerOpener !== null || lastFenceLine === allLines.length - 1)
) {
for (let j = 0; j < lastFenceLine; j++) {
yield allLines[j];
}
return;
}

// If we get here, yield all lines
// Yield everything
for (const line of allLines) {
yield line;
}
Expand Down
Loading
Loading