bingshuo/grok-build-upstream-mirror
Watch
1
0
Fork
You've already forked grok-build-upstream-mirror
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Publish harness and TUI open-source

Browse source 
initial sync from the monorepo
This commit is contained in:
grokkybara[bot] 2026-07-16 06:46:02 +01:00
commit c68e39f604
2734 changed files with 1437016 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

57
crates/codegen/xai-grok-markdown/Cargo.toml Normal file
View file

@ -0,0 +1,57 @@
[package]
license = "Apache-2.0"
name = "xai-grok-markdown"
version = "0.1.0"
edition.workspace = true
authors = ["xAI"]
description = "Streaming markdown renderer for terminal UIs"
[lints.rust]
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(fuzzing)'] }
[dependencies]
anstyle = { workspace = true }
anstyle-lossy = { workspace = true }
anstyle-syntect = { workspace = true }
html-escape = { workspace = true }
linkify = { workspace = true }
pulldown-cmark = { workspace = true }
ratatui = { workspace = true }
supports-color = { workspace = true }
syntect = { workspace = true }
two-face = { workspace = true }
textwrap = { workspace = true }
unicode-width = { workspace = true }
url = { workspace = true }
xai-grok-markdown-core = { workspace = true }
[dependencies.crossterm]
workspace = true
features = ["event-stream", "bracketed-paste"]
optional = true
[dependencies.xai-ratatui-textarea]
workspace = true
optional = true
[dev-dependencies]
criterion = { workspace = true }
pretty_assertions = { workspace = true }
[[bin]]
name = "md-table-test"
path = "bin/md_table_test.rs"
required-features = ["playground"]
[[bin]]
name = "md-mermaid-test"
path = "bin/md_mermaid_test.rs"
required-features = ["playground"]
[features]
default = []
playground = ["dep:crossterm", "dep:xai-ratatui-textarea"]
[[bench]]
name = "bench"
harness = false

1312
crates/codegen/xai-grok-markdown/assets/tokyo-night.tmTheme Normal file
View file

File diff suppressed because it is too large Load diff

603
crates/codegen/xai-grok-markdown/benches/bench.rs Normal file
View file

@ -0,0 +1,603 @@
//! Benchmarks for markdown rendering.
use std::hint::black_box;
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use xai_grok_markdown::{
MarkdownStyle, StreamingMarkdownRenderer, Syntect, render_markdown_ratatui_full,
};
/// Default style for benchmarking.
fn default_style() -> MarkdownStyle {
MarkdownStyle::default()
}
/// Create syntect highlighter for benchmarking.
fn create_syntect() -> Syntect {
Syntect::new(include_bytes!("../assets/tokyo-night.tmTheme"))
}
fn bench_render_markdown(c: &mut Criterion) {
let syntect = create_syntect();
c.bench_function("render_markdown", |b| {
let content = black_box(
r#"# Heading 1
## Heading 2, can `contain` *formatted* **text**
### Heading 3
#### Heading 4
##### Heading 5
###### Heading 6
This also
=========
Works
-----
Some `inline`, **bold**, ~~strikethrough~~, *italic*, $math$.
- [ ] Task (not done)
* [x] Subtask (done)
- *Numbered* lists as well
1. One
2) Two
---
- [Link](https://example.com)
- [*Link*](https://foo.com "Title A") or [**Link**](https://bar.com 'Title B')
- ![alt](https://image.link) or ![alt](https://image.link "Image title")
- <https://www.markdownguide.org> or <fake@example.com>
***
```javascript
function hello() { // some code
console.log("Hello, world!");
}
```
> Multi-line
>
> Block-quote
> > and a *nested* one.
```
Plain fenced code block
```
$$
g(t) = \int_a^b K(t,s) f(s) ds
$$
This is *some <a href="https://google.com">inline html</a> block*.
<html>
<foo a="b">HTML block.</foo>
</html>
> [!NOTE]
> note quote
| H 1 | H 2 |
| ---- | ---- |
| C 1 | C 2 |
"#,
);
b.iter(|| {
render_markdown_ratatui_full(content, black_box(default_style()), true, Some(&syntect))
.0
.lines
.len()
})
});
}
/// Generate a markdown document with multiple blocks for streaming simulation.
fn generate_streaming_content(num_blocks: usize) -> String {
let mut content = String::new();
for i in 0..num_blocks {
match i % 5 {
0 => {
content.push_str(&format!("# Heading {}\n\n", i));
}
1 => {
content.push_str(&format!(
"This is paragraph {} with some **bold** and *italic* text.\n\n",
i
));
}
2 => {
content.push_str(&format!(
"```rust\nfn block_{}() {{\n // code\n}}\n```\n\n",
i
));
}
3 => {
content.push_str(&format!("> Quote block {}\n> More quoted text.\n\n", i));
}
4 => {
content.push_str(&format!(
"- Item {}.1\n- Item {}.2\n- Item {}.3\n\n",
i, i, i
));
}
_ => unreachable!(),
}
}
content
}
/// Benchmark streaming with full re-render on each token (O(N²) baseline).
fn bench_streaming_full_rerender(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming");
let syntect = create_syntect();
for num_blocks in [10, 50, 100] {
let content = generate_streaming_content(num_blocks);
let tokens: Vec<&str> = content.split_inclusive(char::is_whitespace).collect();
group.bench_with_input(
BenchmarkId::from_parameter(format!("{}/full", num_blocks)),
&tokens,
|b, tokens| {
b.iter(|| {
let mut text = String::new();
let mut total_lines = 0;
for token in tokens.iter() {
text.push_str(token);
let (output, _) = render_markdown_ratatui_full(
&text,
default_style(),
true,
Some(&syntect),
);
total_lines = output.lines.len();
}
black_box(total_lines)
});
},
);
}
group.finish();
}
/// Generate a hyperlink-heavy markdown document.
///
/// Each block contains 4 inline links and 1 autolink so the renderer's
/// link-translation path is exercised on most rendered lines. Designed
/// to surface O(lines * link_targets) costs in `translate_link_targets`.
fn generate_hyperlink_content(num_blocks: usize) -> String {
let mut content = String::new();
for i in 0..num_blocks {
content.push_str(&format!("# Section {}\n\n", i));
content.push_str(&format!(
"See [docs {i}](https://example.com/docs/{i}), [api {i}](https://example.com/api/{i}), [src {i}](https://github.com/x/r/blob/main/src{i}.rs), and [issue {i}](https://github.com/x/r/issues/{i}).\n\n",
i = i,
));
content.push_str(&format!(
"Reference link: <https://reference.example.com/path/segment-{}/page>.\n\n",
i
));
content.push_str(&format!(
"Mixed prose with [click](https://a{i}.com) and [click](https://b{i}.com) repeating the same text twice on one line for collision testing.\n\n",
i = i,
));
}
content
}
/// Benchmark a single full render of a hyperlink-heavy document.
///
/// Surfaces the cost of the parse-time `link_targets` collection plus the
/// post-render translation step. Pair with `bench_render_markdown` to see
/// the link-translation overhead in isolation.
fn bench_render_markdown_hyperlinks(c: &mut Criterion) {
let syntect = create_syntect();
let mut group = c.benchmark_group("render_markdown_hyperlinks");
for num_blocks in [10, 50, 200] {
let content = generate_hyperlink_content(num_blocks);
group.bench_with_input(
BenchmarkId::from_parameter(num_blocks),
&content,
|b, content| {
b.iter(|| {
let (out, _) = render_markdown_ratatui_full(
content,
black_box(default_style()),
true,
Some(&syntect),
);
(out.lines.len(), out.hyperlinks.len())
});
},
);
}
group.finish();
}
/// Benchmark incremental streaming of a hyperlink-heavy document.
///
/// Exercises `rerender_tail` repeatedly, which calls the link-translation
/// path on the unfrozen tail every push.
fn bench_streaming_hyperlinks_incremental(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming_hyperlinks");
let syntect = create_syntect();
for num_blocks in [10, 50] {
let content = generate_hyperlink_content(num_blocks);
let tokens: Vec<&str> = content.split_inclusive(char::is_whitespace).collect();
group.bench_with_input(
BenchmarkId::from_parameter(format!("{}/incremental", num_blocks)),
&tokens,
|b, tokens| {
b.iter(|| {
let mut renderer = StreamingMarkdownRenderer::new(default_style(), true);
let mut total_links = 0;
for token in tokens.iter() {
renderer.push_and_render(token, Some(&syntect));
total_links = renderer.view().hyperlinks.len();
}
black_box(total_links)
});
},
);
}
group.finish();
}
/// Benchmark streaming with incremental renderer (O(N) target).
fn bench_streaming_incremental(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming");
let syntect = create_syntect();
for num_blocks in [10, 50, 100] {
let content = generate_streaming_content(num_blocks);
let tokens: Vec<&str> = content.split_inclusive(char::is_whitespace).collect();
group.bench_with_input(
BenchmarkId::from_parameter(format!("{}/incremental", num_blocks)),
&tokens,
|b, tokens| {
b.iter(|| {
let mut renderer = StreamingMarkdownRenderer::new(default_style(), true);
let mut total_lines = 0;
for token in tokens.iter() {
renderer.push_and_render(token, Some(&syntect));
total_lines = renderer.view().lines.len();
}
black_box(total_lines)
});
},
);
}
group.finish();
}
/// Generate a math-heavy markdown document.
///
/// Each block exercises all four delimiter forms (`$...$`, `$$...$$`,
/// `\(...\)`, `\[...\]`) plus the expensive converter paths: scripts,
/// fractions, roots, symbol lookups, alphabets, and multi-row environments
/// (aligned / pmatrix / cases) that go through the MathBox 2D layout.
fn generate_math_content(num_blocks: usize) -> String {
let mut content = String::new();
for i in 0..num_blocks {
content.push_str(&format!("## Block {i}: norm \\(\\|x\\|_{{{i}}}\\)\n\n"));
content.push_str(&format!(
"Inline $e^{{i\\pi}} + {i} = \\frac{{\\alpha_{i}}}{{\\beta^2}}$ and \
\\(\\sqrt[3]{{x_{i}}} \\le \\mathbb{{R}}^n\\) mid-prose, then \
$\\sum_{{k=0}}^{{{i}}} \\binom{{n}}{{k}} \\approx 2^n$.\n\n",
));
content.push_str(&format!(
"$$\n\\int_0^{i} \\hat{{f}}(t) \\, dt = \\lim_{{n \\to \\infty}} \
\\frac{{{i}}}{{n+1}}\n$$\n\n",
));
content.push_str(&format!(
"\\[\n\\begin{{aligned}}\nf_{i}(x) &= x^{i} + \\gamma \\\\\n\
g_{i}(x) &= \\nabla f_{i} \\cdot \\vec{{v}}\n\\end{{aligned}}\n\\]\n\n",
));
content.push_str(
"\\[\nA = \\begin{pmatrix} 1 & 2 \\\\ 3 & 4 \\end{pmatrix}, \\quad \
|x| = \\begin{cases} x & x \\ge 0 \\\\ -x & x < 0 \\end{cases}\n\\]\n\n",
);
}
content
}
/// Benchmark a single full render of a math-heavy document.
///
/// Surfaces the cost of the LaTeX → Unicode converter plus the parse-time
/// `\(...\)` / `\[...\]` source scans and block replacements.
fn bench_render_markdown_math(c: &mut Criterion) {
let syntect = create_syntect();
let mut group = c.benchmark_group("render_markdown_math");
for num_blocks in [10, 50, 200] {
let content = generate_math_content(num_blocks);
group.bench_with_input(
BenchmarkId::from_parameter(num_blocks),
&content,
|b, content| {
b.iter(|| {
let (out, _) = render_markdown_ratatui_full(
content,
black_box(default_style()),
true,
Some(&syntect),
);
out.lines.len()
});
},
);
}
group.finish();
}
/// Benchmark incremental streaming of a math-heavy document.
///
/// Exercises the streaming hot path the `MAX_MATH_SOURCE_LEN` guard
/// protects: every push re-renders the unfrozen tail, re-running the math
/// scans and conversions on it.
fn bench_streaming_math_incremental(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming_math");
let syntect = create_syntect();
for num_blocks in [10, 50] {
let content = generate_math_content(num_blocks);
let tokens: Vec<&str> = content.split_inclusive(char::is_whitespace).collect();
group.bench_with_input(
BenchmarkId::from_parameter(format!("{}/incremental", num_blocks)),
&tokens,
|b, tokens| {
b.iter(|| {
let mut renderer = StreamingMarkdownRenderer::new(default_style(), true);
let mut total_lines = 0;
for token in tokens.iter() {
renderer.push_and_render(token, Some(&syntect));
total_lines = renderer.view().lines.len();
}
black_box(total_lines)
});
},
);
}
group.finish();
}
/// Generate a document with plain URLs in prose (no markdown link syntax).
fn generate_plain_url_content(num_blocks: usize) -> String {
let mut content = String::new();
for i in 0..num_blocks {
content.push_str(&format!(
"See https://example.com/page/{i} for details about topic {i}.\n\n",
));
}
content
}
/// Benchmark streaming + finish() of a plain-URL-heavy document.
///
/// Exercises the `detect_plain_urls` scan, which after the multi-line-URL
/// fix runs inside both `rerender_tail` (every `push_and_render`) and
/// `finish()`. Bench numbers from this point forward are not comparable
/// to historical runs that measured the prior `finish()`-only path.
fn bench_streaming_plain_urls_incremental(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming_plain_urls");
let syntect = create_syntect();
for num_blocks in [10, 50] {
let content = generate_plain_url_content(num_blocks);
let tokens: Vec<&str> = content.split_inclusive(char::is_whitespace).collect();
group.bench_with_input(
BenchmarkId::from_parameter(format!("{}/incremental", num_blocks)),
&tokens,
|b, tokens| {
b.iter(|| {
let mut renderer = StreamingMarkdownRenderer::new(default_style(), true);
for token in tokens.iter() {
renderer.push_and_render(token, Some(&syntect));
}
let view = renderer.finish(Some(&syntect));
black_box(view.hyperlinks.len())
});
},
);
}
group.finish();
}
/// Generate a realistic nested YAML document of roughly `num_lines` lines.
///
/// Produces nested keys, lists, and scalars (the kind of config an LLM streams
/// into a single fenced block) so the syntect highlighter does real work per
/// line rather than trivial whitespace.
fn generate_yaml_lines(num_lines: usize) -> Vec<String> {
let mut lines = Vec::with_capacity(num_lines);
let mut i = 0usize;
while lines.len() < num_lines {
lines.push(format!("service_{i}:"));
lines.push(format!(" name: \"service-{i}\""));
lines.push(" enabled: true".to_string());
lines.push(format!(" replicas: {}", i % 7 + 1));
lines.push(" resources:".to_string());
lines.push(format!(" cpu: \"{}m\"", (i % 4 + 1) * 250));
lines.push(format!(" memory: \"{}Mi\"", (i % 8 + 1) * 128));
lines.push(" env:".to_string());
lines.push(" - name: LOG_LEVEL".to_string());
lines.push(format!(
" value: \"{}\"",
if i.is_multiple_of(2) { "info" } else { "debug" }
));
lines.push(" - name: REGION".to_string());
lines.push(format!(" value: us-east-{}", i % 3 + 1));
lines.push(" ports:".to_string());
lines.push(format!(" - {}", 8000 + i));
lines.push(" tags:".to_string());
lines.push(format!(" team: team-{}", i % 5));
i += 1;
}
lines.truncate(num_lines);
lines
}
/// Benchmark streaming a SINGLE open ```yaml fenced block line-by-line WITHOUT
/// ever closing the fence.
///
/// This reproduces the UI-freeze pathology: while the fence is open the block
/// never checkpoints, so every `push_and_render` re-highlights the whole tail.
/// With the incremental open-code cache, per-line cost should stay roughly flat
/// in block size instead of growing linearly (overall O(N) instead of O(N²)).
fn bench_streaming_open_yaml_incremental(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming_open_yaml");
let syntect = create_syntect();
for num_lines in [100, 500, 1081] {
let lines = generate_yaml_lines(num_lines);
// Only the cache-on path ships; the parameter is just the line count
// (no A/B baseline here — the no-cache baseline was measured ad hoc and
// is not committed).
group.bench_with_input(
BenchmarkId::from_parameter(num_lines),
&lines,
|b, lines| {
b.iter(|| {
let mut renderer = StreamingMarkdownRenderer::new(default_style(), true);
// Open the fence but never close it.
renderer.push_and_render("```yaml\n", Some(&syntect));
let mut total_lines = 0;
for line in lines.iter() {
renderer.push_and_render(line, Some(&syntect));
renderer.push_and_render("\n", Some(&syntect));
total_lines = renderer.view().lines.len();
}
black_box(total_lines)
});
},
);
}
group.finish();
}
/// Closed scala fences inside bullet list items, then `trailing_words` more
/// streamed content in the same never-closing list. Lists can't checkpoint,
/// so the fences stay in the re-rendered tail for the whole stream
/// (~108 ms/token, ~4.5 s UI stall).
fn generate_fence_in_list_content(trailing_words: usize) -> String {
let mut s = String::new();
s.push_str("Here is where you're stuck:\n\n");
// Two list items embedding closed scala fences (the pathological shape).
for i in 0..2 {
s.push_str(&format!(
"- **[File{i}.scala:{}](https://example.com/f{i})** (domain)\n \
```scala\n \
final case class SampleRecord{i}(\n \
baseId: RecordId,\n \
payload: PersistedPayload,\n \
creationTimestamp: Instant,\n \
) extends RecordContent {{\n \
override lazy val raw: Growable[Raw] = ??? // TODO: implement\n \
}}\n \
```\n",
100 + i,
));
}
// Continued streaming within the same (never-closing) list context.
for w in 0..trailing_words {
if w % 12 == 0 {
s.push_str("\n- item: ");
}
s.push_str(&format!("word{w} "));
}
s.push('\n');
s
}
/// Control: same fences and trailing content at top level with blank lines,
/// so checkpoints advance past the fences.
fn generate_fence_top_level_content(trailing_words: usize) -> String {
let mut s = String::new();
s.push_str("Here is where you're stuck:\n\n");
for i in 0..2 {
s.push_str(&format!(
"```scala\nfinal case class SampleRecord{i}(\n \
baseId: RecordId,\n \
payload: PersistedPayload,\n \
creationTimestamp: Instant,\n) extends RecordContent {{\n \
override lazy val raw: Growable[Raw] = ??? // TODO: implement\n}}\n```\n\n",
));
}
for w in 0..trailing_words {
s.push_str(&format!("word{w} "));
if w % 12 == 11 {
s.push_str("\n\n");
}
}
s.push('\n');
s
}
/// Stream closed-fences-in-open-list token-by-token (`in_list`) against the
/// checkpoint-friendly `top_level` control. `in_list` must stay within a
/// small constant of `top_level`; unbounded growth in `trailing_words` is
/// the regression.
fn bench_streaming_fence_in_list(c: &mut Criterion) {
let mut group = c.benchmark_group("streaming_fence_in_list");
group.sample_size(10);
let syntect = create_syntect();
for trailing_words in [200, 400, 800] {
for (variant, content) in [
("in_list", generate_fence_in_list_content(trailing_words)),
(
"top_level",
generate_fence_top_level_content(trailing_words),
),
] {
let tokens: Vec<&str> = content.split_inclusive(char::is_whitespace).collect();
group.bench_with_input(
BenchmarkId::from_parameter(format!("{trailing_words}/{variant}")),
&tokens,
|b, tokens| {
b.iter(|| {
let mut renderer = StreamingMarkdownRenderer::new(default_style(), true);
let mut total_lines = 0;
for token in tokens.iter() {
renderer.push_and_render(token, Some(&syntect));
total_lines = renderer.view().lines.len();
}
black_box(total_lines)
});
},
);
}
}
group.finish();
}
criterion_group!(
benches,
bench_render_markdown,
bench_render_markdown_hyperlinks,
bench_render_markdown_math,
bench_streaming_full_rerender,
bench_streaming_incremental,
bench_streaming_hyperlinks_incremental,
bench_streaming_math_incremental,
bench_streaming_plain_urls_incremental,
bench_streaming_open_yaml_incremental,
bench_streaming_fence_in_list,
);
criterion_main!(benches);

325
crates/codegen/xai-grok-markdown/bin/md_mermaid_test.rs Normal file
View file

@ -0,0 +1,325 @@
//! Interactive Mermaid diagram rendering playground.
//!
//! Run with:
//! cargo run -p xai-grok-markdown --features playground --bin md-mermaid-test
//!
//! Controls:
//! Esc / Tab — toggle textarea focus
//! h / Left — shrink render width (when unfocused)
//! l / Right — grow render width (when unfocused)
//! n — next sample (when unfocused)
//! q / ^C / ^D — quit (when unfocused)
use std::io::{self, stdout};
use std::time::Duration;
use crossterm::ExecutableCommand;
use crossterm::event::{self, Event, KeyCode, KeyEvent, KeyModifiers};
use crossterm::terminal::{self, EnterAlternateScreen, LeaveAlternateScreen};
use ratatui::Terminal;
use ratatui::backend::CrosstermBackend;
use ratatui::layout::{Constraint, Layout, Rect};
use ratatui::style::{Color, Modifier, Style};
use ratatui::text::{Line, Span};
use ratatui::widgets::{Block, Borders, Paragraph, StatefulWidgetRef, Wrap};
use xai_grok_markdown::{
MarkdownBuffers, MarkdownStyle, render_markdown_ratatui_with_buffers_width,
};
use xai_ratatui_textarea::{TextArea, TextAreaState};
#[path = "playground_common.rs"]
mod playground_common;
use playground_common::{fg, get_syntect, md_style, rgb_color};
const MD_STYLE: MarkdownStyle = md_style(fg(rgb_color(192, 202, 245)));
const SAMPLES: &[&str] = &[
"```mermaid\nflowchart TD\n A[Start] --> B{Is it working?}\n B -->|Yes| C[Ship it]\n B -->|No| D[Debug]\n D --> B\n```\n",
"```mermaid\ngraph TD\n A[Client] --> B[Load Balancer]\n B --> C[Server 1]\n B --> D[Server 2]\n C --> E[(Database)]\n D --> E\n```\n",
"```mermaid\nflowchart LR\n A --> B --> C --> D\n```\n",
"```mermaid\nsequenceDiagram\n Alice->>Bob: Hello Bob\n Bob-->>Alice: Hi Alice\n```\n",
"```mermaid\nsequenceDiagram\n autonumber\n participant C as Client\n participant S as Server\n participant D as Database\n C->>S: GET /api/items\n S->>D: SELECT * FROM items\n D-->>S: rows\n S-->>C: 200 OK\n C->>C: render list\n Note over C,S: happy path\n loop retry x3\n C-x S: timeout\n end\n```\n",
"```mermaid\ngraph TD\n Start --> Stop\n```\n",
"```mermaid\nstateDiagram-v2\n [*] --> Idle\n Idle --> Loading: fetch\n Loading --> Ready: ok\n Loading --> Error: fail\n Error --> Idle: retry\n Ready --> [*]\n```\n",
"```mermaid\nflowchart TD\n A[Read config] & B[Load cache] --> C{Valid?}\n C -.->|no| D[Rebuild]\n C ==>|yes| E[Serve]\n D --o E\n E -->|poll| E\n```\n",
"```mermaid\ngraph TD\n C[ccc]\n D[ddd]\n A --> D\n B --> C\n D --> P[pp]\n C --> Q[qq]\n```\n",
"```mermaid\ngraph TD\n A --> D[ddd]\n A --> C[ccc]\n B --> C\n B --> D\n```\n",
"```mermaid\ngraph TD\n U[User] --> gw\n subgraph gw [Gateway]\n LB[load balancer] --> RL[rate limiter]\n end\n subgraph core [Services]\n API[api] --> W[worker]\n W --> Q[(queue)]\n end\n gw --> core\n core --> DB[(postgres)]\n```\n",
"```mermaid\nclassDiagram\n class Animal {\n <<abstract>>\n +int age\n +isMammal() bool\n +mate()\n }\n class Duck {\n +String beakColor\n +swim()\n }\n Animal <|-- Duck\n Animal <|-- Fish\n Duck *-- Bill\n Duck ..> Pond : swims in\n```\n",
"```mermaid\nerDiagram\n CUSTOMER ||--o{ ORDER : places\n ORDER ||--|{ LINE_ITEM : contains\n PRODUCT }o..o{ LINE_ITEM : \"is in\"\n CUSTOMER {\n string name PK\n int custNumber\n }\n ORDER {\n int orderNumber\n date placed\n }\n```\n",
];
fn render_full(source: &str, width: usize) -> Vec<Line<'static>> {
let mut buffers = MarkdownBuffers::new();
let (output, _) = render_markdown_ratatui_with_buffers_width(
source,
MD_STYLE,
true,
&mut buffers,
Some(get_syntect()),
Some(width),
);
output.lines
}
struct App {
textarea: TextArea,
textarea_state: TextAreaState,
textarea_focused: bool,
textarea_area: Rect,
render_width: usize,
sample: usize,
source: String,
lines: Vec<Line<'static>>,
}
impl App {
fn new() -> Self {
let sample = std::env::var("MERMAID_SAMPLE")
.ok()
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or(0)
% SAMPLES.len();
let width = std::env::var("MERMAID_WIDTH")
.ok()
.and_then(|s| s.parse::<usize>().ok())
.unwrap_or(70);
let mut textarea = TextArea::new();
textarea.set_text(SAMPLES[sample]);
textarea.show_scrollbar = false;
let source = SAMPLES[sample].to_string();
let lines = render_full(&source, width);
Self {
textarea,
textarea_state: TextAreaState::default(),
textarea_focused: false,
textarea_area: Rect::default(),
render_width: width,
sample,
source,
lines,
}
}
fn rerender(&mut self) {
self.source = self.textarea.text().to_string();
self.lines = render_full(&self.source, self.render_width);
}
fn adjust_width(&mut self, delta: isize) {
let new_w = (self.render_width as isize + delta).max(10) as usize;
if new_w != self.render_width {
self.render_width = new_w;
self.lines = render_full(&self.source, self.render_width);
}
}
fn next_sample(&mut self) {
self.sample = (self.sample + 1) % SAMPLES.len();
self.textarea.set_text(SAMPLES[self.sample]);
self.rerender();
}
}
fn main() -> io::Result<()> {
terminal::enable_raw_mode()?;
stdout().execute(EnterAlternateScreen)?;
let backend = CrosstermBackend::new(stdout());
let mut terminal = Terminal::new(backend)?;
let mut app = App::new();
loop {
terminal.draw(|f| draw(f, &mut app))?;
if event::poll(Duration::from_millis(100))? {
let ev = event::read()?;
match &ev {
Event::Key(KeyEvent {
code: KeyCode::Char('q'),
modifiers: KeyModifiers::CONTROL,
..
}) => break,
Event::Key(KeyEvent {
code: KeyCode::Esc | KeyCode::Tab,
..
}) if app.textarea_focused => {
app.textarea_focused = false;
}
Event::Key(KeyEvent {
code: KeyCode::Char('q'),
..
}) if !app.textarea_focused => break,
Event::Key(KeyEvent {
code: KeyCode::Char('c') | KeyCode::Char('d'),
modifiers: KeyModifiers::CONTROL,
..
}) if !app.textarea_focused => break,
Event::Key(KeyEvent {
code: KeyCode::Tab, ..
}) if !app.textarea_focused => {
app.textarea_focused = true;
}
Event::Key(KeyEvent {
code: KeyCode::Char('n'),
..
}) if !app.textarea_focused => app.next_sample(),
Event::Key(KeyEvent {
code: KeyCode::Char('h') | KeyCode::Left,
..
}) if !app.textarea_focused => app.adjust_width(-2),
Event::Key(KeyEvent {
code: KeyCode::Char('l') | KeyCode::Right,
..
}) if !app.textarea_focused => app.adjust_width(2),
Event::Key(key) if app.textarea_focused => {
app.textarea.input(*key);
app.rerender();
}
_ => {}
}
}
}
stdout().execute(LeaveAlternateScreen)?;
terminal::disable_raw_mode()?;
Ok(())
}
fn draw(f: &mut ratatui::Frame, app: &mut App) {
let size = f.area();
let render_w = app.render_width as u16;
let render_height = wrapped_line_count(&app.lines, render_w).max(1) + 2;
let header_height = 2u16;
let textarea_height = size
.height
.saturating_sub(header_height + render_height)
.max(6);
let chunks = Layout::vertical([
Constraint::Length(header_height),
Constraint::Length(textarea_height),
Constraint::Length(render_height),
])
.split(size);
let focus_indicator = if app.textarea_focused {
Span::styled(
" EDITING ",
Style::default().fg(Color::Black).bg(Color::Green),
)
} else {
Span::styled(
" VIEW ",
Style::default().fg(Color::Black).bg(Color::Yellow),
)
};
let width_info = Span::styled(
format!(
" width: {} | sample {}/{} ",
app.render_width,
app.sample + 1,
SAMPLES.len()
),
Style::default().fg(Color::Cyan),
);
let keys = if app.textarea_focused {
Span::styled(
" (live) Esc/Tab: defocus ",
Style::default().fg(Color::DarkGray),
)
} else {
Span::styled(
" Tab: edit | h/l: width | n: next sample | q/^C: quit ",
Style::default().fg(Color::DarkGray),
)
};
let header = Paragraph::new(vec![
Line::from(vec![focus_indicator, Span::raw(" "), width_info]),
Line::from(keys),
]);
f.render_widget(header, chunks[0]);
let border_color = if app.textarea_focused {
Color::Green
} else {
Color::DarkGray
};
let textarea_block = Block::default()
.borders(Borders::ALL)
.border_style(Style::default().fg(border_color))
.title(Span::styled(
" Mermaid Source ",
Style::default()
.fg(Color::White)
.add_modifier(Modifier::BOLD),
));
let textarea_inner = textarea_block.inner(chunks[1]);
f.render_widget(textarea_block, chunks[1]);
app.textarea_area = textarea_inner;
(&app.textarea).render_ref(textarea_inner, f.buffer_mut(), &mut app.textarea_state);
if app.textarea_focused
&& let Some((cx, cy)) = app
.textarea
.cursor_pos_with_state(textarea_inner, app.textarea_state)
{
f.set_cursor_position((cx, cy));
}
let title = format!(" rendered (inner width {}) ", app.render_width);
render_panel(f, chunks[2], &title, &app.lines, render_w);
}
fn wrapped_line_count(lines: &[Line<'_>], width: u16) -> u16 {
use unicode_width::UnicodeWidthStr;
if width == 0 {
return lines.len() as u16;
}
let w = width as usize;
lines
.iter()
.map(|line| {
let display_w: usize = line
.spans
.iter()
.map(|s| UnicodeWidthStr::width(s.content.as_ref()))
.sum();
if display_w == 0 {
1u16
} else {
display_w.div_ceil(w) as u16
}
})
.sum()
}
fn render_panel(
f: &mut ratatui::Frame,
area: Rect,
title: &str,
lines: &[Line<'static>],
inner_w: u16,
) {
let outer_w = (inner_w + 2).min(area.width);
let box_area = Rect {
x: area.x,
y: area.y,
width: outer_w,
height: area.height,
};
let block = Block::default()
.borders(Borders::ALL)
.border_style(Style::default().fg(Color::DarkGray))
.title(Span::styled(
title.to_string(),
Style::default()
.fg(Color::White)
.add_modifier(Modifier::BOLD),
));
let inner = block.inner(box_area);
f.render_widget(block, box_area);
let para = Paragraph::new(lines.to_vec()).wrap(Wrap { trim: false });
f.render_widget(para, inner);
}

468
crates/codegen/xai-grok-markdown/bin/md_table_test.rs Normal file
View file

@ -0,0 +1,468 @@
//! Interactive markdown table rendering playground.
//!
//! Run with:
//! cargo run -p xai-grok-markdown --features playground --bin md-table-test
//!
//! Controls:
//! Space — toggle textarea focus
//! Alt/Ctrl/Shift+Enter — submit markdown & defocus (when textarea focused)
//! h / Left — shrink render width (when unfocused)
//! l / Right — grow render width (when unfocused)
//! Esc — quit (always)
use std::io::{self, stdout};
use std::time::Duration;
use crossterm::ExecutableCommand;
use crossterm::event::{
self, DisableMouseCapture, EnableMouseCapture, Event, KeyCode, KeyEvent, KeyModifiers,
};
use crossterm::terminal::{self, EnterAlternateScreen, LeaveAlternateScreen};
use ratatui::Terminal;
use ratatui::backend::CrosstermBackend;
use ratatui::layout::{Constraint, Layout, Rect};
use ratatui::style::{Color, Modifier, Style};
use ratatui::text::{Line, Span};
use ratatui::widgets::{Block, Borders, Paragraph, StatefulWidgetRef, Wrap};
use xai_grok_markdown::{
MarkdownBuffers, MarkdownStyle, StreamingMarkdownRenderer,
render_markdown_ratatui_with_buffers_width,
};
use xai_ratatui_textarea::{TextArea, TextAreaState};
// ── Tokyo Night Storm palette (matches xai-grok-pager) ──────────────────────
#[path = "playground_common.rs"]
mod playground_common;
use playground_common::{get_syntect, md_style};
const MD_STYLE: MarkdownStyle = md_style(anstyle::Style::new());
// ── Compute minimum render width ─────────────────────────────────────────────
/// Minimum width = 4k + 1, where k = number of table columns.
/// Columns = max(`|` count per line) - 1 (the outer pipes are borders).
/// This ensures every column gets at least 1 char + padding. Floor of 10.
fn min_render_width(source: &str) -> usize {
let max_pipes = source
.lines()
.map(|line| line.chars().filter(|&c| c == '|').count())
.max()
.unwrap_or(0);
let num_cols = max_pipes.saturating_sub(1);
if num_cols == 0 {
10
} else {
(4 * num_cols + 1).max(10)
}
}
// ── Render helpers ───────────────────────────────────────────────────────────
/// One-shot full render at a given width.
fn render_full(source: &str, width: usize) -> Vec<Line<'static>> {
let mut buffers = MarkdownBuffers::new();
let (output, _) = render_markdown_ratatui_with_buffers_width(
source,
MD_STYLE,
true,
&mut buffers,
Some(get_syntect()),
Some(width),
);
output.lines
}
/// Streaming render: feed each char individually, return final lines.
fn render_streaming(source: &str, width: usize) -> Vec<Line<'static>> {
let mut renderer = StreamingMarkdownRenderer::new(MD_STYLE, true);
renderer.set_max_table_width(Some(width));
for ch in source.chars() {
renderer.push_and_render(&ch.to_string(), Some(get_syntect()));
}
renderer.view().lines.to_vec()
}
// ── App state ────────────────────────────────────────────────────────────────
const DEFAULT_MARKDOWN: &str = "\
| A | B | C |
|---|---|---|
| 1 | 2 | 3 |
";
struct App {
textarea: TextArea,
textarea_state: TextAreaState,
textarea_focused: bool,
textarea_area: Rect,
render_width: usize,
source: String,
full_lines: Vec<Line<'static>>,
streaming_lines: Vec<Line<'static>>,
}
impl App {
fn new() -> Self {
let mut textarea = TextArea::new();
textarea.set_text(DEFAULT_MARKDOWN);
textarea.show_scrollbar = false;
let source = DEFAULT_MARKDOWN.to_string();
let width = 24usize;
let full_lines = render_full(&source, width);
let streaming_lines = render_streaming(&source, width);
Self {
textarea,
textarea_state: TextAreaState::default(),
textarea_focused: true,
textarea_area: Rect::default(),
render_width: width,
source,
full_lines,
streaming_lines,
}
}
fn rerender(&mut self) {
self.source = self.textarea.text().to_string();
let min_w = min_render_width(&self.source);
if self.render_width < min_w {
self.render_width = min_w;
}
self.full_lines = render_full(&self.source, self.render_width);
self.streaming_lines = render_streaming(&self.source, self.render_width);
}
fn adjust_width(&mut self, delta: isize) {
let min_w = min_render_width(&self.source);
let new_w = (self.render_width as isize + delta).max(min_w as isize) as usize;
if new_w != self.render_width {
self.render_width = new_w;
self.full_lines = render_full(&self.source, self.render_width);
self.streaming_lines = render_streaming(&self.source, self.render_width);
}
}
}
// ── Main ─────────────────────────────────────────────────────────────────────
fn main() -> io::Result<()> {
// Terminal setup
terminal::enable_raw_mode()?;
stdout().execute(EnterAlternateScreen)?;
stdout().execute(EnableMouseCapture)?;
let backend = CrosstermBackend::new(stdout());
let mut terminal = Terminal::new(backend)?;
let mut app = App::new();
loop {
terminal.draw(|f| draw(f, &mut app))?;
if event::poll(Duration::from_millis(100))? {
let ev = event::read()?;
match &ev {
// ── Global: Ctrl-Q quits from anywhere ──
Event::Key(KeyEvent {
code: KeyCode::Char('q'),
modifiers: KeyModifiers::CONTROL,
..
}) => break,
// ── Focused: defocus on Esc or Tab ──
Event::Key(KeyEvent {
code: KeyCode::Esc | KeyCode::Tab,
..
}) if app.textarea_focused => {
app.textarea_focused = false;
}
// ── Unfocused: quit on q, Ctrl-C, Ctrl-D ──
Event::Key(KeyEvent {
code: KeyCode::Char('q'),
..
}) if !app.textarea_focused => break,
Event::Key(KeyEvent {
code: KeyCode::Char('c') | KeyCode::Char('d'),
modifiers: KeyModifiers::CONTROL,
..
}) if !app.textarea_focused => break,
// ── Unfocused: Space or Enter to re-focus ──
Event::Key(KeyEvent {
code: KeyCode::Char(' ') | KeyCode::Enter,
..
}) if !app.textarea_focused => {
app.textarea_focused = true;
}
// ── Unfocused: width controls ──
Event::Key(KeyEvent {
code: KeyCode::Char('h') | KeyCode::Left,
..
}) if !app.textarea_focused => {
app.adjust_width(-1);
}
Event::Key(KeyEvent {
code: KeyCode::Char('l') | KeyCode::Right,
..
}) if !app.textarea_focused => {
app.adjust_width(1);
}
Event::Key(KeyEvent {
code: KeyCode::Char('H'),
..
}) if !app.textarea_focused => {
app.adjust_width(-5);
}
Event::Key(KeyEvent {
code: KeyCode::Char('L'),
..
}) if !app.textarea_focused => {
app.adjust_width(5);
}
// ── Focused: forward keys to textarea, live re-render ──
Event::Key(key) if app.textarea_focused => {
app.textarea.input(*key);
app.rerender();
}
// ── Focused: forward mouse to textarea ──
Event::Mouse(mouse) if app.textarea_focused => {
app.textarea
.handle_mouse(*mouse, app.textarea_area, app.textarea_state);
}
_ => {}
}
}
}
// Terminal cleanup
stdout().execute(DisableMouseCapture)?;
stdout().execute(LeaveAlternateScreen)?;
terminal::disable_raw_mode()?;
Ok(())
}
// ── Drawing ──────────────────────────────────────────────────────────────────
fn draw(f: &mut ratatui::Frame, app: &mut App) {
let size = f.area();
// Layout: header (1) | textarea (flexible) | full render | streaming render
//
// We compute the heights we need for the render panels, then give the rest
// to the textarea.
let render_w = app.render_width as u16;
let full_height = wrapped_line_count(&app.full_lines, render_w).max(1) + 2; // +2 for border
let stream_height = wrapped_line_count(&app.streaming_lines, render_w).max(1) + 2;
// Detect mismatches between full and streaming
let mismatch = detect_mismatch(&app.full_lines, &app.streaming_lines);
let header_height = 2u16;
let render_height = full_height + stream_height;
let textarea_min = 5u16;
let textarea_height = size
.height
.saturating_sub(header_height + render_height)
.max(textarea_min);
let chunks = Layout::vertical([
Constraint::Length(header_height),
Constraint::Length(textarea_height),
Constraint::Length(full_height),
Constraint::Length(stream_height),
])
.split(size);
// ── Header ──
let focus_indicator = if app.textarea_focused {
Span::styled(
" EDITING ",
Style::default().fg(Color::Black).bg(Color::Green),
)
} else {
Span::styled(
" VIEW ",
Style::default().fg(Color::Black).bg(Color::Yellow),
)
};
let width_info = Span::styled(
format!(
" width: {} (min: {}) ",
app.render_width,
min_render_width(&app.source)
),
Style::default().fg(Color::Cyan),
);
let keys = if app.textarea_focused {
Span::styled(
" (live) Esc/Tab: defocus ",
Style::default().fg(Color::DarkGray),
)
} else {
Span::styled(
" Space/Enter: edit | h/l: width -/+ | H/L: -5/+5 | q/^C/^D: quit ",
Style::default().fg(Color::DarkGray),
)
};
let mismatch_indicator = if mismatch {
Span::styled(
" MISMATCH! ",
Style::default()
.fg(Color::White)
.bg(Color::Red)
.add_modifier(Modifier::BOLD),
)
} else {
Span::styled(" OK ", Style::default().fg(Color::Black).bg(Color::Green))
};
let header = Paragraph::new(vec![
Line::from(vec![
focus_indicator,
Span::raw(" "),
width_info,
Span::raw(" "),
mismatch_indicator,
]),
Line::from(keys),
]);
f.render_widget(header, chunks[0]);
// ── Textarea ──
let border_color = if app.textarea_focused {
Color::Green
} else {
Color::DarkGray
};
let textarea_block = Block::default()
.borders(Borders::ALL)
.border_style(Style::default().fg(border_color))
.title(Span::styled(
" Markdown Input ",
Style::default()
.fg(Color::White)
.add_modifier(Modifier::BOLD),
));
let textarea_inner = textarea_block.inner(chunks[1]);
f.render_widget(textarea_block, chunks[1]);
app.textarea_area = textarea_inner;
(&app.textarea).render_ref(textarea_inner, f.buffer_mut(), &mut app.textarea_state);
if app.textarea_focused
&& let Some((cx, cy)) = app
.textarea
.cursor_pos_with_state(textarea_inner, app.textarea_state)
{
f.set_cursor_position((cx, cy));
}
// ── Full render panel ──
let full_title = format!(" full: {} ", app.render_width);
render_panel(f, chunks[2], &full_title, &app.full_lines, render_w, false);
// ── Streaming render panel ──
let stream_title = format!(" stream: {} ", app.render_width);
render_panel(
f,
chunks[3],
&stream_title,
&app.streaming_lines,
render_w,
mismatch,
);
}
/// Count the number of visual rows a set of lines occupies when soft-wrapped
/// to `width` columns. Each line takes ceil(display_width / width) rows,
/// with a minimum of 1 row per line (empty lines still occupy a row).
fn wrapped_line_count(lines: &[Line<'_>], width: u16) -> u16 {
use unicode_width::UnicodeWidthStr;
if width == 0 {
return lines.len() as u16;
}
let w = width as usize;
lines
.iter()
.map(|line| {
let display_w: usize = line
.spans
.iter()
.map(|s| UnicodeWidthStr::width(s.content.as_ref()))
.sum();
if display_w == 0 {
1u16
} else {
display_w.div_ceil(w) as u16 // ceil division
}
})
.sum()
}
/// Render a bordered panel whose *inner* width is exactly `inner_w`.
///
/// Content is soft-wrapped with `Wrap { trim: false }` so long non-table
/// lines fold inside the box. The box is left-aligned within the available
/// `area`. If `is_error` is true the border turns red.
fn render_panel(
f: &mut ratatui::Frame,
area: Rect,
title: &str,
lines: &[Line<'static>],
inner_w: u16,
is_error: bool,
) {
// The block border adds 1 column on each side, so outer width = inner_w + 2.
let outer_w = (inner_w + 2).min(area.width);
let box_area = Rect {
x: area.x,
y: area.y,
width: outer_w,
height: area.height,
};
let border_color = if is_error {
Color::Red
} else {
Color::DarkGray
};
let block = Block::default()
.borders(Borders::ALL)
.border_style(Style::default().fg(border_color))
.title(Span::styled(
title.to_string(),
Style::default()
.fg(Color::White)
.add_modifier(Modifier::BOLD),
));
let inner = block.inner(box_area);
f.render_widget(block, box_area);
let para = Paragraph::new(lines.to_vec()).wrap(Wrap { trim: false });
f.render_widget(para, inner);
}
/// Detect if full and streaming outputs differ in text content.
fn detect_mismatch(full: &[Line<'static>], streaming: &[Line<'static>]) -> bool {
if full.len() != streaming.len() {
return true;
}
for (f_line, s_line) in full.iter().zip(streaming.iter()) {
let f_text: String = f_line.spans.iter().map(|s| s.content.as_ref()).collect();
let s_text: String = s_line.spans.iter().map(|s| s.content.as_ref()).collect();
if f_text != s_text {
return true;
}
}
false
}

84
crates/codegen/xai-grok-markdown/bin/playground_common.rs Normal file
View file

@ -0,0 +1,84 @@
#![allow(dead_code)]
use xai_grok_markdown::{MarkdownStyle, Syntect};
pub const fn rgb_color(r: u8, g: u8, b: u8) -> anstyle::Color {
anstyle::Color::Rgb(anstyle::RgbColor(r, g, b))
}
pub const fn fg(color: anstyle::Color) -> anstyle::Style {
anstyle::Style::new().fg_color(Some(color))
}
pub const fn bg(color: anstyle::Color) -> anstyle::Style {
anstyle::Style::new().bg_color(Some(color))
}
pub const TEAL: anstyle::Color = rgb_color(26, 188, 156);
pub const BLUE: anstyle::Color = rgb_color(122, 162, 247);
pub const ORANGE: anstyle::Color = rgb_color(255, 158, 100);
pub const RED: anstyle::Color = rgb_color(247, 118, 142);
pub const GREEN: anstyle::Color = rgb_color(158, 206, 106);
pub const MAGENTA: anstyle::Color = rgb_color(187, 154, 247);
pub const YELLOW: anstyle::Color = rgb_color(224, 175, 104);
pub const CYAN: anstyle::Color = rgb_color(125, 207, 255);
pub const COMMENT: anstyle::Color = rgb_color(86, 95, 137);
pub const BG_DARK: anstyle::Color = rgb_color(31, 35, 53);
pub const HEADING_COLORS: [anstyle::Color; 6] = [TEAL, BLUE, ORANGE, RED, GREEN, MAGENTA];
pub const fn heading_styles(bold: bool, dimmed: bool, hidden: bool) -> [anstyle::Style; 6] {
let mut styles = [anstyle::Style::new(); 6];
let mut i = 0;
while i < HEADING_COLORS.len() {
styles[i] = fg(HEADING_COLORS[i]);
if bold {
styles[i] = styles[i].bold();
}
if dimmed {
styles[i] = styles[i].dimmed();
}
if hidden {
styles[i] = styles[i].hidden();
}
i += 1;
}
styles
}
pub const fn md_style(text: anstyle::Style) -> MarkdownStyle {
MarkdownStyle {
heading_inner: heading_styles(true, false, false),
heading_outer: heading_styles(false, true, true),
strong_inner: anstyle::Style::new().bold(),
strong_outer: anstyle::Style::new().dimmed().hidden(),
emphasis_inner: anstyle::Style::new().italic(),
emphasis_outer: anstyle::Style::new().dimmed().hidden(),
strikethrough_inner: anstyle::Style::new().strikethrough(),
strikethrough_outer: anstyle::Style::new().dimmed().hidden(),
inline_code_inner: fg(YELLOW).bold(),
inline_code_outer: fg(YELLOW).dimmed().hidden(),
blockquote_outer: fg(COMMENT).dimmed(),
task_checked: fg(CYAN),
task_unchecked: fg(BLUE).dimmed(),
list_item: fg(BLUE).dimmed(),
rule: fg(COMMENT),
link_outer: fg(COMMENT),
link_text: anstyle::Style::new().bold(),
link_url: fg(COMMENT),
link_title: fg(GREEN),
code_outer: fg(YELLOW).dimmed().hidden(),
code_language: fg(ORANGE).hidden(),
code_untagged: anstyle::Style::new(),
code_background: bg(BG_DARK),
table_outer: fg(BLUE).hidden(),
text,
math: anstyle::Style::new().italic(),
}
}
pub fn get_syntect() -> &'static Syntect {
use std::sync::OnceLock;
static SYNTECT: OnceLock<Syntect> = OnceLock::new();
SYNTECT.get_or_init(|| Syntect::new(include_bytes!("../assets/tokyo-night.tmTheme")))
}

4
crates/codegen/xai-grok-markdown/fuzz/.gitignore vendored Normal file
View file

@ -0,0 +1,4 @@
target
corpus
artifacts
coverage

1175
crates/codegen/xai-grok-markdown/fuzz/Cargo.lock generated Normal file
View file

File diff suppressed because it is too large Load diff

25
crates/codegen/xai-grok-markdown/fuzz/Cargo.toml Normal file
View file

@ -0,0 +1,25 @@
[package]
name = "xai-grok-markdown-fuzz"
version = "0.0.0"
publish = false
edition = "2021"
[workspace]
members = ["."]
[package.metadata]
cargo-fuzz = true
[dependencies]
libfuzzer-sys = "0.4"
[dependencies.xai-grok-markdown]
path = ".."
# All render modes: pretty/non-pretty × syntect/no-syntect × full/streaming
[[bin]]
name = "render_all"
path = "fuzz_targets/render_all.rs"
test = false
doc = false
bench = false

47
crates/codegen/xai-grok-markdown/fuzz/README.md Normal file
View file

@ -0,0 +1,47 @@
# Fuzzing xai-grok-markdown
Coverage-guided fuzzing for the markdown renderer using [cargo-fuzz](https://rust-fuzz.github.io/book/cargo-fuzz.html) (libFuzzer).
## Prerequisites
```bash
cargo install cargo-fuzz # if not already installed
rustup toolchain install nightly
```
## Targets
| Target | What it fuzzes |
|---|---|
| `render_all` | All 8 combos: `pretty × syntect × {full, streaming}` for every input |
Each iteration runs:
- `render_markdown_ratatui_full()` — 4 combos (pretty/non-pretty × syntect/no-syntect)
- `StreamingMarkdownRenderer` char-by-char — same 4 combos
## Running
From `crates/codegen/xai-grok-markdown`:
```bash
# Run indefinitely (Ctrl-C to stop):
cargo +nightly fuzz run render_all fuzz/corpus/render_all fuzz/seeds/render_all -- -max_len=16384
# Run for 5 minutes:
cargo +nightly fuzz run render_all fuzz/corpus/render_all fuzz/seeds/render_all -- -max_len=16384 -max_total_time=300
```
- `corpus/` — auto-generated inputs (gitignored)
- `seeds/` — hand-written seed inputs (checked in)
## Reproducing a crash
When a crash is found, the input is saved to `artifacts/render_all/crash-<hash>`. Reproduce it with:
```bash
cargo +nightly fuzz run render_all fuzz/artifacts/render_all/crash-<hash>
```
## Adding seed inputs
Drop `.txt` or `.md` files into `seeds/render_all/`. Good seeds cover distinct markdown features (tables, code blocks, emoji, nested lists, etc.) and help the fuzzer reach new code paths faster.

35
crates/codegen/xai-grok-markdown/fuzz/fuzz_targets/render_all.rs Normal file
View file

@ -0,0 +1,35 @@
#![no_main]
use libfuzzer_sys::fuzz_target;
use xai_grok_markdown::style::test_style::STYLE;
use xai_grok_markdown::{render_markdown_ratatui_full, StreamingMarkdownRenderer};
const CHUNK_SIZES: [usize; 3] = [1, 16, 32];
fuzz_target!(|data: &[u8]| {
let Ok(s) = std::str::from_utf8(data) else {
return;
};
// Full render: pretty / non-pretty
for pretty in [true, false] {
let _ = render_markdown_ratatui_full(s, STYLE, pretty, None);
}
// Streaming with rotating chunk sizes: pretty / non-pretty
for pretty in [true, false] {
let mut r = StreamingMarkdownRenderer::new(STYLE, pretty);
let mut pos = 0;
let mut ci = 0;
while pos < s.len() {
let mut end = (pos + CHUNK_SIZES[ci]).min(s.len());
// snap to char boundary
while end < s.len() && !s.is_char_boundary(end) {
end += 1;
}
r.push_and_render(&s[pos..end], None);
pos = end;
ci = (ci + 1) % CHUNK_SIZES.len();
}
}
});

337
crates/codegen/xai-grok-markdown/fuzz/seeds/render_all/bench.md Normal file
View file

@ -0,0 +1,337 @@
# 🚀 Architecture Overview — `xai-grok-pager` Rendering Engine
The **xai-grok-pager** rendering engine is built on a layered pipeline that transforms raw markdown into terminal-ready cells. This document covers every major subsystem — from markdown parsing through syntax highlighting, word wrapping, block layout, viewport clipping, and final buffer composition. Understanding these layers is critical for anyone profiling or optimising the renderer.
---
## 📐 The Rendering Pipeline
Every frame follows the same sequence of stages. Content flows **downward** through transforms, each adding structure:
1. **Markdown parsing** — `StreamingMarkdownRenderer` converts source text into a tree of styled `Line<'static>` spans. Code fences trigger **syntect** highlighting.
2. **Word wrapping** — `word_wrap_lines_with_joiners()` breaks logical lines into physical rows that fit the viewport width, tracking *joiners* (continuation markers like `↳`) for copy/paste fidelity.
3. **Block output** — `BlockContent::output()` packages wrapped lines into a `BlockOutput` with per-line metadata: background colour, joiner strings, and optional decorations.
4. **Entry rendering** — `EntryRenderer` composes the accent column (`┃`), left/right padding, and block content into a horizontal strip. Vertical padding (vpad) adds breathing room above and below.
5. **Viewport clipping** — `render_scrolled_entries_with_scratch()` walks the entry list, skips off-screen entries, and uses a `ScratchBuffer` to render partially-visible entries into a temp buffer before copying the visible slice.
6. **Buffer diff** — ratatui's `Terminal::flush()` diffs the old and new `Buffer` and emits only changed cells as escape sequences. This is **O(changed cells)**, not O(total cells).
> **💡 Key insight**: steps 1–3 are **cached** across frames. Only step 4–5 run every frame. Profiling should focus there.
### Performance characteristics
| Stage | Complexity | Cached? | Hot path? |
|---|---|---|---|
| Markdown parse | `O(n)` in source length | ✅ Yes, per-generation | ❌ No |
| Syntax highlight | `O(n)` with syntect DFA | ✅ Yes, per-generation | ❌ No |
| Word wrap | `O(lines × width)` | ✅ Yes, `(width, gen)` key | ❌ No |
| `BlockContent::output()` | `O(wrapped_lines)` | ✅ Via `WrapCache` | ⚠️ First call only |
| `EntryRenderer::render()` | `O(height × width)` cell writes | ❌ No | ✅ **Yes** |
| Scratch buffer copy | `O(visible_rows × width)` clones | ❌ No | ✅ **Yes** |
| Buffer diff + flush | `O(changed_cells)` | N/A | ✅ **Yes** |
---
## 🧱 Block Types and Their Render Cost
Each `RenderBlock` variant has different rendering characteristics. Here's a breakdown of the major block types with their typical content patterns and associated costs:
### `AgentMessageBlock` — the heaviest hitter 🔥
Agent messages contain **arbitrary markdown**: paragraphs, code blocks, tables, lists, inline formatting. A single agent response can easily exceed 200 wrapped lines. The `MarkdownContent` subsystem does the heavy lifting:
- `StreamingMarkdownRenderer::push_and_render()` incrementally parses and highlights
- `word_wrap_lines_with_joiners()` handles Unicode-aware line breaking with `unicode-width`
- Wide characters (CJK, emoji) consume 2 columns: `'🦀'.width() == 2`, `'λ'.width() == 1`
```rust
/// The core markdown-to-lines pipeline.
///
/// This function is called on every content mutation (push_chunk, finish)
/// and produces the canonical `Vec<Line<'static>>` that gets cached.
pub fn render_markdown(source: &str, pretty: bool) -> Vec<Line<'static>> {
let mut renderer = StreamingMarkdownRenderer::new(MD_STYLE, pretty);
renderer.push(source);
renderer.render(Some(get_syntect()));
renderer.view().lines.to_vec()
}
/// Word-wrap with joiner tracking for copy fidelity.
///
/// Each output line knows whether it's a continuation of the previous
/// logical line (joiner = Some("↳")) or a fresh line (joiner = None).
/// This matters for selection/copy: we strip joiners when copying.
pub fn word_wrap_lines_with_joiners(
lines: Vec<Line<'static>>,
max_width: usize,
) -> (Vec<Line<'static>>, Vec<Option<String>>) {
let mut wrapped = Vec::with_capacity(lines.len() * 2);
let mut joiners = Vec::with_capacity(lines.len() * 2);
for line in lines {
let line_width = line.width();
if line_width <= max_width {
wrapped.push(line);
joiners.push(None);
} else {
// Split at grapheme cluster boundaries respecting unicode width.
// This is the expensive path — O(spans × chars) per line.
let parts = split_line_at_width(&line, max_width);
for (i, part) in parts.into_iter().enumerate() {
wrapped.push(part);
joiners.push(if i > 0 { Some("↳".into()) } else { None });
}
}
}
(wrapped, joiners)
}
```
### `ThinkingBlock` — truncated by default
Thinking blocks render identically to agent messages but default to `DisplayMode::Truncated` (3 visible lines + `⋯ N more lines`). When expanded, they're as expensive as agent messages. The truncation logic runs *after* wrapping, so the full wrap cost is paid even when collapsed — a potential optimisation target.
### `ToolCallBlock` variants
| Variant | Collapsed height | Expanded cost | Notes |
|---|---|---|---|
| `Execute` | 1 line (command summary) | `O(output_lines)` | Bash output can be huge |
| `Read` | 1 line (path + line count) | `O(file_lines)` | Syntax-highlighted file content |
| `Edit` | 1 line (path + edit count) | `O(diff_lines)` | Diff hunks with `+`/`-` colouring |
| `ListDir` | 1 line (path) | `O(entries)` | Directory tree listing |
| `Search` | 1 line (pattern + count) | `O(matches)` | Grep results with context |
| `Other` | 1 line (tool name) | `O(output)` | Generic tool output |
### `UserPromptBlock` — lightweight ✨
User prompts are short (1–5 lines typically), render with a `┃` accent in `accent_user` colour, and are **never foldable**. They're the cheapest block to render.
---
## 🎨 The Accent Column and Colour Blending
The leftmost column of every entry shows a vertical accent bar `┃`. This serves as a visual type indicator:
- **User prompts**: `accent_user` (Tokyo Night blue, `#7aa2f7`)
- **Tool calls**: `accent_tool` / `accent_success` / `accent_error`
- **Thinking**: `accent_thinking` (purple, `#bb9af7`)
- **Running blocks**: animated wave effect 🌊
The animation uses `blend_color(bg, fg, brightness)` per-row per-frame:
```rust
/// Compute wave brightness for a single row at a given tick.
///
/// Returns a value in [0.2, 1.0] — never fully invisible.
/// The wave travels downward at WAVE_SPEED radians per tick.
pub fn wave_brightness(tick: u64, row: u16, wave_rows: u16, speed: f32) -> f32 {
let phase = (tick as f32 * speed) - (row as f32 * std::f32::consts::PI / wave_rows as f32);
let raw = (phase.sin() + 1.0) / 2.0; // normalize to [0, 1]
0.2 + raw * 0.8 // scale to [0.2, 1.0]
}
/// Linearly blend two RGB colours.
///
/// `opacity = 0.0` → pure `base`; `opacity = 1.0` → pure `color`.
/// Returns `None` if either colour isn't RGB (indexed colours can't blend).
pub fn blend_color(base: Color, color: Color, opacity: f32) -> Option<Color> {
match (base, color) {
(Color::Rgb(br, bg, bb), Color::Rgb(cr, cg, cb)) => {
let r = br as f32 + (cr as f32 - br as f32) * opacity;
let g = bg as f32 + (cg as f32 - bg as f32) * opacity;
let b = bb as f32 + (cb as f32 - bb as f32) * opacity;
Some(Color::Rgb(r as u8, g as u8, b as u8))
}
_ => None,
}
}
```
---
## 📦 The `ScratchBuffer` and Partial Rendering
When an entry is **partially visible** (clipped at top or bottom of the viewport), we can't render directly into the output buffer — we'd write cells outside the visible area. Instead:
1. Resize a reusable `ScratchBuffer` to the entry's full height
2. Render the complete entry into scratch
3. Copy only the visible rows (`skip_rows..skip_rows + visible_height`) into the output
This is the **cell-by-cell copy loop** — one of the hottest paths:
```rust
for dy in 0..visible_rows {
let src_y = skip_rows + dy;
let dst_y = dest_area.y + dy;
for dx in 0..dest_area.width {
if let Some(src_cell) = temp_buf.cell((dx, src_y))
&& let Some(dst_cell) = buf.cell_mut((dest_area.x + dx, dst_y))
{
dst_cell.clone_from(src_cell);
}
}
}
```
> **🔬 Optimisation opportunity**: `Cell::clone_from` copies `symbol: String` (24 bytes on stack + possible heap), `fg`, `bg`, `underline_color`, `modifier`, `skip`. A `memcpy`-based bulk row copy could be significantly faster for wide terminals. At `width=200`, that's 200 `clone_from` calls per visible row per frame — potentially 6000 calls for a 30-row viewport with top+bottom clipping.
---
## 🔤 Unicode Width Challenges
Terminal rendering must account for **variable-width characters**. The `unicode-width` crate provides `UnicodeWidthChar::width()` and `UnicodeWidthStr::width()`:
| Character | Example | `width()` | Notes |
|---|---|---|---|
| ASCII | `A`, `z`, `!` | 1 | Basic Latin |
| CJK Unified | `漢`, `字`, `中` | 2 | Chinese/Japanese/Korean ideographs |
| Fullwidth forms | `Ａ`, `Ｂ`, `１` | 2 | Fullwidth ASCII variants |
| Emoji | `🦀`, `🚀`, `🎨` | 2 | Most emoji are wide |
| Combining marks | `é` (e + ◌́) | 1 | Combining char has width 0 |
| Zero-width | ZWJ, ZWNJ | 0 | Used in emoji sequences like 👨‍👩‍👧‍👦 |
| Tab | `\t` | — | Not handled by unicode-width; we expand to spaces |
The word wrapper must **never split a wide character** across the column boundary. If a 2-cell-wide char would start at column `width - 1`, we must wrap it to the next line and pad the current line with a space.
Here's a stress test: `漢字テスト🦀🚀🎨` contains 5 double-width CJK chars (10 columns) plus 3 double-width emoji (6 columns) = 16 columns total. At `width = 10`, this wraps to 2 lines. At `width = 7`, it wraps to 3 lines with padding cells.
---
## 📊 Inline Code and Syntax Highlighting Deep Dive
Inline code uses backtick syntax: `HashMap<String, Vec<u8>>`, `Option<&'a mut T>`, `impl Fn(usize) -> bool`. Each inline code span gets a distinct background colour (`bg_code`) to visually separate it from prose. The renderer must:
1. Parse the backtick delimiter (single `` ` `` or double ``` `` ```)
2. Extract the code content
3. Apply `Style::default().bg(theme.bg_code).fg(theme.fg_code)`
4. Handle **nested formatting** — e.g., `**bold `code` bold**` where code is inside bold
Fenced code blocks trigger full **syntect** highlighting. The highlighting pipeline:
1. Look up the `SyntaxReference` by language identifier (`rust`, `python`, `typescript`, etc.)
2. Create a `HighlightLines` with the Tokyo Night theme
3. Iterate source lines, calling `highlight_line()` to get `Vec<(syntect::Style, &str)>`
4. Convert syntect styles to ratatui `Span` styles (mapping RGB colours)
5. Each line gets `Style::default().bg(theme.bg_dark)` as a block background
The syntect state machine is **line-stateful** — each line's highlighting depends on the parse state at the end of the previous line. This means we can't parallelise highlighting within a single code block, but we *can* cache the result.
---
## 🧪 Testing Patterns
The scrollback rendering has comprehensive snapshot tests using `insta`. Here's the typical pattern:
```python
# This is a Python code block to exercise a different syntax highlighter.
# The renderer must detect the language and switch syntect grammars.
import asyncio
from dataclasses import dataclass, field
from typing import Optional, Dict, List, Tuple
@dataclass
class TrainingConfig:
"""Configuration for a distributed training run. 🔧"""
model_name: str
batch_size: int = 32
learning_rate: float = 3e-4
max_epochs: int = 100
gradient_accumulation_steps: int = 1
warmup_ratio: float = 0.1
weight_decay: float = 0.01
devices: List[str] = field(default_factory=lambda: ["cuda:0"])
mixed_precision: bool = True
compile_model: bool = False # torch.compile — can 2× throughput
checkpoint_dir: Optional[str] = None
@property
def effective_batch_size(self) -> int:
return self.batch_size * self.gradient_accumulation_steps * len(self.devices)
def validate(self) -> None:
assert self.batch_size > 0, f"batch_size must be positive, got {self.batch_size}"
assert 0 < self.learning_rate < 1, f"learning_rate out of range: {self.learning_rate}"
assert self.max_epochs > 0, f"max_epochs must be positive, got {self.max_epochs}"
for device in self.devices:
assert device.startswith(("cuda", "cpu")), f"unknown device: {device}"
async def train_epoch(
model,
dataloader,
optimizer,
scheduler,
config: TrainingConfig,
epoch: int,
) -> Dict[str, float]:
"""Run a single training epoch. Returns metrics dict. 📈"""
model.train()
total_loss = 0.0
num_batches = 0
for batch_idx, batch in enumerate(dataloader):
# Forward pass — compute loss on this micro-batch
outputs = model(**batch)
loss = outputs.loss / config.gradient_accumulation_steps
loss.backward()
if (batch_idx + 1) % config.gradient_accumulation_steps == 0:
optimizer.step()
scheduler.step()
optimizer.zero_grad()
total_loss += loss.item() * config.gradient_accumulation_steps
num_batches += 1
avg_loss = total_loss / max(num_batches, 1)
return {"epoch": epoch, "avg_loss": avg_loss, "num_batches": num_batches}
```
---
## ⚡ Benchmarking Strategy
To measure render performance, we need to isolate the **per-frame** cost from one-time setup:
- **Setup** (not measured): Parse markdown, create `ScrollbackEntry`, compute initial wrap cache
- **Measured**: For each scroll offset `0..total_height`, render into a `Buffer` of size `width × viewport_height`
This simulates a user holding down `j` (scroll down) and measures the **worst case** — every frame re-renders the viewport at a new scroll position, exercising:
- `EntryRenderer::render()` — accent, padding, content layout
- `BlockRenderer::render()` — vpad, content lines, background fills
- Partial rendering via `ScratchBuffer` — for clipped entries at top/bottom
- Cell-by-cell copy — the innermost hot loop
### Expected results
On a modern machine (M2 Pro), we expect:
- **~50–200 µs/frame** for a 120×30 viewport with a 200-line markdown document
- **~80% of time** in `EntryRenderer::render()` + scratch buffer copy
- **~15% of time** in `BlockContent::output()` (cache hit path — just iterating cached lines)
- **~5% of time** in layout computation (`HorizontalLayout`, `EntryLayout`, gap math)
If the benchmark shows >500 µs/frame, there's likely an unexpected cache miss or allocation in the hot path. Use `cargo bench -- --profile-time 10` with `flamegraph` to identify the culprit.
---
## 🌐 Miscellaneous Wide Characters and Edge Cases
Here are some strings that exercise interesting rendering edge cases:
- **Emoji sequences**: 👨‍👩‍👧‍👦 (family ZWJ sequence, should be width 2 but terminal support varies)
- **Flags**: 🇺🇸 🇯🇵 🇩🇪 (regional indicator pairs)
- **Fullwidth**: `ＡＢＣＤＥ` (each char is 2 columns wide)
- **Combining**: `naïve` vs `naïve` (precomposed U+00EF vs combining U+0308)
- **Box drawing**: `┌─────────┐│ content │└─────────┘` (all width 1)
- **Mathematical**: `∀x ∈ ℝ : x² ≥ 0`, `∑_{i=0}^{n} aᵢ = S`, `∫₀^∞ e^{-x} dx = 1`
- **CJK mixed**: `これはテストです — this is a test — 這是測試 — 이것은 시험이다`
- **RTL markers**: `Hello ‮dlrow‬!` (contains RLO/PDF override characters)
The renderer must handle all of these without panicking or producing garbled output. The word wrapper is the critical component — it must correctly account for each character's display width when deciding where to break lines.
> **⚠️ Warning**: Some terminals render emoji sequences incorrectly (showing them as 1-wide or as multiple glyphs). Our renderer uses `unicode-width` which reports the **Unicode standard** width, not the terminal's actual rendering width. This is a known source of misalignment — there is no perfect solution without querying the terminal.
---
*Generated for benchmarking purposes. Total: ~230 lines of rich markdown content with multiple code blocks, tables, inline code, emoji, wide Unicode characters, and varied formatting.*

5
crates/codegen/xai-grok-markdown/fuzz/seeds/render_all/code_block.txt Normal file
View file

@ -0,0 +1,5 @@
# Heading
```rust
fn main() {}
```

5
crates/codegen/xai-grok-markdown/fuzz/seeds/render_all/inline.txt Normal file
View file

@ -0,0 +1,5 @@
[link](https://example.com) ![alt](img.png)
***bold italic*** ~~strike~~
`code` **`bold code`** *`italic code`*

12
crates/codegen/xai-grok-markdown/fuzz/seeds/render_all/lists.txt Normal file
View file

@ -0,0 +1,12 @@
# H1
## H2
### H3
---
1. one
2. two
3. three
- [ ] todo
- [x] done

29
crates/codegen/xai-grok-markdown/fuzz/seeds/render_all/math.md Normal file
View file

@ -0,0 +1,29 @@
# Math seed
Euler's identity: $e^{i\pi} + 1 = 0$ and a fraction $\frac{a+b}{2}$.
Paren form \(\alpha_1 + \beta^2 \le \gamma\) inline.
$$
\int_0^\infty e^{-x}\,dx = 1
$$
\[
\begin{aligned}
f(x) &= \sqrt{x^2 + 1} \\
g(x) &= \begin{cases} x & x \ge 0 \\ -x & x < 0 \end{cases}
\end{aligned}
\]
Matrix: $$\begin{pmatrix} 1 & 2 \\ 3 & 4 \end{pmatrix}$$
| Col | Math |
|-----|------|
| a | $x^2$ |
- item \(p \to q\)
- sets $S \subseteq \mathbb{R}^n$
> quote $$E = mc^2$$
Escapes: \\(not math\\) and `$code$` and \[unterminated

5
crates/codegen/xai-grok-markdown/fuzz/seeds/render_all/mixed.txt Normal file
View file

@ -0,0 +1,5 @@
> **bold** *italic* `code`
- item 1
- nested
- deep

3
crates/codegen/xai-grok-markdown/fuzz/seeds/render_all/table.txt Normal file
View file

@ -0,0 +1,3 @@
| a | b |
|---|---|
| 1 | 2 |

5
crates/codegen/xai-grok-markdown/fuzz/seeds/render_all/thematic_emoji.txt Normal file
View file

@ -0,0 +1,5 @@
---
## 📐 Heading
Text after thematic break with emoji.

7
crates/codegen/xai-grok-markdown/fuzz/seeds/render_all/unicode.txt Normal file
View file

@ -0,0 +1,7 @@
漢字テスト ＡＢＣＤＥ 👨‍👩‍👧‍👦 ∀x ∈ ℝ
---
## 📐 Wide chars
Ü ö ñ é à ß µ ∞ ≠ ≤ ≥ ÷ × ← → ↑ ↓

373
crates/codegen/xai-grok-markdown/src/buffers.rs Normal file
View file

@ -0,0 +1,373 @@
//! Reusable buffers and internal data types for markdown parsing and rendering.
//!
//! This module contains all the intermediate data structures used by
//! MarkdownHighlighter during parsing and rendering.
use std::ops::Range;
use anstyle::Style as AnsiStyle;
use ratatui::text::{Line, Span};
use syntect::highlighting::Style as SyntectStyle;
/// A range of text with optional styling.
#[derive(Debug, Clone)]
pub struct Highlight {
pub style: Option<AnsiStyle>,
pub range: Range<usize>,
}
/// Syntax-highlighted code block replacement.
///
/// Stores the raw highlighted spans per line (intermediate representation).
/// This allows rendering to either ANSI strings or ratatui Lines on demand.
#[derive(Debug, Clone)]
pub struct Replace {
/// Raw highlighted spans per line: Vec<(style, text)>.
/// Each inner Vec represents one line of the code block.
pub highlighted: Vec<Vec<(SyntectStyle, String)>>,
/// Source byte range this replaces.
pub range: Range<usize>,
}
/// Internal representation of a hyperlink target discovered during parsing.
///
/// Populated in the `Tag::Link` / `Tag::Image` arm of `MarkdownParser::on_start`.
/// Consumed during rendering to produce public `HyperlinkTarget`s in the output.
#[derive(Debug, Clone)]
pub struct LinkTarget {
/// Source byte range of the *link text* (not the full `[text](url)` span).
pub source_range: Range<usize>,
/// Destination URL.
pub url: String,
/// Monotonically increasing identifier assigned during parsing.
pub id: u32,
}
/// Parse-time record of a closed fenced code block.
///
/// Populated in the `Tag::CodeBlock` arm of `MarkdownParser`; consumed during
/// rendering (see `output::build_code_block_spans`) to produce the public
/// [`crate::CodeBlockSpan`] once the output line range is known. Only **closed**
/// fences are recorded — an unterminated trailing fence yields no entry.
#[derive(Debug, Clone)]
pub struct CodeBlockMeta {
/// Fence info string (e.g. `"mermaid"`), verbatim from pulldown-cmark.
pub info: String,
/// De-prefixed body content (container markers stripped, CRLF normalized) —
/// pulldown's merged body text, i.e. the clean code/diagram source.
pub body: String,
/// Source byte range of the fence body (delimiter lines excluded).
pub body_source_range: Range<usize>,
}
/// Text transformation for substituting characters (e.g., bullets).
#[derive(Debug, Clone)]
pub struct Transform {
/// Source byte range to transform.
pub(crate) range: Range<usize>,
/// Replacement text.
pub(crate) to: String,
/// Apply this transform even in raw (non-pretty) mode.
///
/// Invariant: `to.len() == range.end - range.start` and the
/// substitution must stay valid UTF-8 at the same byte offsets.
/// `render_ansi` substitutes force transforms in place into a byte
/// buffer; violating the invariant panics at `copy_from_slice` or
/// `String::from_utf8` before any bytes escape the renderer.
pub(crate) force: bool,
}
/// A styled segment within a table cell.
#[derive(Debug, Clone)]
pub struct CellSpan {
pub text: String,
pub bold: bool,
pub italic: bool,
pub code: bool,
/// Hyperlink (url, id) when this span is inside a `[label](url)` link
/// or autolink inside a table cell. `None` for plain text.
pub link: Option<(String, u32)>,
}
impl CellSpan {
pub fn new(
text: String,
bold: bool,
italic: bool,
code: bool,
link: Option<(String, u32)>,
) -> Self {
Self {
text,
bold,
italic,
code,
link,
}
}
}
/// A table cell with styled content.
#[derive(Debug, Clone, Default)]
pub struct StyledCell {
pub spans: Vec<CellSpan>,
}
impl StyledCell {
pub fn new() -> Self {
Self { spans: Vec::new() }
}
/// Get plain text content (for width calculation).
pub fn plain_text(&self) -> String {
self.spans.iter().map(|s| s.text.as_str()).collect()
}
/// Clear the cell content.
pub fn clear(&mut self) {
self.spans.clear();
}
}
/// State for buffering table content during parsing.
#[derive(Debug, Clone)]
pub struct TableState {
/// Column alignments from the table header.
pub alignments: Vec<pulldown_cmark::Alignment>,
/// Header row cells.
pub header: Vec<StyledCell>,
/// Body rows (each row is a Vec of styled cells).
pub rows: Vec<Vec<StyledCell>>,
/// Current row being built.
pub current_row: Vec<StyledCell>,
/// Current cell content being accumulated.
pub current_cell: StyledCell,
/// Current style state for the cell.
pub cell_bold: bool,
pub cell_italic: bool,
pub cell_code: bool,
/// Current link state: `Some((url, id))` while inside a `Tag::Link` /
/// `Tag::Image` inside a table cell. Text events captured while this
/// is set produce link-tagged `CellSpan`s so the table renderer can
/// apply link styling and emit `HyperlinkTarget`s.
pub cell_link: Option<(String, u32)>,
/// Whether we're in the header section.
pub in_header: bool,
/// Source byte range of the entire table.
pub range: Range<usize>,
}
impl TableState {
pub fn new(alignments: Vec<pulldown_cmark::Alignment>, start: usize) -> Self {
Self {
alignments,
header: Vec::new(),
rows: Vec::new(),
current_row: Vec::new(),
current_cell: StyledCell::new(),
cell_bold: false,
cell_italic: false,
cell_code: false,
cell_link: None,
in_header: false,
range: start..start,
}
}
/// Push text with current styling to the cell.
pub fn push_text(&mut self, text: &str) {
self.current_cell.spans.push(CellSpan::new(
text.to_string(),
self.cell_bold,
self.cell_italic,
self.cell_code,
self.cell_link.clone(),
));
}
}
/// One hyperlink target inside a formatted table.
///
/// Coordinates are local to the table's `styled_lines`:
/// `line_offset` indexes into `TableReplace::styled_lines`; the renderer
/// adds the current absolute line count to produce a public
/// `HyperlinkTarget`.
#[derive(Debug, Clone)]
pub struct TableHyperlink {
/// Index within `TableReplace::styled_lines`.
pub line_offset: usize,
/// Column range (display cells) on that line.
pub column_range: Range<usize>,
/// Destination URL.
pub url: String,
/// Stable identifier shared with the paragraph link path.
pub id: u32,
}
/// Formatted table replacement for pretty mode rendering.
#[derive(Debug, Clone)]
pub struct TableReplace {
/// Formatted table lines (plain strings for ANSI rendering).
pub lines: Vec<String>,
/// Styled table lines for ratatui rendering.
pub styled_lines: Vec<Line<'static>>,
/// Source byte range this replaces.
pub range: Range<usize>,
/// Per-rendered-line source offset from the table start.
///
/// Maps each entry in `styled_lines` to the source line offset
/// within the table (0 = header, 1 = separator, 2+ = body rows).
/// Used by the renderer to produce correct `line_source_map` entries
/// instead of the naive `table_start + line_idx` which overshoots
/// when the rendered table has more lines than the source (borders,
/// separators, wrapped cells).
pub line_source_offsets: Vec<usize>,
/// Hyperlinks for `[label](url)` / autolinks inside table cells.
///
/// The paragraph link path (`LinkTarget` -> `chunk_link_offsets`)
/// cannot project links onto a rendered table because the table
/// replace consumes the entire source range — no text chunk's
/// rendering walks over the link text. The parser instead emits
/// `TableHyperlink`s during table formatting with positions in
/// table-local coordinates; the renderer translates them to absolute
/// `HyperlinkTarget`s.
pub hyperlinks: Vec<TableHyperlink>,
}
/// Rendered Mermaid diagram replacement for pretty mode rendering.
#[derive(Debug, Clone)]
pub struct MermaidReplace {
/// Plain lines for ANSI rendering.
pub lines: Vec<String>,
/// Styled lines for ratatui rendering.
pub styled_lines: Vec<Line<'static>>,
/// Source byte range this replaces.
pub range: Range<usize>,
}
/// Calculate the display width of a string (accounting for Unicode).
pub fn unicode_display_width(s: &str) -> usize {
use unicode_width::UnicodeWidthStr;
s.width()
}
/// Polyfill for `str::floor_char_boundary` (stable in Rust 1.91+).
///
/// Snaps `index` down to the nearest UTF-8 char boundary in `s`. Indices
/// past the end of `s` are clamped to `s.len()`. Replace with the std
/// method once the workspace toolchain is bumped to 1.91+.
pub(crate) fn floor_char_boundary(s: &str, index: usize) -> usize {
let mut i = index.min(s.len());
while i > 0 && !s.is_char_boundary(i) {
i -= 1;
}
i
}
/// Polyfill for `str::ceil_char_boundary` (stable in Rust 1.91+).
///
/// Snaps `index` up to the nearest UTF-8 char boundary in `s`. Indices
/// past the end of `s` are clamped to `s.len()`. Replace with the std
/// method once the workspace toolchain is bumped to 1.91+.
pub(crate) fn ceil_char_boundary(s: &str, index: usize) -> usize {
let mut i = index.min(s.len());
while i < s.len() && !s.is_char_boundary(i) {
i += 1;
}
i
}
/// Event kind for the render loop.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
#[repr(u8)]
pub enum RenderEventKind {
Highlight = 0,
Replace = 1,
Table = 2,
Mermaid = 3,
}
/// Render event: marks where a highlight/replace/table starts or ends.
/// Derives Ord for sorting by (pos, kind, index, is_end).
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct RenderEvent {
pub pos: usize,
pub kind: RenderEventKind,
pub index: usize,
pub is_end: bool,
}
/// Reusable buffers for markdown highlighting and rendering.
///
/// All vectors are cleared (keeping capacity) between renders, eliminating
/// allocation overhead in the streaming hot path.
///
/// # Buffer Categories
///
/// **Parse output buffers** - populated during `run()`, read-only during `render()`:
/// - `highlights`: Style ranges for inline formatting
/// - `replaces`: Syntax-highlighted code blocks
/// - `transforms`: Character substitutions (e.g., bullets)
/// - `untagged_code_ranges`: Code blocks without language tags
/// - `table_replaces`: Formatted table replacements
///
/// **Render scratch buffers** - temporary storage during `render()`:
/// - `render_events`: Sorted event queue for the render loop
/// - `current_spans`: Building current line's spans
/// - `active_highlights`: Stack of active highlight indices
pub struct MarkdownBuffers {
// Parse output buffers (written by run(), read by render())
pub highlights: Vec<Highlight>,
pub replaces: Vec<Replace>,
pub transforms: Vec<Transform>,
pub untagged_code_ranges: Vec<Range<usize>>,
pub table_replaces: Vec<TableReplace>,
pub mermaid_replaces: Vec<MermaidReplace>,
pub link_targets: Vec<LinkTarget>,
/// Closed fenced code blocks, in document order (see [`CodeBlockMeta`]).
pub code_blocks: Vec<CodeBlockMeta>,
// Render scratch buffers (used only during render())
pub render_events: Vec<RenderEvent>,
pub current_spans: Vec<Span<'static>>,
pub active_highlights: Vec<usize>,
}
impl MarkdownBuffers {
pub fn new() -> Self {
Self {
highlights: Vec::new(),
replaces: Vec::new(),
transforms: Vec::new(),
untagged_code_ranges: Vec::new(),
table_replaces: Vec::new(),
mermaid_replaces: Vec::new(),
link_targets: Vec::new(),
code_blocks: Vec::new(),
render_events: Vec::new(),
current_spans: Vec::new(),
active_highlights: Vec::new(),
}
}
/// Clear all buffers, keeping allocated capacity.
pub fn clear(&mut self) {
self.highlights.clear();
self.replaces.clear();
self.transforms.clear();
self.untagged_code_ranges.clear();
self.table_replaces.clear();
self.mermaid_replaces.clear();
self.link_targets.clear();
self.code_blocks.clear();
self.render_events.clear();
self.current_spans.clear();
self.active_highlights.clear();
}
}
impl Default for MarkdownBuffers {
fn default() -> Self {
Self::new()
}
}

58
crates/codegen/xai-grok-markdown/src/checkpoint.rs Normal file
View file

@ -0,0 +1,58 @@
//! Checkpoint types for incremental markdown rendering.
//!
//! This module defines types for identifying stable boundaries in markdown text
//! where rendered output can be "frozen" and cached. Content before a checkpoint
//! will not change regardless of what text is appended after it.
//!
//! # Design
//!
//! Checkpoints are only created at **top-level** (depth=0) block boundaries. Blocks
//! nested inside lists, blockquotes, or tables cannot be checkpoints because the
//! outer container might continue.
//!
//! # Example
//!
//! ```text
//! # Heading <- Checkpoint after this (heading at depth=0)
//!
//! Paragraph text. <- Checkpoint after blank line (paragraph at depth=0)
//!
//! - List item <- NO checkpoint (inside list)
//! ```code``` <- NO checkpoint (code block inside list)
//! - Another item
//! <- Checkpoint here (list closed at depth=0)
//! ```
/// A position in the source text where rendered content can be frozen.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Checkpoint {
/// Byte offset in source text (exclusive end of frozen region).
/// Content in `text[..source_bytes]` can be cached.
pub source_bytes: usize,
/// Number of output lines that correspond to this checkpoint.
/// Lines `0..output_lines` can be frozen.
pub output_lines: usize,
/// What kind of block ended at this checkpoint.
pub kind: CheckpointKind,
}
/// The type of markdown block that created a checkpoint.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CheckpointKind {
/// A heading (any level: h1-h6)
Heading,
/// A paragraph followed by a blank line
Paragraph,
/// A fenced or indented code block
CodeBlock,
/// A blockquote that closed at top level
BlockQuote,
/// A list (ordered or unordered) that closed at top level
List,
/// A thematic break (horizontal rule: ---, ***, ___)
ThematicBreak,
/// A table that closed at top level
Table,
/// A raw HTML block
HtmlBlock,
}

287
crates/codegen/xai-grok-markdown/src/colors.rs Normal file
View file

@ -0,0 +1,287 @@
//! Terminal color support detection and color conversion utilities.
//!
//! This module provides functionality to detect the terminal's color capabilities
//! and downgrade RGB colors to the appropriate level when needed.
use std::sync::OnceLock;
use std::sync::atomic::{AtomicU8, Ordering};
use anstyle::{Ansi256Color, AnsiColor, Color, RgbColor};
/// The level of color support detected for the terminal.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Default)]
pub enum ColorLevel {
/// No color support (monochrome terminals)
None,
/// Basic 16-color ANSI support (colors 0-15)
Basic,
/// 256-color support (colors 0-255)
Ansi256,
/// 24-bit truecolor RGB support (16 million colors)
#[default]
TrueColor,
}
impl ColorLevel {
/// Returns true if at least basic color is supported.
pub fn has_color(self) -> bool {
self >= Self::Basic
}
/// Returns true if 256-color mode is supported.
pub fn has_256(self) -> bool {
self >= Self::Ansi256
}
/// Returns true if 24-bit truecolor is supported.
pub fn has_truecolor(self) -> bool {
self >= Self::TrueColor
}
}
static COLOR_LEVEL: OnceLock<ColorLevel> = OnceLock::new();
/// Detect the terminal's color support level.
///
/// This uses the `supports-color` crate which checks:
/// - `COLORTERM` environment variable (for truecolor detection)
/// - `TERM` environment variable
/// - Terminal-specific environment variables (like `ITERM_SESSION_ID`)
/// - Whether stdout is a TTY
///
/// The result is cached after the first call.
pub fn detect_color_level() -> ColorLevel {
*COLOR_LEVEL.get_or_init(|| {
// Explicit opt-out via NO_COLOR takes priority.
if std::env::var_os("NO_COLOR").is_some() {
return ColorLevel::None;
}
let level = match supports_color::on(supports_color::Stream::Stdout) {
// Not a TTY (tests, piped) — default to TrueColor.
// The pager is a TUI app that always runs inside a terminal;
// stdout may not be a TTY when the pager renders to stderr.
None => ColorLevel::TrueColor,
Some(level) => {
if level.has_16m {
ColorLevel::TrueColor
} else if level.has_256 {
ColorLevel::Ansi256
} else if level.has_basic {
ColorLevel::Basic
} else {
ColorLevel::None
}
}
};
// The `supports-color` crate relies on COLORTERM=truecolor, but
// tmux/SSH/mosh often strip that variable. When the crate reports
// only 256-color support, upgrade to TrueColor if we can identify
// a known truecolor-capable terminal via its env vars.
if level < ColorLevel::TrueColor && terminal_supports_truecolor() {
return ColorLevel::TrueColor;
}
level
})
}
/// Check whether the terminal emulator is known to support truecolor.
///
/// Used as a fallback when `COLORTERM` is missing (e.g. inside tmux or over
/// SSH). Checks terminal-specific env vars that survive session forwarding
/// even when `COLORTERM` and `TERM_PROGRAM` are stripped.
fn terminal_supports_truecolor() -> bool {
use std::env;
// TERM_PROGRAM is the most reliable signal (set by the emulator itself).
if let Ok(prog) = env::var("TERM_PROGRAM") {
let norm: String = prog
.trim()
.chars()
.filter(|c| !matches!(c, ' ' | '-' | '_' | '.'))
.map(|c| c.to_ascii_lowercase())
.collect();
// Every modern terminal except Apple Terminal supports truecolor.
if matches!(
norm.as_str(),
"iterm"
| "iterm2"
| "itermapp"
| "ghostty"
| "kitty"
| "wezterm"
| "alacritty"
| "warp"
| "warpterminal"
| "vscode"
) {
return true;
}
}
// Terminal-specific env vars that often survive tmux/SSH.
env::var("ITERM_SESSION_ID").is_ok()
|| env::var("ITERM_PROFILE").is_ok()
|| env::var("WEZTERM_VERSION").is_ok()
|| env::var("KITTY_WINDOW_ID").is_ok()
|| env::var("ALACRITTY_SOCKET").is_ok()
}
/// Process-wide upper bound on the effective color level, stored as the
/// `ColorLevel` declaration-order discriminant.
static COLOR_LEVEL_CAP: AtomicU8 = AtomicU8::new(ColorLevel::TrueColor as u8);
/// Set the process-wide upper bound on the effective color level. Pass
/// [`ColorLevel::TrueColor`] to remove the cap.
pub fn set_color_level_cap(cap: ColorLevel) {
COLOR_LEVEL_CAP.store(cap as u8, Ordering::Relaxed);
}
fn color_level_cap() -> ColorLevel {
match COLOR_LEVEL_CAP.load(Ordering::Relaxed) {
0 => ColorLevel::None,
1 => ColorLevel::Basic,
2 => ColorLevel::Ansi256,
_ => ColorLevel::TrueColor,
}
}
/// Get the current color level (detecting if not already done), bounded by
/// the process-wide cap (see [`set_color_level_cap`]).
pub fn get_color_level() -> ColorLevel {
detect_color_level().min(color_level_cap())
}
/// Override the color level (useful for testing or user preference).
///
/// Returns `Err` if the color level was already set.
#[allow(dead_code)]
pub fn set_color_level(level: ColorLevel) -> Result<(), ColorLevel> {
COLOR_LEVEL.set(level)
}
/// Convert an `anstyle::Color` to the appropriate level based on terminal support.
///
/// This will downgrade colors as needed:
/// - TrueColor terminals: pass through unchanged
/// - 256-color terminals: RGB colors are converted to closest ANSI 256 color
/// - Basic terminals: colors are converted to closest ANSI 16 color
/// - No color: returns None
pub fn adapt_color(color: Color) -> Option<Color> {
let level = get_color_level();
match level {
ColorLevel::None => None,
ColorLevel::TrueColor => Some(color),
ColorLevel::Ansi256 => Some(match color {
Color::Rgb(rgb) => Color::Ansi256(rgb_to_ansi256(rgb)),
other => other,
}),
ColorLevel::Basic => Some(match color {
Color::Rgb(rgb) => Color::Ansi(rgb_to_ansi16(rgb)),
Color::Ansi256(idx) => Color::Ansi(ansi256_to_ansi16(idx)),
Color::Ansi(ansi) => Color::Ansi(ansi),
}),
}
}
/// Convert an `anstyle::Style` to the appropriate color level.
pub fn adapt_style(style: anstyle::Style) -> anstyle::Style {
let fg = style.get_fg_color().and_then(adapt_color);
let bg = style.get_bg_color().and_then(adapt_color);
let effects = style.get_effects();
let mut new_style = anstyle::Style::new();
if let Some(fg) = fg {
new_style = new_style.fg_color(Some(fg));
}
if let Some(bg) = bg {
new_style = new_style.bg_color(Some(bg));
}
new_style | effects
}
/// Convert an RGB color to the closest ANSI 256-color palette entry.
pub fn rgb_to_ansi256(rgb: RgbColor) -> Ansi256Color {
anstyle_lossy::rgb_to_xterm(rgb)
}
/// Convert an RGB color to the closest basic ANSI 16-color.
pub fn rgb_to_ansi16(rgb: RgbColor) -> AnsiColor {
anstyle_lossy::rgb_to_ansi(rgb, anstyle_lossy::palette::VGA)
}
/// Convert an ANSI 256-color to the closest basic ANSI 16-color.
pub fn ansi256_to_ansi16(idx: Ansi256Color) -> AnsiColor {
anstyle_lossy::xterm_to_ansi(idx, anstyle_lossy::palette::VGA)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_rgb_to_ansi256_grayscale() {
// Pure black should map to near-black
let result = rgb_to_ansi256(RgbColor(0, 0, 0));
assert!(result.index() == 16 || result.index() >= 232);
// Pure white should map to near-white
let result = rgb_to_ansi256(RgbColor(255, 255, 255));
assert!(result.index() == 231 || result.index() == 255);
// Medium gray
let result = rgb_to_ansi256(RgbColor(128, 128, 128));
assert!(result.index() >= 232); // Should be in grayscale range
}
#[test]
fn test_rgb_to_ansi256_colors() {
// Pure red
let result = rgb_to_ansi256(RgbColor(255, 0, 0));
assert_eq!(result.index(), 196); // Bright red in the cube
// Pure green
let result = rgb_to_ansi256(RgbColor(0, 255, 0));
assert_eq!(result.index(), 46); // Bright green in the cube
// Pure blue
let result = rgb_to_ansi256(RgbColor(0, 0, 255));
assert_eq!(result.index(), 21); // Bright blue in the cube
}
#[test]
fn test_rgb_to_ansi16() {
// Test basic color mapping
let red = rgb_to_ansi16(RgbColor(200, 0, 0));
assert!(matches!(red, AnsiColor::Red | AnsiColor::BrightRed));
let green = rgb_to_ansi16(RgbColor(0, 200, 0));
assert!(matches!(green, AnsiColor::Green | AnsiColor::BrightGreen));
let blue = rgb_to_ansi16(RgbColor(0, 0, 200));
assert!(matches!(blue, AnsiColor::Blue | AnsiColor::BrightBlue));
// White
let white = rgb_to_ansi16(RgbColor(250, 250, 250));
assert!(matches!(white, AnsiColor::White | AnsiColor::BrightWhite));
}
#[test]
fn test_ansi256_to_ansi16_standard() {
// First 16 colors should map directly
assert_eq!(ansi256_to_ansi16(Ansi256Color(0)), AnsiColor::Black);
assert_eq!(ansi256_to_ansi16(Ansi256Color(1)), AnsiColor::Red);
assert_eq!(ansi256_to_ansi16(Ansi256Color(7)), AnsiColor::White);
assert_eq!(ansi256_to_ansi16(Ansi256Color(15)), AnsiColor::BrightWhite);
}
#[test]
fn test_color_level_ordering() {
assert!(ColorLevel::None < ColorLevel::Basic);
assert!(ColorLevel::Basic < ColorLevel::Ansi256);
assert!(ColorLevel::Ansi256 < ColorLevel::TrueColor);
}
}

784
crates/codegen/xai-grok-markdown/src/hyperlinks.rs Normal file
View file

@ -0,0 +1,784 @@
//! Project parser-emitted `LinkTarget`s onto rendered display cells.
//!
//! Three coordinate systems are in play:
//!
//! 1. **Source bytes** -- offsets into the raw markdown the parser saw.
//! `LinkTarget::source_range` lives here.
//! 2. **Transformed bytes** -- what `apply_transforms` produces for a *chunk*
//! of source bytes between two render events. In pretty mode the
//! transforms strip `[` and rewrite `](` as ` (`, so transformed bytes
//! do not line up with source bytes.
//! 3. **Display cells** -- `(line_index, display_column)`. What
//! `HyperlinkTarget` exposes for the OSC 8 layer to consume.
//!
//! A chunk's transformed string is split on `\n` into *segments*; one
//! segment becomes one rendered line. A link spanning multiple segments
//! (a wrapped or autolink-bracketed link) produces one `HyperlinkTarget`
//! per segment, all sharing the same `id`.
use crate::buffers::{
LinkTarget, Transform, ceil_char_boundary, floor_char_boundary, unicode_display_width,
};
use crate::output::HyperlinkTarget;
/// One link's projection onto the current chunk's transformed string.
///
/// Returned by `chunk_link_offsets`; bounds are in coordinate system #2
/// (transformed bytes within the chunk), to be mapped onto display cells
/// later by `emit_segment_hyperlinks`.
#[derive(Debug, Clone, Copy)]
pub(crate) struct ChunkLinkRange {
/// Start byte (inclusive) within the chunk's transformed string.
pub(crate) xform_start: usize,
/// End byte (exclusive) within the chunk's transformed string.
pub(crate) xform_end: usize,
/// Index into the `link_targets` slice the caller passed in.
pub(crate) link_idx: usize,
}
/// Project a source byte position into the chunk's transformed coordinate
/// space (system #1 -> system #2). See file docstring.
///
/// Walks `transforms` in source order, accumulating `(to.len() - range.len())`
/// for every transform fully consumed before `src_pos`. When the chunk has
/// no transforms (or `pretty` is false), the caller skips this and uses
/// `src_pos - chunk_start` directly.
///
/// **Invariants assumed of the inputs:**
/// 1. `transforms` is sorted by `range.start` (the existing `apply_transforms`
/// relies on the same invariant; the parser pushes transforms in source
/// order).
/// 2. No transform's source range overlaps the bytes a caller intends to
/// locate — i.e. transforms touch *boundary* characters around link text
/// (the `[` and `](` markers), never the link text itself. All transforms
/// pushed by the parser today (link bracket removal, bullet substitutions)
/// satisfy this; the `debug_assert!` at the call sites in `render_ratatui`
/// enforces it via the cursor invariant.
///
/// **Straddle policy** (when a transform DOES contain `src_pos` despite the
/// invariant above): the source position is clamped to the start of the
/// transform's replacement string. Both endpoints (start/end) clamp the
/// same direction, so a link whose endpoint straddles a transform produces
/// a column range that excludes the straddling bytes. This is intentional
/// rather than precise — a future transform that intentionally rewrites
/// link text should add a typed mapping instead of relying on this clamp.
pub(crate) fn source_to_chunk_offset(
src_pos: usize,
chunk_start: usize,
transforms: &[Transform],
) -> usize {
let mut delta: isize = 0;
for t in transforms {
if t.range.end <= chunk_start {
continue;
}
if t.range.start >= src_pos {
break;
}
let t_src_start = t.range.start.max(chunk_start);
if t.range.end <= src_pos {
let src_len = (t.range.end - t_src_start) as isize;
let dst_len = t.to.len() as isize;
delta += dst_len - src_len;
} else {
// Transform straddles src_pos. See "Straddle policy" above.
debug_assert!(
false,
"source_to_chunk_offset: transform [{}..{}) straddles src_pos {}; \
link text should never overlap a transform. See straddle policy.",
t.range.start, t.range.end, src_pos,
);
let consumed = (src_pos - t_src_start) as isize;
delta -= consumed;
break;
}
}
let raw = (src_pos - chunk_start) as isize + delta;
raw.max(0) as usize
}
/// One `ChunkLinkRange` per link whose source range overlaps
/// `[chunk_start, chunk_end)`.
///
/// `from_idx` is the caller's monotonic cursor: links before this index
/// have already been processed in earlier chunks (see the module doc on
/// the source-order invariant). Returned bounds live in the chunk's
/// transformed coordinate space.
pub(crate) fn chunk_link_offsets(
link_targets: &[LinkTarget],
from_idx: usize,
chunk_start: usize,
chunk_end: usize,
pretty: bool,
transforms: &[Transform],
) -> Vec<ChunkLinkRange> {
let mut out = Vec::new();
for (idx, lt) in link_targets.iter().enumerate().skip(from_idx) {
if lt.source_range.start >= chunk_end {
break;
}
if lt.source_range.end <= chunk_start {
continue;
}
let src_start = lt.source_range.start.max(chunk_start);
let src_end = lt.source_range.end.min(chunk_end);
let (xform_start, xform_end) = if !pretty || transforms.is_empty() {
(src_start - chunk_start, src_end - chunk_start)
} else {
(
source_to_chunk_offset(src_start, chunk_start, transforms),
source_to_chunk_offset(src_end, chunk_start, transforms),
)
};
if xform_end > xform_start {
out.push(ChunkLinkRange {
xform_start,
xform_end,
link_idx: idx,
});
}
}
out
}
/// Push one `HyperlinkTarget` per `ChunkLinkRange` that overlaps this
/// segment (system #2 -> system #3).
///
/// `seg_x_offset` is where this segment starts within the chunk's
/// transformed string; the caller advances it by `segment.len() + 1`
/// per iteration to account for the `\n` consumed by `split('\n')`.
/// `col` is the running display column on the in-progress line.
pub(crate) fn emit_segment_hyperlinks(
chunk_links: &[ChunkLinkRange],
link_targets: &[LinkTarget],
segment: &str,
seg_x_offset: usize,
col: usize,
line_index: usize,
out: &mut Vec<HyperlinkTarget>,
) {
let seg_x_end = seg_x_offset + segment.len();
for clr in chunk_links {
if clr.xform_end <= seg_x_offset || clr.xform_start >= seg_x_end {
continue;
}
let s_in = clr
.xform_start
.saturating_sub(seg_x_offset)
.min(segment.len());
let e_in = (clr.xform_end - seg_x_offset).min(segment.len());
let s_in = floor_char_boundary(segment, s_in);
let e_in = ceil_char_boundary(segment, e_in);
if s_in >= e_in {
continue;
}
let col_start = col + unicode_display_width(&segment[..s_in]);
let col_end = col_start + unicode_display_width(&segment[s_in..e_in]);
let lt = &link_targets[clr.link_idx];
out.push(HyperlinkTarget {
line_index,
column_range: col_start..col_end,
url: lt.url.clone(),
id: lt.id,
});
}
}
#[cfg(test)]
mod hyperlink_tests {
use crate::output::HyperlinkTarget;
use crate::style::test_style;
use crate::{StreamingMarkdownRenderer, render_markdown_ratatui_full};
use pretty_assertions::assert_eq;
use ratatui::text::Line;
fn line_to_string(line: &Line<'static>) -> String {
line.spans.iter().map(|s| s.content.as_ref()).collect()
}
/// Slice the rendered line by display-cell `column_range`. Display
/// width != char count for CJK and other wide characters, so we
/// accumulate width per char until we land inside the requested
/// range.
///
/// Zero-width chars (combining marks, ZWJ, control chars) at the
/// boundary are AMBIGUOUSLY attached: a zero-width char at exactly
/// `col == range.start` is included in the slice (it does not
/// advance `col`), while a zero-width char at `col == range.end`
/// is also included (it satisfies `end <= range.end`). No callers
/// in the test suite use combining marks today; if a future caller
/// needs to disambiguate, change the boundary condition to attach
/// zero-width chars to whichever side semantically owns the grapheme
/// cluster.
fn slice_by_cells(rendered: &str, range: std::ops::Range<usize>) -> String {
use unicode_width::UnicodeWidthChar;
let mut col = 0usize;
let mut out = String::new();
for ch in rendered.chars() {
let w = ch.width().unwrap_or(0);
let end = col + w;
if col >= range.start && end <= range.end {
out.push(ch);
} else if end > range.end {
break;
}
col = end;
}
out
}
/// Find the parser-produced (link-text) hyperlink — by convention the
/// one whose `column_range` slices to `expected_slice` in the
/// rendered output. Since `render_markdown_ratatui_full` now also
/// emits a url_scan target for the pretty-mode `(url)` suffix, tests
/// that previously checked `hyperlinks.len() == 1` must explicitly
/// pick the parser-produced entry.
fn parser_link_text<'a>(
out: &'a crate::output::MarkdownRenderOutput,
expected_slice: &str,
) -> &'a HyperlinkTarget {
out.hyperlinks
.iter()
.find(|h| {
let rendered = line_to_string(&out.lines[h.line_index]);
slice_by_cells(&rendered, h.column_range.clone()) == expected_slice
})
.unwrap_or_else(|| {
panic!(
"expected a hyperlink whose column range covers {expected_slice:?}; got {:?}",
out.hyperlinks,
)
})
}
/// `[link](url)` in pretty mode renders as `link (url)`. The
/// `HyperlinkTarget`'s column range must cover the rendered "link"
/// glyphs (4 cells), not include the stripped `[` or the rewritten ` (`.
#[test]
fn pretty_inline_link_column_range_excludes_brackets() {
let text = "Here is a [link](https://example.com) in text.\n";
let (out, _) = render_markdown_ratatui_full(text, test_style::STYLE, true, None);
// The parser produces one HyperlinkTarget over the link text;
// the url_scan pass produces a second over the `(url)` suffix.
let h = parser_link_text(&out, "link");
assert_eq!(h.url, "https://example.com");
let rendered = line_to_string(&out.lines[h.line_index]);
let slice: String = rendered
.chars()
.skip(h.column_range.start)
.take(h.column_range.len())
.collect();
assert_eq!(
slice, "link",
"column_range should cover only the link text glyphs"
);
}
/// In non-pretty mode the rendered text keeps `[link](url)` literally
/// in place, but the parser's `LinkTarget` source range still points at
/// just `link`. The column range therefore covers `link` (4 cells),
/// shifted by the leading `[` that's now visible in the output.
#[test]
fn non_pretty_inline_link_column_range_covers_text_not_brackets() {
let text = "[link](https://example.com)\n";
let (out, _) = render_markdown_ratatui_full(text, test_style::STYLE, false, None);
// Non-pretty: `[link](url)` is rendered literally; url_scan also
// finds the URL inside `(url)` so two hyperlinks are emitted.
let h = parser_link_text(&out, "link");
let rendered = line_to_string(&out.lines[h.line_index]);
let slice: String = rendered
.chars()
.skip(h.column_range.start)
.take(h.column_range.len())
.collect();
assert_eq!(slice, "link");
}
/// Two links with identical text on the same line MUST produce two
/// distinct `HyperlinkTarget`s with distinct URLs and disjoint column
/// ranges. This is the case the substring approach got wrong (both
/// would resolve to the first occurrence).
#[test]
fn duplicated_link_text_on_one_line_produces_distinct_targets() {
let text = "See [click](https://a.example) and [click](https://b.example) here.\n";
let (out, _) = render_markdown_ratatui_full(text, test_style::STYLE, true, None);
// Two parser-produced link-text hyperlinks (cover "click") plus
// two url_scan-produced hyperlinks for the `(url)` suffixes.
let click_targets: Vec<&HyperlinkTarget> = out
.hyperlinks
.iter()
.filter(|h| {
let rendered = line_to_string(&out.lines[h.line_index]);
let slice: String = rendered
.chars()
.skip(h.column_range.start)
.take(h.column_range.len())
.collect();
slice == "click"
})
.collect();
assert_eq!(
click_targets.len(),
2,
"two parser-produced link-text hyperlinks expected",
);
let urls: Vec<&str> = click_targets.iter().map(|h| h.url.as_str()).collect();
assert_eq!(urls, vec!["https://a.example", "https://b.example"]);
let h0 = click_targets[0];
let h1 = click_targets[1];
assert_eq!(
h0.line_index, h1.line_index,
"both links should be on the same rendered line"
);
assert_ne!(h0.id, h1.id, "ids must differ");
assert!(
h0.column_range.end <= h1.column_range.start,
"column ranges must be disjoint and in order, got {:?} vs {:?}",
h0.column_range,
h1.column_range,
);
}
/// CJK characters in link text consume 2 cells each. The column range
/// must reflect display width, not byte length (`日本語` is 9 bytes / 6 cells).
#[test]
fn cjk_link_uses_display_width_for_column_range() {
let text = "[日本語](https://example.com)\n";
let (out, _) = render_markdown_ratatui_full(text, test_style::STYLE, true, None);
// The parser produces one hyperlink over the CJK link text; the
// url_scan pass produces a second over the `(url)` suffix.
let h = parser_link_text(&out, "日本語");
assert_eq!(
h.column_range.len(),
6,
"三 wide CJK chars -> 6 display cells"
);
}
/// `<https://example.com>` autolink: parser records the source range
/// over the entire `<...>`-bounded text. Because pulldown-cmark fires
/// multiple sub-chunks within a single autolink (the `<`, the URL
/// text, and the `>`), the in-render translation may emit multiple
/// `HyperlinkTarget`s — but they MUST all share the same `id` and
/// `url`, and their column ranges must collectively cover the
/// rendered URL on a single line. This is the same semantic shape
/// as a link that wraps across two rendered lines.
#[test]
fn autolink_emits_grouped_targets_for_same_logical_link() {
let text = "Visit <https://example.com> for info.\n";
let (out, _) = render_markdown_ratatui_full(text, test_style::STYLE, true, None);
assert!(
!out.hyperlinks.is_empty(),
"expected at least one hyperlink"
);
let url = &out.hyperlinks[0].url;
let id = out.hyperlinks[0].id;
let line_index = out.hyperlinks[0].line_index;
for h in &out.hyperlinks {
assert_eq!(&h.url, url, "all autolink fragments share the same URL");
assert_eq!(h.id, id, "all autolink fragments share the same id");
assert_eq!(
h.line_index, line_index,
"autolink stays on one rendered line"
);
}
assert_eq!(url, "https://example.com");
let rendered = line_to_string(&out.lines[line_index]);
let mut covered: Vec<bool> = vec![false; rendered.chars().count()];
for h in &out.hyperlinks {
for col in h.column_range.clone() {
if col < covered.len() {
covered[col] = true;
}
}
}
let covered_text: String = rendered
.chars()
.zip(covered.iter())
.filter_map(|(c, &k)| if k { Some(c) } else { None })
.collect();
assert!(
covered_text.contains("https://example.com"),
"combined column ranges must cover the rendered URL; got covered={:?} on rendered={:?}",
covered_text,
rendered,
);
}
/// Two links with prose between them on the same line: the second
/// link's column range must be measured from the start of the line
/// (i.e. the running `cur_col_in_line` survives across emit chunks).
#[test]
fn two_links_with_prose_between_have_correct_columns() {
let text = "Pre [a](https://a.example) mid [b](https://b.example) post.\n";
let (out, _) = render_markdown_ratatui_full(text, test_style::STYLE, true, None);
// Two parser-produced link-text targets (covering "a" and "b")
// plus two url_scan-produced targets for the `(url)` suffixes.
let h0 = parser_link_text(&out, "a");
let h1 = parser_link_text(&out, "b");
assert_eq!(h0.line_index, h1.line_index);
}
/// Streaming byte-by-byte must produce the same hyperlinks (parser +
/// url_scan) as a single full render. Both code paths run the
/// `url_scan` pass after parsing, so the sets must be equal (modulo
/// ordering, which both paths normalise via the same sort key).
#[test]
fn streaming_byte_by_byte_matches_full_render() {
let text = "# Header\n\nSee [docs](https://example.com/docs) and [api](https://example.com/api).\n\n";
let (full, _) = render_markdown_ratatui_full(text, test_style::STYLE, true, None);
let mut renderer = StreamingMarkdownRenderer::new(test_style::STYLE, true);
for byte in text.as_bytes() {
// Push one byte at a time. The input is pure ASCII so each
// single-byte slice is a valid UTF-8 string.
let buf = [*byte];
let s = std::str::from_utf8(&buf).expect("ascii test input");
renderer.push_and_render(s, None);
}
renderer.finish(None);
let view = renderer.view();
// Compare on `(url, line_index, column_range)` — ids are
// intentionally independent between the two code paths (full
// re-render restarts id counters; streaming preserves continuity).
let extract = |hs: &[HyperlinkTarget]| -> Vec<(String, usize, std::ops::Range<usize>)> {
let mut v: Vec<_> = hs
.iter()
.map(|h| (h.url.clone(), h.line_index, h.column_range.clone()))
.collect();
v.sort_by(|a, b| (a.1, a.2.start).cmp(&(b.1, b.2.start)));
v
};
assert_eq!(
extract(&full.hyperlinks),
extract(view.hyperlinks),
"full-render and streaming+finish must produce the same hyperlinks",
);
}
/// A link whose source bytes straddle the frozen/tail boundary in the
/// streaming renderer must still produce a `HyperlinkTarget` pointing
/// at the right rendered line and columns. In pretty mode,
/// `[my link](url)` renders as `my link (url)`, so the renderer
/// produces 2 targets: one parser-produced over the link text, and
/// one from the url_scan pass over the `(url)` suffix.
#[test]
fn streaming_link_across_chunk_boundaries_resolves_correctly() {
let part1 = "Para one.\n\nSee [my ";
let part2 = "link](https://example.com) here.\n\n";
let full_text = format!("{part1}{part2}");
let (full, _) = render_markdown_ratatui_full(&full_text, test_style::STYLE, true, None);
// Both code paths now run url_scan, so the full-render output
// contains the parser-produced link-text hyperlink and the
// url_scan-produced URL-suffix hyperlink.
assert_eq!(full.hyperlinks.len(), 2);
let expected = parser_link_text(&full, "my link");
let mut renderer = StreamingMarkdownRenderer::new(test_style::STYLE, true);
renderer.push_and_render(part1, None);
renderer.push_and_render(part2, None);
renderer.finish(None);
let view = renderer.view();
assert_eq!(
view.hyperlinks.len(),
2,
"expected 2 hyperlinks (parser link text + URL in pretty-mode suffix)"
);
let got = view
.hyperlinks
.iter()
.find(|h| {
h.column_range == expected.column_range && h.line_index == expected.line_index
})
.expect("parser-produced hyperlink should be present after finish()");
assert_eq!(got.url, expected.url);
assert_eq!(got.line_index, expected.line_index);
assert_eq!(got.column_range, expected.column_range);
}
/// Covers the case where the URL literal itself contains `](` (plus a streaming
/// split inside the tag source). The dest-anchored rfind logic also protects
/// realistic nested-image-in-link cases (see nested_image_in_link_finds_outer_closer).
#[test]
fn dest_url_containing_bracket_paren_with_streaming_split() {
let text = "[t](<u](v>) end\n";
let (full, _) = render_markdown_ratatui_full(text, test_style::STYLE, true, None);
let line0 = line_to_string(&full.lines[0]);
assert!(line0.contains("t (<u](v>)") || line0.contains("t ( <u](v> )"));
let link = full
.hyperlinks
.iter()
.find(|h| h.url.contains("u](v"))
.expect("link");
let slice: String = line0
.chars()
.skip(link.column_range.start)
.take(link.column_range.len())
.collect();
assert_eq!(slice, "t");
let mut r = StreamingMarkdownRenderer::new(test_style::STYLE, true);
r.push_and_render("[t](<u]", None);
r.push_and_render("(v>) end\n", None);
r.finish(None);
let view = r.view();
let view_link = view
.hyperlinks
.iter()
.find(|h| h.url.contains("u](v"))
.expect("view");
assert_eq!(view_link.url, link.url);
assert_eq!(view_link.column_range, link.column_range);
}
/// Realistic trigger for the bug: nested image inside a link (common in LLM
/// output: badges, thumbnails, etc.). The outer closer must be found correctly.
#[test]
fn nested_image_in_link_finds_outer_closer() {
let text = "[![badge](https://img.shields.io/v1.svg)](https://github.com/repo) end\n";
let (full, _) = render_markdown_ratatui_full(text, test_style::STYLE, true, None);
let link = full
.hyperlinks
.iter()
.find(|h| h.url.contains("github.com/repo"))
.expect("outer link should target repo URL");
assert!(
!link.url.contains("shields.io"),
"should not pick the inner image URL"
);
}
/// Markdown links inside table cells must produce `HyperlinkTarget`s
/// the same way links inside paragraphs do — otherwise the pager's OSC 8
/// overlay never learns about them and the link is not clickable and
/// not styled. Before the fix, `Tag::Link` events inside table cells
/// were swallowed by the table state machine, leaving the link text
/// as plain text in `StyledCell::spans` with no URL attached.
///
/// This test asserts:
/// 1. A `HyperlinkTarget` is emitted with the cell's URL.
/// 2. Its `column_range` covers the rendered link text glyphs
/// (not the brackets, not the URL).
/// 3. The link text span carries the same `link_text` styling
/// paragraph links get (bold in the test style).
#[test]
fn link_inside_table_cell_emits_hyperlink_and_styling() {
let text = "\
| Name | Link |
|------|------|
| Foo | [click](https://example.com) |
";
let (out, _) = render_markdown_ratatui_full(text, test_style::STYLE, true, None);
// (1) Hyperlink present with the right URL.
let link = out
.hyperlinks
.iter()
.find(|h| h.url == "https://example.com")
.expect("table cell link should produce a HyperlinkTarget");
// (2) Column range covers only the rendered "click" glyphs.
let rendered = line_to_string(&out.lines[link.line_index]);
let slice: String = rendered
.chars()
.skip(link.column_range.start)
.take(link.column_range.len())
.collect();
assert_eq!(
slice, "click",
"column_range should cover only the link text glyphs in the cell, \
got slice={slice:?} from rendered={rendered:?}"
);
// (3) The link text span carries `link_text` styling (bold in the
// test style). The cell wrapper splits the cell into multiple
// spans; find the span whose content is "click".
let cell_line = &out.lines[link.line_index];
let click_span = cell_line
.spans
.iter()
.find(|s| s.content.as_ref() == "click")
.expect("expected a span containing exactly the link text");
assert!(
click_span
.style
.add_modifier
.contains(ratatui::style::Modifier::BOLD),
"link text span inside the cell should carry link_text styling \
(bold in test_style), got style={:?}",
click_span.style,
);
}
/// Paragraph links must keep the `link_text` foreground color even when
/// the `text` style sets its own foreground. Previously the parser
/// pushed `ms.text` as a highlight after the link_text highlight whenever
/// no `Heading`/`Emphasis`/`Strong`/`Strikethrough` ancestor was present
/// — and `merge_styles` lets the later fg color win, so `ms.text`'s color
/// silently clobbered `link_text`'s color on plain paragraph links.
/// Regression: extending `ancestor_styles` to recognise `Link`/`Image`
/// keeps the link_text color intact.
#[test]
fn paragraph_link_keeps_link_text_fg_over_default_text_fg() {
use crate::MarkdownStyle;
use anstyle::{AnsiColor, Color, Style as AStyle};
let style = MarkdownStyle {
text: AStyle::new().fg_color(Some(Color::Ansi(AnsiColor::Red))),
link_text: AStyle::new()
.fg_color(Some(Color::Ansi(AnsiColor::Blue)))
.underline(),
..test_style::STYLE
};
let text = "Hello [click](https://x.com) world.\n";
let (out, _) = render_markdown_ratatui_full(text, style, true, None);
let line = &out.lines[0];
let click_span = line
.spans
.iter()
.find(|s| s.content == "click")
.expect("expected a span containing the link text");
assert_eq!(
click_span.style.fg,
Some(ratatui::style::Color::Blue),
"link text fg must be link_text's blue, not ms.text's red; got {:?}",
click_span.style,
);
assert!(
click_span
.style
.add_modifier
.contains(ratatui::style::Modifier::UNDERLINED),
"link text must remain underlined; got {:?}",
click_span.style,
);
}
/// Links wrapped in inline formatting (`**[click](url)**`,
/// `[**click**](url)`, `*[click](url)*`, `~~[click](url)~~`) must keep
/// the `link_text` foreground while still gaining the formatting effect. The
/// Strong/Emphasis ancestor's inner style carries the theme's default
/// text fg and its highlight is pushed at `Event::Text` time — *after*
/// the `link_text` highlight from `Tag::Link` start — so merge_styles'
/// last-wins fg ordering let it clobber the link color. Regression:
/// inline-format ancestors contribute effects only inside a link.
#[test]
fn formatted_link_keeps_link_fg_and_gains_effects() {
use crate::MarkdownStyle;
use anstyle::{AnsiColor, Color, Style as AStyle};
use ratatui::style::Modifier;
let style = MarkdownStyle {
text: AStyle::new().fg_color(Some(Color::Ansi(AnsiColor::Red))),
strong_inner: AStyle::new()
.fg_color(Some(Color::Ansi(AnsiColor::Red)))
.bold(),
emphasis_inner: AStyle::new()
.fg_color(Some(Color::Ansi(AnsiColor::Red)))
.italic(),
strikethrough_inner: AStyle::new()
.fg_color(Some(Color::Ansi(AnsiColor::Red)))
.strikethrough(),
link_text: AStyle::new()
.fg_color(Some(Color::Ansi(AnsiColor::Blue)))
.underline(),
..test_style::STYLE
};
for (md, effect) in [
("**[click](https://x.com)** end.\n", Modifier::BOLD),
("[**click**](https://x.com) end.\n", Modifier::BOLD),
("*[click](https://x.com)* end.\n", Modifier::ITALIC),
("~~[click](https://x.com)~~ end.\n", Modifier::CROSSED_OUT),
] {
let (out, _) = render_markdown_ratatui_full(md, style, true, None);
let line = &out.lines[0];
let click_span = line
.spans
.iter()
.find(|s| s.content == "click")
.unwrap_or_else(|| panic!("expected a span for the link text in {md:?}"));
assert_eq!(
click_span.style.fg,
Some(ratatui::style::Color::Blue),
"in {md:?} the link text fg must stay link_text's blue, \
not strong/emphasis_inner's red; got {:?}",
click_span.style,
);
assert!(
click_span.style.add_modifier.contains(effect),
"in {md:?} the link text must gain {effect:?}; got {:?}",
click_span.style,
);
assert!(
click_span.style.add_modifier.contains(Modifier::UNDERLINED),
"in {md:?} the link text must remain underlined; got {:?}",
click_span.style,
);
}
}
// Soft break inside link text: the link stays on one rendered line
// and the fragments sharing this link's id cover exactly "link text".
// SoftBreak splits a link into multiple HyperlinkTargets with the
// same id (OSC 8 wrapped-link grouping), so we check the union.
#[test]
fn soft_break_inside_link_text_preserves_column_range() {
let md = "foo [link\ntext](https://example.com) bar";
let (out, _) = render_markdown_ratatui_full(md, test_style::STYLE, true, None);
assert_eq!(out.lines.len(), 1, "{:?}", out.lines);
// url_scan also emits a target over the `(url)` suffix; filter to the parser id.
let parser_id = out
.hyperlinks
.iter()
.find(|h| h.url == "https://example.com")
.expect("at least one hyperlink target")
.id;
let mut fragments: Vec<&HyperlinkTarget> = out
.hyperlinks
.iter()
.filter(|h| h.id == parser_id && h.url == "https://example.com")
.collect();
fragments.sort_by_key(|h| h.column_range.start);
for f in &fragments {
assert_eq!(f.line_index, 0, "{f:?}");
}
for w in fragments.windows(2) {
assert_eq!(
w[0].column_range.end, w[1].column_range.start,
"gap: {:?} -> {:?}",
w[0].column_range, w[1].column_range,
);
}
let union_start = fragments.first().unwrap().column_range.start;
let union_end = fragments.last().unwrap().column_range.end;
let rendered = line_to_string(&out.lines[0]);
assert_eq!(
slice_by_cells(&rendered, union_start..union_end),
"link text",
"rendered={rendered:?} union={union_start}..{union_end}",
);
assert_eq!(
union_end - union_start,
crate::buffers::unicode_display_width("link text"),
);
}
}

532
crates/codegen/xai-grok-markdown/src/latex/commands.rs Normal file
View file

@ -0,0 +1,532 @@
//! Core renderer: sequences, commands, scripts, fractions, accents.
use std::fmt::Write as _;
use super::cursor::Cursor;
use super::environments::render_environment;
use super::math_box::MathBox;
use super::symbols::{
map_mathbb, map_mathbf, map_mathcal, map_mathfrak, symbol, to_subscript, to_superscript,
};
use super::{MAX_DEPTH, Mode};
/// Render an atom's source to a flat (single-line) Unicode string.
///
/// Atoms are arguments to commands (fraction sides, script bodies, accent
/// targets); they always render flat — multi-row content inside them joins
/// with `; `.
pub(super) fn render_atom(atom: &str, depth: usize, mode: Mode) -> String {
let mut cursor = Cursor::new(atom);
let mut out = MathBox::new(true);
render_sequence(&mut cursor, &mut out, depth + 1, mode, None);
out.into_lines().concat()
}
/// Core renderer: walks `cursor`, appending Unicode to `out`.
///
/// `stop_at` optionally terminates the sequence at an unbalanced `}` (used
/// when rendering inside a group whose `{` was consumed by the caller).
pub(super) fn render_sequence(
cursor: &mut Cursor<'_>,
out: &mut MathBox,
depth: usize,
mode: Mode,
stop_at: Option<char>,
) {
while let Some(ch) = cursor.peek() {
if Some(ch) == stop_at {
cursor.bump();
return;
}
match ch {
'\\' => {
cursor.bump();
render_command(cursor, out, depth, mode);
}
'{' => {
cursor.bump();
if depth >= MAX_DEPTH {
// Too deep: render the group body flat, without recursing.
out.push_str(cursor.read_group_body());
} else {
// Render the group body into the same box so environments
// inside groups keep their 2D layout.
let body = cursor.read_group_body();
let mut sub = Cursor::new(body);
render_sequence(&mut sub, out, depth + 1, mode, None);
}
}
'}' => {
// Unbalanced closing brace: drop it.
cursor.bump();
}
'^' => {
cursor.bump();
render_script(cursor, out, depth, mode, Script::Super);
}
'_' => {
cursor.bump();
render_script(cursor, out, depth, mode, Script::Sub);
}
'~' => {
cursor.bump();
out.push(' ');
}
'&' => {
// Alignment marker outside an environment: drop.
cursor.bump();
}
'$' => {
// Stray math delimiter inside math: drop.
cursor.bump();
}
'-' if mode == Mode::Math => {
cursor.bump();
out.push('−');
}
'\'' if mode == Mode::Math => {
cursor.bump();
out.push('′');
}
c if c.is_whitespace() => {
cursor.skip_ws();
// TeX collapses whitespace runs (including newlines) to
// nothing semantically; keep a single space for readability.
if !out.at_line_start() && !out.ends_with_space() {
out.push(' ');
}
}
c => {
cursor.bump();
out.push(c);
}
}
}
}
/// Which script position is being rendered.
#[derive(Copy, Clone, PartialEq, Eq)]
enum Script {
Super,
Sub,
}
/// Render `^atom` / `_atom` using Unicode script chars when every char of
/// the rendered atom has a script form; otherwise `^x` / `^(...)` fallback.
///
/// Word-like atoms take the fallback even when fully mappable: labels such as
/// `p_{\text{torso}}` or `x_{max}` would otherwise become long modifier-letter
/// runs (`pₜₒᵣₛₒ`) that are hard to read and render with visible gaps in
/// terminal fonts lacking those glyphs. Index-like atoms (`x_{ij}`,
/// `T_{i+1}`, `n^{th}`) keep the compact Unicode form.
fn render_script(
cursor: &mut Cursor<'_>,
out: &mut MathBox,
depth: usize,
mode: Mode,
kind: Script,
) {
let Some(atom) = cursor.read_atom() else {
out.push(match kind {
Script::Super => '^',
Script::Sub => '_',
});
return;
};
let rendered = render_atom(atom, depth, mode);
let mapped: Option<String> = if script_atom_is_wordlike(atom, &rendered) {
None
} else {
rendered
.chars()
.map(|c| match kind {
Script::Super => to_superscript(c),
Script::Sub => to_subscript(c),
})
.collect()
};
match mapped {
Some(s) if !s.is_empty() => out.push_str(&s),
_ => {
out.push(match kind {
Script::Super => '^',
Script::Sub => '_',
});
if rendered.chars().count() > 1 {
let _ = write!(out, "({rendered})");
} else {
out.push_str(&rendered);
}
}
}
}
/// `true` if a script atom is a word-like label rather than indices.
///
/// Two signals, checked on the atom *source* and its rendered form:
///
/// - the source routes through a text-family command (`\text{…}`, `\mathrm{…}`,
/// `\operatorname{…}`, …): the author explicitly marked the content as a
/// word;
/// - the rendered form contains a run of 3+ ASCII letters: multi-letter runs
/// read as words (`max`, `torso`), while 1–2 letter runs are index
/// juxtapositions (`ij`, `th`) that stay compact.
fn script_atom_is_wordlike(atom: &str, rendered: &str) -> bool {
// `\text` also catches `\textrm`/`\textbf`/`\textit`/`\textsf`/`\texttt`/
// `\textnormal` by prefix; `\math…` variants and box commands likewise.
const TEXT_MARKERS: [&str; 8] = [
"\\text",
"\\mathrm",
"\\mathsf",
"\\mathtt",
"\\mathit",
"\\operatorname",
"\\mbox",
"\\hbox",
];
if TEXT_MARKERS.iter().any(|m| atom.contains(m)) {
return true;
}
let mut run = 0usize;
for c in rendered.chars() {
if c.is_ascii_alphabetic() {
run += 1;
if run >= 3 {
return true;
}
} else {
run = 0;
}
}
false
}
/// Render a `\command` whose backslash was already consumed.
fn render_command(cursor: &mut Cursor<'_>, out: &mut MathBox, depth: usize, mode: Mode) {
let name = cursor.read_command_name();
match name {
// ── Structure ────────────────────────────────────────────────────
"" => out.push('\\'),
"\\" => out.push('\n'),
"begin" => render_environment(cursor, out, depth, mode),
"end" => {
// Stray \end without matching \begin: drop its argument.
let _ = take_brace_arg(cursor);
}
"left" | "right" => {
// Keep the delimiter that follows; `.` means "no delimiter".
cursor.skip_ws();
match cursor.peek() {
Some('.') => {
cursor.bump();
}
Some('\\') => {
cursor.bump();
render_command(cursor, out, depth, mode);
}
Some(c) => {
cursor.bump();
out.push(c);
}
None => {}
}
}
// ── Fractions / binomials / roots ────────────────────────────────
"frac" | "dfrac" | "tfrac" | "cfrac" => {
let num = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
let den = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
match (num, den) {
(Some(n), Some(d)) => out.push_str(&format_fraction(&n, &d)),
(Some(n), None) => out.push_str(&n),
_ => {}
}
}
"binom" | "tbinom" | "dbinom" => {
let n = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
let k = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
if let (Some(n), Some(k)) = (n, k) {
let _ = write!(out, "C({n}, {k})");
}
}
"sqrt" => {
cursor.skip_ws();
let index = if cursor.peek() == Some('[') {
cursor.bump();
let start = cursor.pos;
while let Some(c) = cursor.peek() {
if c == ']' {
break;
}
cursor.bump();
}
let idx = &cursor.src[start..cursor.pos];
cursor.bump(); // consume `]`
Some(render_atom(idx, depth, mode))
} else {
None
};
let radical = match index.as_deref() {
None | Some("2") => "√",
Some("3") => "∛",
Some("4") => "∜",
Some(other) => {
// ⁿ√ style prefix for other indices.
let sup: Option<String> = other.chars().map(to_superscript).collect();
out.push_str(&sup.unwrap_or_else(|| format!("({other})")));
"√"
}
};
out.push_str(radical);
if let Some(arg) = cursor.read_atom() {
let rendered = render_atom(arg, depth, mode);
// Parenthesize any multi-char radicand: `√ab` would read as
// `(√a)b`.
if rendered.chars().count() > 1 {
let _ = write!(out, "({rendered})");
} else {
out.push_str(&rendered);
}
}
}
// ── Boxes (frame dropped; content preserved) ─────────────────────
"boxed" => {
if let Some(arg) = take_brace_arg(cursor) {
out.push_str(&render_atom(arg, depth, mode));
}
}
"fbox" | "framebox" => {
if let Some(arg) = take_brace_arg(cursor) {
out.push_str(&render_atom(arg, depth, Mode::Text));
}
}
// ── Text / alphabets ─────────────────────────────────────────────
"text" | "textrm" | "textit" | "textbf" | "textsf" | "texttt" | "textnormal" | "mbox"
| "hbox" => {
if let Some(arg) = take_brace_arg(cursor) {
out.push_str(&render_atom(arg, depth, Mode::Text));
}
}
"mathrm" | "operatorname" | "mathit" | "mathsf" | "mathtt" | "mathnormal" => {
if let Some(arg) = take_brace_arg(cursor) {
out.push_str(&render_atom(arg, depth, Mode::Text));
}
}
"mathbb" => render_mapped_alphabet(cursor, out, depth, mode, map_mathbb),
"mathcal" | "mathscr" => render_mapped_alphabet(cursor, out, depth, mode, map_mathcal),
"mathfrak" => render_mapped_alphabet(cursor, out, depth, mode, map_mathfrak),
"mathbf" | "boldsymbol" | "bm" | "bold" => {
render_mapped_alphabet(cursor, out, depth, mode, map_mathbf)
}
// ── Accents (combining marks) ────────────────────────────────────
"hat" | "widehat" => render_accent(cursor, out, depth, mode, '\u{0302}'),
"bar" | "overline" => render_accent(cursor, out, depth, mode, '\u{0304}'),
"tilde" | "widetilde" => render_accent(cursor, out, depth, mode, '\u{0303}'),
"vec" => render_accent(cursor, out, depth, mode, '\u{20D7}'),
"dot" => render_accent(cursor, out, depth, mode, '\u{0307}'),
"ddot" => render_accent(cursor, out, depth, mode, '\u{0308}'),
"check" => render_accent(cursor, out, depth, mode, '\u{030C}'),
"breve" => render_accent(cursor, out, depth, mode, '\u{0306}'),
"acute" => render_accent(cursor, out, depth, mode, '\u{0301}'),
"grave" => render_accent(cursor, out, depth, mode, '\u{0300}'),
"mathring" => render_accent(cursor, out, depth, mode, '\u{030A}'),
"underline" => render_accent(cursor, out, depth, mode, '\u{0332}'),
// ── Negation ─────────────────────────────────────────────────────
"not" => {
if let Some(atom) = cursor.read_atom() {
let rendered = render_atom(atom, depth, mode);
match rendered.as_str() {
"∈" => out.push('∉'),
"=" => out.push('≠'),
"<" => out.push('≮'),
">" => out.push('≯'),
"≡" => out.push('≢'),
"⊂" => out.push('⊄'),
"⊆" => out.push('⊈'),
"∃" => out.push('∄'),
other => {
out.push_str(other);
// Combining long solidus overlay on the last char.
if !other.is_empty() {
out.push('\u{0338}');
}
}
}
}
}
// ── Decorations rendered as base + script ────────────────────────
"overset" | "stackrel" => {
let over = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
let base = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
if let (Some(over), Some(base)) = (over, base) {
out.push_str(&base);
let sup: Option<String> = over.chars().map(to_superscript).collect();
match sup {
Some(s) if !s.is_empty() => out.push_str(&s),
_ => {}
}
}
}
"underset" => {
let under = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
let base = take_brace_arg(cursor).map(|a| render_atom(a, depth, mode));
if let (Some(under), Some(base)) = (under, base) {
out.push_str(&base);
let sub: Option<String> = under.chars().map(to_subscript).collect();
match sub {
Some(s) if !s.is_empty() => out.push_str(&s),
_ => {}
}
}
}
// ── Modular arithmetic ───────────────────────────────────────────
"pmod" => {
if let Some(arg) = take_brace_arg(cursor) {
if !out.at_line_start() && !out.ends_with_space() {
out.push(' ');
}
let _ = write!(out, "(mod {})", render_atom(arg, depth, mode));
}
}
"bmod" => {
if !out.at_line_start() && !out.ends_with_space() {
out.push(' ');
}
out.push_str("mod ");
}
// ── Spacing ──────────────────────────────────────────────────────
"," | ";" | ":" | ">" | " " | "space" | "thinspace" | "medspace" | "thickspace"
| "enspace" => {
if !out.at_line_start() && !out.ends_with_space() {
out.push(' ');
}
}
"quad" => out.push_str(" "),
"qquad" => out.push_str(" "),
"!" | "negthinspace" | "negmedspace" | "negthickspace" => {}
// ── No-ops (sizing/styling/structure hints) ──────────────────────
"limits" | "nolimits" | "displaystyle" | "textstyle" | "scriptstyle"
| "scriptscriptstyle" | "big" | "Big" | "bigg" | "Bigg" | "bigl" | "Bigl" | "biggl"
| "Biggl" | "bigr" | "Bigr" | "biggr" | "Biggr" | "bigm" | "Bigm" | "biggm" | "Biggm"
| "mathstrut" | "strut" | "allowbreak" | "nonumber" | "notag" | "mathopen"
| "mathclose" | "mathbin" | "mathrel" | "mathord" | "mathpunct" | "mathinner"
| "mathop" | "ensuremath" | "label" | "tag" => {
// \label/\tag carry non-visual arguments: drop them.
if matches!(name, "label" | "tag") {
let _ = take_brace_arg(cursor);
}
}
// ── Symbol table ─────────────────────────────────────────────────
_ => {
if let Some(sym) = symbol(name) {
out.push_str(sym);
} else {
// Unknown command: keep its name as plain text.
out.push_str(name);
}
}
}
}
/// Consume `{...}` (after optional whitespace) and return the body source.
pub(super) fn take_brace_arg<'a>(cursor: &mut Cursor<'a>) -> Option<&'a str> {
cursor.skip_ws();
if cursor.peek() == Some('{') {
cursor.bump();
Some(cursor.read_group_body())
} else {
None
}
}
/// `true` if a fraction/root operand needs parentheses for readability.
fn needs_parens(s: &str) -> bool {
s.chars().count() > 1 && s.contains([' ', '+', '−', '-', '=', '/'])
}
/// Format `num/den`, mapping common numeric fractions to vulgar fractions.
fn format_fraction(num: &str, den: &str) -> String {
let vulgar = match (num, den) {
("1", "2") => Some('½'),
("1", "3") => Some('⅓'),
("2", "3") => Some('⅔'),
("1", "4") => Some('¼'),
("3", "4") => Some('¾'),
("1", "5") => Some('⅕'),
("2", "5") => Some('⅖'),
("3", "5") => Some('⅗'),
("4", "5") => Some('⅘'),
("1", "6") => Some('⅙'),
("5", "6") => Some('⅚'),
("1", "7") => Some('⅐'),
("1", "8") => Some('⅛'),
("3", "8") => Some('⅜'),
("5", "8") => Some('⅝'),
("7", "8") => Some('⅞'),
("1", "9") => Some('⅑'),
("1", "10") => Some('⅒'),
_ => None,
};
if let Some(v) = vulgar {
return v.to_string();
}
let n = if needs_parens(num) {
format!("({num})")
} else {
num.to_string()
};
let d = if needs_parens(den) {
format!("({den})")
} else {
den.to_string()
};
format!("{n}/{d}")
}
/// Render an alphabet-mapping command (`\mathbb{R}` etc.): map chars that
/// have a styled form, keep the rest as rendered.
fn render_mapped_alphabet(
cursor: &mut Cursor<'_>,
out: &mut MathBox,
depth: usize,
mode: Mode,
map: fn(char) -> Option<char>,
) {
let Some(atom) = cursor.read_atom() else {
return;
};
let rendered = render_atom(atom, depth, mode);
for c in rendered.chars() {
out.push(map(c).unwrap_or(c));
}
}
/// Render an accent command by appending a combining mark to each char of
/// the argument.
fn render_accent(
cursor: &mut Cursor<'_>,
out: &mut MathBox,
depth: usize,
mode: Mode,
combining: char,
) {
let Some(atom) = cursor.read_atom() else {
return;
};
let rendered = render_atom(atom, depth, mode);
for c in rendered.chars() {
out.push(c);
if !c.is_whitespace() {
out.push(combining);
}
}
}

99
crates/codegen/xai-grok-markdown/src/latex/cursor.rs Normal file
View file

@ -0,0 +1,99 @@
//! Byte cursor over TeX source.
/// Byte cursor over the TeX source.
pub(super) struct Cursor<'a> {
pub(super) src: &'a str,
pub(super) pos: usize,
}
impl<'a> Cursor<'a> {
pub(super) fn new(src: &'a str) -> Self {
Self { src, pos: 0 }
}
pub(super) fn peek(&self) -> Option<char> {
self.src[self.pos..].chars().next()
}
pub(super) fn bump(&mut self) -> Option<char> {
let ch = self.peek()?;
self.pos += ch.len_utf8();
Some(ch)
}
/// Consume `\command` (alphabetic name) or `\<single char>`; the leading
/// backslash must already be consumed. Returns the command name.
///
/// Unlike TeX we do NOT consume trailing whitespace: the caller's
/// whitespace collapsing keeps `\to 0` rendering as `→ 0`.
pub(super) fn read_command_name(&mut self) -> &'a str {
let start = self.pos;
match self.peek() {
Some(c) if c.is_ascii_alphabetic() => {
while matches!(self.peek(), Some(c) if c.is_ascii_alphabetic()) {
self.bump();
}
&self.src[start..self.pos]
}
Some(_) => {
self.bump();
&self.src[start..self.pos]
}
None => "",
}
}
/// Skip whitespace (TeX collapses it; meaning comes from commands).
pub(super) fn skip_ws(&mut self) {
while matches!(self.peek(), Some(c) if c.is_whitespace()) {
self.bump();
}
}
/// Read a balanced `{...}` group body, assuming `{` was already consumed.
/// Returns the inner source (without braces). Unbalanced input returns
/// the remainder of the source.
pub(super) fn read_group_body(&mut self) -> &'a str {
let start = self.pos;
let mut depth = 1usize;
while let Some(ch) = self.bump() {
match ch {
'\\' => {
// Skip escaped char so `\{`/`\}` don't affect depth.
self.bump();
}
'{' => depth += 1,
'}' => {
depth -= 1;
if depth == 0 {
return &self.src[start..self.pos - 1];
}
}
_ => {}
}
}
&self.src[start..self.pos]
}
/// Read the next "atom": a `{...}` group body, a `\command` (returned
/// with backslash), or a single char. Skips leading whitespace.
pub(super) fn read_atom(&mut self) -> Option<&'a str> {
self.skip_ws();
let start = self.pos;
match self.peek()? {
'{' => {
self.bump();
Some(self.read_group_body())
}
'\\' => {
self.bump();
self.read_command_name();
Some(&self.src[start..self.pos])
}
_ => {
self.bump();
Some(&self.src[start..self.pos])
}
}
}
}

325
crates/codegen/xai-grok-markdown/src/latex/environments.rs Normal file
View file

@ -0,0 +1,325 @@
//! `\\begin{env}...\\end{env}` environments: matrices, cases, alignments.
use crate::buffers::unicode_display_width;
use super::Mode;
use super::commands::{render_atom, take_brace_arg};
use super::cursor::Cursor;
use super::math_box::MathBox;
/// Render `\begin{env}...\end{env}`. The `\begin` name was already consumed.
pub(super) fn render_environment(
cursor: &mut Cursor<'_>,
out: &mut MathBox,
depth: usize,
mode: Mode,
) {
let Some(env_name) = take_brace_arg(cursor) else {
return;
};
let env_name = env_name.trim().trim_end_matches('*');
// Capture body source until the matching `\end{name}`, tracking nesting
// of same-named environments. Scans raw source from the cursor.
let body_start = cursor.pos;
let mut body_end = cursor.src.len();
let mut resume = cursor.src.len();
let mut nest = 0usize;
let mut search = cursor.pos;
while search < cursor.src.len() {
let rest = &cursor.src[search..];
let Some(rel) = rest.find('\\') else {
break;
};
let bs_pos = search + rel;
let after_bs = &cursor.src[bs_pos + 1..];
let kw_len = if command_at(after_bs, "begin") {
"begin".len()
} else if command_at(after_bs, "end") {
"end".len()
} else {
// Not begin/end: skip the backslash and the char after it (so
// `\\` and `\{` never confuse the scan).
let skip = after_bs.chars().next().map_or(0, char::len_utf8);
search = bs_pos + 1 + skip.max(1);
continue;
};
let is_begin = kw_len == "begin".len();
let mut probe = Cursor {
src: cursor.src,
pos: bs_pos + 1 + kw_len,
};
let arg = take_brace_arg(&mut probe).map(|a| a.trim().trim_end_matches('*'));
if arg == Some(env_name) {
if is_begin {
nest += 1;
} else if nest == 0 {
body_end = bs_pos;
resume = probe.pos;
break;
} else {
nest -= 1;
}
}
search = probe.pos.max(bs_pos + 1 + kw_len);
}
cursor.pos = resume;
let mut body = &cursor.src[body_start..body_end.min(cursor.src.len())];
// Optional column spec for array environments: `\begin{array}{ll}`.
if env_name == "array" || env_name == "alignat" {
let mut probe = Cursor::new(body);
probe.skip_ws();
if probe.peek() == Some('{') {
probe.bump();
let _ = probe.read_group_body();
body = &body[probe.pos..];
}
}
let rows = env_rows_to_strings(body, env_name, out.flat, depth, mode);
out.hcat_rows(rows);
}
/// `true` if `rest` starts with command word `word` NOT followed by another
/// ASCII letter (so `\endx` is not mistaken for `\end`).
fn command_at(rest: &str, word: &str) -> bool {
rest.starts_with(word)
&& !rest[word.len()..]
.chars()
.next()
.is_some_and(|c| c.is_ascii_alphabetic())
}
/// Split an environment body into rows (`\\`) and cells (`&`) at brace and
/// environment depth 0, render each cell, then lay the rows out according to
/// the environment. Returns one string per visual row; the caller attaches
/// them as a box. In `flat` mode, matrix/cases environments render as a
/// single row with `; ` between matrix rows.
fn env_rows_to_strings(
body: &str,
env_name: &str,
flat: bool,
depth: usize,
mode: Mode,
) -> Vec<String> {
let mut rows: Vec<Vec<String>> = Vec::new();
let mut row: Vec<String> = Vec::new();
let mut cell_start = 0usize;
let mut brace_depth = 0usize;
let mut env_depth = 0usize;
let bytes = body.as_bytes();
let mut i = 0usize;
while i < bytes.len() {
match bytes[i] {
b'\\' => {
if bytes.get(i + 1) == Some(&b'\\') {
if brace_depth == 0 && env_depth == 0 {
row.push(body[cell_start..i].to_string());
rows.push(std::mem::take(&mut row));
i += 2;
cell_start = i;
continue;
}
i += 2;
continue;
}
let rest = &body[i + 1..];
if command_at(rest, "begin") {
env_depth += 1;
} else if command_at(rest, "end") {
env_depth = env_depth.saturating_sub(1);
}
// Skip the backslash plus the char after it so escaped
// delimiters (`\&`, `\{`, `\}`) never affect depth/splits.
let skip = rest.chars().next().map_or(0, char::len_utf8);
i += 1 + skip.max(1);
continue;
}
b'{' => brace_depth += 1,
b'}' => brace_depth = brace_depth.saturating_sub(1),
b'&' if brace_depth == 0 && env_depth == 0 => {
row.push(body[cell_start..i].to_string());
cell_start = i + 1;
}
_ => {}
}
i += 1;
}
row.push(body[cell_start.min(bytes.len())..].to_string());
rows.push(row);
// Render each cell, drop fully-empty rows.
let mut rendered_rows: Vec<Vec<String>> = rows
.into_iter()
.map(|cells| {
cells
.into_iter()
.map(|c| render_atom(c.trim(), depth, mode).trim().to_string())
.collect::<Vec<_>>()
})
.collect();
rendered_rows.retain(|cells| cells.iter().any(|c| !c.is_empty()));
if rendered_rows.is_empty() {
return Vec::new();
}
let is_matrix = matches!(
env_name,
"matrix"
| "pmatrix"
| "bmatrix"
| "Bmatrix"
| "vmatrix"
| "Vmatrix"
| "smallmatrix"
| "array"
);
let n_rows = rendered_rows.len();
if is_matrix {
// Flat (inline) mode: one row, single delimiter pair, rows joined
// with `; ` — `(1 2; 3 4)`.
if flat {
let inner = rendered_rows
.iter()
.map(|cells| cells.join(" "))
.collect::<Vec<_>>()
.join("; ");
// Single-row delimiter pair; plain `matrix` has none (' ').
let (l, r) = matrix_delims(env_name, 0, 1);
let mut s = String::new();
if l != ' ' {
s.push(l);
}
s.push_str(&inner);
if r != ' ' {
s.push(r);
}
return vec![s];
}
// Pad columns to equal width so rows align.
let n_cols = rendered_rows.iter().map(Vec::len).max().unwrap_or(0);
let mut widths = vec![0usize; n_cols];
for cells in &rendered_rows {
for (i, cell) in cells.iter().enumerate() {
widths[i] = widths[i].max(unicode_display_width(cell));
}
}
rendered_rows
.iter()
.enumerate()
.map(|(row_idx, cells)| {
let mut content = String::new();
for (i, cell) in cells.iter().enumerate() {
if i > 0 {
content.push_str(" ");
}
content.push_str(cell);
if i + 1 < cells.len() {
let pad = widths[i].saturating_sub(unicode_display_width(cell));
content.push_str(&" ".repeat(pad));
}
}
let (l, r) = matrix_delims(env_name, row_idx, n_rows);
format!("{l}{content}{r}")
})
.collect()
} else if env_name == "cases" {
if flat {
let inner = rendered_rows
.iter()
.map(|cells| cells.join(" "))
.collect::<Vec<_>>()
.join("; ");
return vec![format!("{{{inner}}}")];
}
rendered_rows
.iter()
.enumerate()
.map(|(row_idx, cells)| {
let brace = cases_brace(row_idx, n_rows);
format!("{brace} {}", cells.join(" "))
})
.collect()
} else {
// aligned/align/gather/split/equation/…: `&` is an invisible
// alignment marker; rejoin cells with a single space. One string per
// row; the caller's box attachment (or flat `; ` join) handles the
// rest.
rendered_rows
.iter()
.map(|cells| {
let mut s = cells
.iter()
.filter(|c| !c.is_empty())
.cloned()
.collect::<Vec<_>>()
.join(" ");
// Collapse any double spaces introduced around markers.
while s.contains(" ") {
s = s.replace(" ", " ");
}
s
})
.collect()
}
}
/// Per-row delimiters for matrix-family environments.
fn matrix_delims(env: &str, row: usize, n_rows: usize) -> (char, char) {
let single = n_rows == 1;
let first = row == 0;
let last = row + 1 == n_rows;
match env {
"pmatrix" => {
if single {
('(', ')')
} else if first {
('⎛', '⎞')
} else if last {
('⎝', '⎠')
} else {
('⎜', '⎟')
}
}
"bmatrix" | "array" => {
if single {
('[', ']')
} else if first {
('⎡', '⎤')
} else if last {
('⎣', '⎦')
} else {
('⎢', '⎥')
}
}
"Bmatrix" => {
if single {
('{', '}')
} else if first {
('⎧', '⎫')
} else if last {
('⎩', '⎭')
} else {
('⎨', '⎬')
}
}
"vmatrix" | "Vmatrix" => ('│', '│'),
_ => (' ', ' '),
}
}
/// Left-brace column char for `cases` rows.
fn cases_brace(row: usize, n_rows: usize) -> char {
if n_rows == 1 {
'{'
} else if row == 0 {
'⎧'
} else if row + 1 == n_rows {
'⎩'
} else if row == n_rows / 2 {
'⎨'
} else {
'⎪'
}
}

156
crates/codegen/xai-grok-markdown/src/latex/math_box.rs Normal file
View file

@ -0,0 +1,156 @@
//! Two-dimensional math layout box.
use crate::buffers::unicode_display_width;
/// Two-dimensional text box with an anchor row where horizontal flow
/// attaches.
///
/// Multi-row content (matrix-family environments) extends above/below the
/// anchor row; subsequent output continues on the anchor row. This keeps a
/// prefix, a matrix, and a suffix aligned:
///
/// ```text
/// A = ⎛1 2⎞, det(A) = −2
/// ⎝3 4⎠
/// ```
pub(super) struct MathBox {
lines: Vec<String>,
/// Row index that horizontal flow currently appends to.
anchor: usize,
/// First row belonging to the current visual line. Rows before `floor`
/// are completed lines from earlier `\\` breaks and must never be
/// touched by box attachment.
floor: usize,
/// Flat mode (inline math): vertical layout is impossible, so row breaks
/// render as `; ` and environments render single-row.
pub(super) flat: bool,
}
impl MathBox {
pub(super) fn new(flat: bool) -> Self {
Self {
lines: vec![String::new()],
anchor: 0,
floor: 0,
flat,
}
}
fn cur(&mut self) -> &mut String {
&mut self.lines[self.anchor]
}
/// `true` when nothing has been emitted on the current flow row yet.
pub(super) fn at_line_start(&self) -> bool {
self.lines[self.anchor].is_empty()
}
pub(super) fn ends_with_space(&self) -> bool {
self.lines[self.anchor].ends_with(' ')
}
pub(super) fn push(&mut self, c: char) {
if c == '\n' {
self.vbreak();
} else {
self.cur().push(c);
}
}
pub(super) fn push_str(&mut self, s: &str) {
if s.contains('\n') {
self.hcat_rows(s.split('\n').map(str::to_string).collect());
} else {
self.cur().push_str(s);
}
}
/// End the current visual line; flow continues on a fresh row below all
/// existing rows. Flat mode renders the break as `; `.
fn vbreak(&mut self) {
if self.flat {
if !self.at_line_start() {
let cur = self.cur();
while cur.ends_with(' ') {
cur.pop();
}
cur.push_str("; ");
}
} else {
self.lines.push(String::new());
self.anchor = self.lines.len() - 1;
self.floor = self.anchor;
}
}
/// Attach `rows` as a box at the current flow position, anchored at the
/// box's upper-middle row. All box rows start at the same column; flow
/// resumes on the anchor row past the box's widest row.
pub(super) fn hcat_rows(&mut self, rows: Vec<String>) {
if rows.is_empty() {
return;
}
if self.flat || rows.len() == 1 {
for (i, row) in rows.iter().enumerate() {
if i > 0 {
self.vbreak();
}
self.cur().push_str(row);
}
return;
}
let box_anchor = (rows.len() - 1) / 2;
let attach_col = unicode_display_width(&self.lines[self.anchor]);
let box_width = rows
.iter()
.map(|r| unicode_display_width(r))
.max()
.unwrap_or(0);
// Ensure enough rows above the anchor within the current visual line.
let have_above = self.anchor - self.floor;
if box_anchor > have_above {
let add = box_anchor - have_above;
for _ in 0..add {
self.lines.insert(self.floor, String::new());
}
self.anchor += add;
}
// Ensure enough rows below the anchor.
let below = rows.len() - box_anchor - 1;
let have_below = self.lines.len() - self.anchor - 1;
if below > have_below {
for _ in 0..(below - have_below) {
self.lines.push(String::new());
}
}
// Place the box rows, left-padded to the attach column.
for (i, row) in rows.iter().enumerate() {
let target = self.anchor - box_anchor + i;
let line = &mut self.lines[target];
let cur_w = unicode_display_width(line);
if cur_w < attach_col {
line.push_str(&" ".repeat(attach_col - cur_w));
}
line.push_str(row);
}
// Flow resumes past the box's widest row.
let frontier = attach_col + box_width;
let cur_w = unicode_display_width(&self.lines[self.anchor]);
if cur_w < frontier {
let pad = frontier - cur_w;
self.lines[self.anchor].push_str(&" ".repeat(pad));
}
}
pub(super) fn into_lines(self) -> Vec<String> {
self.lines
}
}
impl std::fmt::Write for MathBox {
fn write_str(&mut self, s: &str) -> std::fmt::Result {
self.push_str(s);
Ok(())
}
}

96
crates/codegen/xai-grok-markdown/src/latex/mod.rs Normal file
View file

@ -0,0 +1,96 @@
//! Best-effort LaTeX math → Unicode plain-text conversion.
//!
//! Converts TeX math source (the content of `$...$`, `$$...$$`, `\(...\)`,
//! `\[...\]`) into a readable Unicode approximation for terminal display:
//!
//! - Greek letters and symbol commands (`\alpha` → `α`, `\le` → `≤`, …)
//! - Superscripts/subscripts via Unicode script characters (`x^2` → `x²`,
//! `a_1` → `a₁`) with `^(...)`/`_(...)` fallback when a char has no
//! Unicode script form
//! - Fractions (`\frac{1}{2}` → `½`, `\frac{a+b}{c}` → `(a+b)/c`)
//! - Roots (`\sqrt{x}` → `√x`, `\sqrt[3]{x}` → `∛x`)
//! - Alphabets (`\mathbb{R}` → `ℝ`, `\mathcal{L}` → `ℒ`, `\mathbf{v}` → `𝐯`)
//! - Accents via combining marks (`\hat{x}` → `x̂`, `\vec{v}` → `v⃗`)
//! - Environments (`aligned`, `cases`, `pmatrix`, …) → multi-line layout
//!
//! The converter is total: it never panics and always produces *some* output
//! (unknown commands degrade to their bare name). Callers decide whether to
//! use the conversion or fall back to raw TeX source.
mod commands;
mod cursor;
mod environments;
mod math_box;
mod symbols;
#[cfg(test)]
mod tests;
use commands::render_sequence;
use cursor::Cursor;
use math_box::MathBox;
/// Inputs larger than this are not converted (callers fall back to raw
/// display). Guards the streaming hot path: the tail is re-rendered on every
/// chunk, so conversion cost must stay trivially small.
pub(crate) const MAX_MATH_SOURCE_LEN: usize = 4096;
/// Hard cap on group-nesting recursion. Inputs deeper than this render their
/// remaining content flatly rather than recursing further.
const MAX_DEPTH: usize = 32;
/// Convert inline math to a single-line Unicode string.
///
/// Row separators (`\\`) collapse to `; ` and multi-row environments render
/// single-row, so inline math never introduces a line break mid-paragraph.
/// Returns `None` when the source is too large to convert (see
/// [`MAX_MATH_SOURCE_LEN`]).
pub(crate) fn latex_to_unicode_inline(src: &str) -> Option<String> {
if src.len() > MAX_MATH_SOURCE_LEN {
return None;
}
let lines = convert(src, true);
let joined = lines
.iter()
.map(|l| l.trim())
.filter(|l| !l.is_empty())
.collect::<Vec<_>>()
.join("; ");
Some(joined)
}
/// Convert display math to one or more Unicode lines.
///
/// Lines come from `\\` row separators and multi-row environments, which lay
/// out as 2D boxes anchored to the surrounding flow (see [`MathBox`]).
/// Leading whitespace is structural (box alignment) and preserved; only line
/// ends are trimmed. Returns `None` when the source is too large to convert,
/// and an empty `Vec` when the math has no visible content (callers should
/// fall back in both cases).
pub(crate) fn latex_to_unicode_display(src: &str) -> Option<Vec<String>> {
if src.len() > MAX_MATH_SOURCE_LEN {
return None;
}
let lines: Vec<String> = convert(src, false)
.into_iter()
.map(|l| l.trim_end().to_string())
.filter(|l| !l.is_empty())
.collect();
Some(lines)
}
/// Run the converter and return the output lines.
fn convert(src: &str, flat: bool) -> Vec<String> {
let mut cursor = Cursor::new(src);
let mut out = MathBox::new(flat);
render_sequence(&mut cursor, &mut out, 0, Mode::Math, None);
out.into_lines()
}
/// Rendering mode: math mode applies typographic substitutions (`-` → `−`,
/// `'` → `′`) that text fragments (`\text{...}`) must not receive.
#[derive(Copy, Clone, PartialEq, Eq)]
enum Mode {
Math,
Text,
}

412
crates/codegen/xai-grok-markdown/src/latex/symbols.rs Normal file
View file

@ -0,0 +1,412 @@
//! Character and symbol mapping tables.
pub(super) fn to_superscript(c: char) -> Option<char> {
Some(match c {
'0' => '⁰',
'1' => '¹',
'2' => '²',
'3' => '³',
'4' => '⁴',
'5' => '⁵',
'6' => '⁶',
'7' => '⁷',
'8' => '⁸',
'9' => '⁹',
'+' => '⁺',
'-' | '−' => '⁻',
'=' => '⁼',
'(' => '⁽',
')' => '⁾',
'a' => 'ᵃ',
'b' => 'ᵇ',
'c' => 'ᶜ',
'd' => 'ᵈ',
'e' => 'ᵉ',
'f' => 'ᶠ',
'g' => 'ᵍ',
'h' => 'ʰ',
'i' => 'ⁱ',
'j' => 'ʲ',
'k' => 'ᵏ',
'l' => 'ˡ',
'm' => 'ᵐ',
'n' => 'ⁿ',
'o' => 'ᵒ',
'p' => 'ᵖ',
'r' => 'ʳ',
's' => 'ˢ',
't' => 'ᵗ',
'u' => 'ᵘ',
'v' => 'ᵛ',
'w' => 'ʷ',
'x' => 'ˣ',
'y' => 'ʸ',
'z' => 'ᶻ',
'T' => 'ᵀ',
'∗' | '*' => '*',
'′' | '\'' => '′',
' ' => ' ',
_ => return None,
})
}
pub(super) fn to_subscript(c: char) -> Option<char> {
Some(match c {
'0' => '₀',
'1' => '₁',
'2' => '₂',
'3' => '₃',
'4' => '₄',
'5' => '₅',
'6' => '₆',
'7' => '₇',
'8' => '₈',
'9' => '₉',
'+' => '₊',
'-' | '−' => '₋',
'=' => '₌',
'(' => '₍',
')' => '₎',
'a' => 'ₐ',
'e' => 'ₑ',
'h' => 'ₕ',
'i' => 'ᵢ',
'j' => 'ⱼ',
'k' => 'ₖ',
'l' => 'ₗ',
'm' => 'ₘ',
'n' => 'ₙ',
'o' => 'ₒ',
'p' => 'ₚ',
'r' => 'ᵣ',
's' => 'ₛ',
't' => 'ₜ',
'u' => 'ᵤ',
'v' => 'ᵥ',
'x' => 'ₓ',
' ' => ' ',
_ => return None,
})
}
pub(super) fn map_mathbb(c: char) -> Option<char> {
Some(match c {
'C' => 'ℂ',
'H' => 'ℍ',
'N' => 'ℕ',
'P' => 'ℙ',
'Q' => 'ℚ',
'R' => 'ℝ',
'Z' => 'ℤ',
'A'..='Z' => char::from_u32(0x1D538 + (c as u32 - 'A' as u32))?,
'a'..='z' => char::from_u32(0x1D552 + (c as u32 - 'a' as u32))?,
'0'..='9' => char::from_u32(0x1D7D8 + (c as u32 - '0' as u32))?,
_ => return None,
})
}
pub(super) fn map_mathcal(c: char) -> Option<char> {
Some(match c {
'B' => 'ℬ',
'E' => 'ℰ',
'F' => 'ℱ',
'H' => 'ℋ',
'I' => 'ℐ',
'L' => 'ℒ',
'M' => 'ℳ',
'R' => 'ℛ',
'e' => 'ℯ',
'g' => 'ℊ',
'o' => 'ℴ',
'A'..='Z' => char::from_u32(0x1D49C + (c as u32 - 'A' as u32))?,
'a'..='z' => char::from_u32(0x1D4B6 + (c as u32 - 'a' as u32))?,
_ => return None,
})
}
pub(super) fn map_mathfrak(c: char) -> Option<char> {
Some(match c {
'C' => 'ℭ',
'H' => 'ℌ',
'I' => 'ℑ',
'R' => 'ℜ',
'Z' => 'ℨ',
'A'..='Z' => char::from_u32(0x1D504 + (c as u32 - 'A' as u32))?,
'a'..='z' => char::from_u32(0x1D51E + (c as u32 - 'a' as u32))?,
_ => return None,
})
}
pub(super) fn map_mathbf(c: char) -> Option<char> {
Some(match c {
'A'..='Z' => char::from_u32(0x1D400 + (c as u32 - 'A' as u32))?,
'a'..='z' => char::from_u32(0x1D41A + (c as u32 - 'a' as u32))?,
'0'..='9' => char::from_u32(0x1D7CE + (c as u32 - '0' as u32))?,
_ => return None,
})
}
/// Symbol command table (commands with no arguments).
pub(super) fn symbol(name: &str) -> Option<&'static str> {
Some(match name {
// Greek lowercase
"alpha" => "α",
"beta" => "β",
"gamma" => "γ",
"delta" => "δ",
"epsilon" => "ϵ",
"varepsilon" => "ε",
"zeta" => "ζ",
"eta" => "η",
"theta" => "θ",
"vartheta" => "ϑ",
"iota" => "ι",
"kappa" => "κ",
"lambda" => "λ",
"mu" => "μ",
"nu" => "ν",
"xi" => "ξ",
"omicron" => "ο",
"pi" => "π",
"varpi" => "ϖ",
"rho" => "ρ",
"varrho" => "ϱ",
"sigma" => "σ",
"varsigma" => "ς",
"tau" => "τ",
"upsilon" => "υ",
"phi" => "ϕ",
"varphi" => "φ",
"chi" => "χ",
"psi" => "ψ",
"omega" => "ω",
// Greek uppercase
"Gamma" => "Γ",
"Delta" => "Δ",
"Theta" => "Θ",
"Lambda" => "Λ",
"Xi" => "Ξ",
"Pi" => "Π",
"Sigma" => "Σ",
"Upsilon" => "Υ",
"Phi" => "Φ",
"Psi" => "Ψ",
"Omega" => "Ω",
// Big operators
"sum" => "∑",
"prod" => "∏",
"coprod" => "∐",
"int" => "∫",
"iint" => "∬",
"iiint" => "∭",
"oint" => "∮",
"bigcup" => "⋃",
"bigcap" => "⋂",
"bigvee" => "⋁",
"bigwedge" => "⋀",
"bigoplus" => "⨁",
"bigotimes" => "⨂",
"bigodot" => "⨀",
"biguplus" => "⨄",
// Named operators (render as plain words)
"lim" => "lim",
"limsup" => "lim sup",
"liminf" => "lim inf",
"sin" => "sin",
"cos" => "cos",
"tan" => "tan",
"cot" => "cot",
"sec" => "sec",
"csc" => "csc",
"arcsin" => "arcsin",
"arccos" => "arccos",
"arctan" => "arctan",
"sinh" => "sinh",
"cosh" => "cosh",
"tanh" => "tanh",
"coth" => "coth",
"log" => "log",
"ln" => "ln",
"lg" => "lg",
"exp" => "exp",
"max" => "max",
"min" => "min",
"sup" => "sup",
"inf" => "inf",
"det" => "det",
"dim" => "dim",
"ker" => "ker",
"deg" => "deg",
"arg" => "arg",
"gcd" => "gcd",
"hom" => "hom",
"Pr" => "Pr",
// Binary operators
"times" => "×",
"cdot" => "⋅",
"div" => "÷",
"pm" => "±",
"mp" => "∓",
"ast" => "∗",
"star" => "⋆",
"circ" => "∘",
"bullet" => "•",
"oplus" => "⊕",
"ominus" => "⊖",
"otimes" => "⊗",
"oslash" => "⊘",
"odot" => "⊙",
"wedge" | "land" => "∧",
"vee" | "lor" => "∨",
"cap" => "∩",
"cup" => "∪",
"setminus" => "∖",
"smallsetminus" => "∖",
"uplus" => "⊎",
"sqcap" => "⊓",
"sqcup" => "⊔",
"triangleleft" => "◁",
"triangleright" => "▷",
"wr" => "≀",
"diamond" => "⋄",
"dagger" => "†",
"ddagger" => "‡",
"amalg" => "⨿",
// Relations
"le" | "leq" | "leqslant" => "≤",
"ge" | "geq" | "geqslant" => "≥",
"ne" | "neq" => "≠",
"ll" => "≪",
"gg" => "≫",
"approx" => "≈",
"sim" => "∼",
"simeq" => "≃",
"cong" => "≅",
"equiv" => "≡",
"doteq" => "≐",
"propto" => "∝",
"prec" => "≺",
"succ" => "≻",
"preceq" => "⪯",
"succeq" => "⪰",
"asymp" => "≍",
"in" => "∈",
"ni" | "owns" => "∋",
"notin" => "∉",
"subset" => "⊂",
"supset" => "⊃",
"subseteq" => "⊆",
"supseteq" => "⊇",
"subsetneq" => "⊊",
"supsetneq" => "⊋",
"sqsubseteq" => "⊑",
"sqsupseteq" => "⊒",
"vdash" => "⊢",
"dashv" => "⊣",
"models" | "vDash" => "⊨",
"perp" => "⊥",
"parallel" => "∥",
"nparallel" => "∦",
"mid" => "∣",
"nmid" => "∤",
"smile" => "⌣",
"frown" => "⌢",
"bowtie" => "⋈",
// Arrows
"to" | "rightarrow" => "→",
"leftarrow" | "gets" => "←",
"leftrightarrow" => "↔",
"Rightarrow" => "⇒",
"Leftarrow" => "⇐",
"Leftrightarrow" => "⇔",
"implies" => "⟹",
"impliedby" => "⟸",
"iff" => "⟺",
"longrightarrow" => "⟶",
"longleftarrow" => "⟵",
"longmapsto" => "⟼",
"mapsto" => "↦",
"uparrow" => "↑",
"downarrow" => "↓",
"updownarrow" => "↕",
"Uparrow" => "⇑",
"Downarrow" => "⇓",
"nearrow" => "↗",
"searrow" => "↘",
"swarrow" => "↙",
"nwarrow" => "↖",
"hookrightarrow" => "↪",
"hookleftarrow" => "↩",
"rightharpoonup" => "⇀",
"leftharpoonup" => "↼",
"rightleftharpoons" => "⇌",
// Logic / sets / misc letters
"forall" => "∀",
"exists" => "∃",
"nexists" => "∄",
"neg" | "lnot" => "¬",
"emptyset" | "varnothing" => "∅",
"infty" => "∞",
"nabla" => "∇",
"partial" => "∂",
"hbar" => "ℏ",
"ell" => "ℓ",
"Re" => "ℜ",
"Im" => "ℑ",
"aleph" => "ℵ",
"beth" => "ℶ",
"wp" => "℘",
"imath" => "ı",
"jmath" => "ȷ",
"top" => "⊤",
"bot" => "⊥",
"angle" => "∠",
"measuredangle" => "∡",
"triangle" => "△",
"square" | "Box" => "□",
"blacksquare" => "■",
"diamondsuit" => "♦",
"heartsuit" => "♥",
"clubsuit" => "♣",
"spadesuit" => "♠",
"flat" => "♭",
"natural" => "♮",
"sharp" => "♯",
"checkmark" => "✓",
"degree" => "°",
"prime" => "′",
"dprime" => "″",
"therefore" => "∴",
"because" => "∵",
"dots" | "ldots" | "dotsc" | "dotso" | "dotsb" | "dotsm" => "…",
"cdots" => "⋯",
"vdots" => "⋮",
"ddots" => "⋱",
"surd" => "√",
"AA" => "Å",
// Delimiters
"langle" => "⟨",
"rangle" => "⟩",
"lceil" => "⌈",
"rceil" => "⌉",
"lfloor" => "⌊",
"rfloor" => "⌋",
"lbrace" => "{",
"rbrace" => "}",
"lbrack" => "[",
"rbrack" => "]",
"vert" => "|",
"Vert" | "|" => "‖",
"backslash" => "\\",
"setbslash" => "∖",
// Escaped literals
"{" => "{",
"}" => "}",
"%" => "%",
"$" => "$",
"&" => "&",
"#" => "#",
"_" => "_",
_ => return None,
})
}

370
crates/codegen/xai-grok-markdown/src/latex/tests.rs Normal file
View file

@ -0,0 +1,370 @@
use super::*;
fn inline(src: &str) -> String {
latex_to_unicode_inline(src).expect("within size limit")
}
fn display(src: &str) -> Vec<String> {
latex_to_unicode_display(src).expect("within size limit")
}
#[test]
fn plain_expression_passes_through() {
assert_eq!(inline("E = mc"), "E = mc");
}
#[test]
fn superscripts_map_to_unicode() {
assert_eq!(inline("E = mc^2"), "E = mc²");
assert_eq!(inline("x^{10}"), "x¹⁰");
assert_eq!(inline("e^{-x}"), "e⁻ˣ");
assert_eq!(inline("x^T"), "xᵀ");
}
#[test]
fn subscripts_map_to_unicode() {
assert_eq!(inline("a_1 + a_2"), "a₁ + a₂");
assert_eq!(inline("x_{ij}"), "xᵢⱼ");
}
#[test]
fn script_fallback_uses_parens() {
// φ has no superscript form → fall back to ^(...)
assert_eq!(inline("x^{\\alpha\\beta}"), "x^(αβ)");
assert_eq!(inline("x^\\alpha"), "x^α");
// Single unmappable subscript char.
assert_eq!(inline("a_q"), "a_q");
}
#[test]
fn wordlike_scripts_fall_back_to_parens() {
// Text-family commands mark the atom as a word → no modifier-letter runs
// (`pₜₒᵣₛₒ` is unreadable and gappy in many terminal fonts).
assert_eq!(inline("p_{\\text{torso}}"), "p_(torso)");
assert_eq!(inline("z_{\\mathrm{draft}}"), "z_(draft)");
assert_eq!(inline("x^{\\text{opt}}"), "x^(opt)");
// 3+ letter runs read as words even without \text.
assert_eq!(inline("x_{max}"), "x_(max)");
assert_eq!(inline("z_{torso}"), "z_(torso)");
}
#[test]
fn indexlike_scripts_keep_unicode_forms() {
// 1–2 letter runs are index juxtapositions, not words.
assert_eq!(inline("x_{ij}"), "xᵢⱼ");
assert_eq!(inline("T_{i+1}"), "Tᵢ₊₁");
assert_eq!(inline("n^{th}"), "nᵗʰ");
assert_eq!(inline("\\sum_{i=0}^{2} \\gamma^{i}"), "∑ᵢ₌₀² γⁱ");
}
#[test]
fn boxed_renders_content_without_frame() {
assert_eq!(inline("\\boxed{x = 1}"), "x = 1");
assert_eq!(inline("\\boxed{\\mathcal{L}}"), "ℒ");
assert_eq!(inline("\\fbox{done}"), "done");
// Math typography applies inside \boxed (math mode) …
assert_eq!(inline("\\boxed{a - b}"), "a − b");
// … but not inside \fbox (text mode).
assert_eq!(inline("\\fbox{a-b}"), "a-b");
}
#[test]
fn mtp_loss_equation_converts_fully() {
// A complex real-world equation: every command must
// convert — no literal command names in the output.
let src = "\\boxed{\n\\mathcal{L}_{\\text{MTP}}\n=\n\\sum_{i=0}^{2}\n\\gamma^{i}\\,\n\\mathbb{E}_{\\text{positions, mask}}\n\\Big[\n\\mathrm{KL}\\big(\n \\mathrm{softmax}(z_{\\text{torso}}^{(s_i)})\n \\;\\big\\|\\;\n \\mathrm{softmax}(z_{\\text{draft}}^{(i)})\n\\big)\n\\Big]\n}";
let joined = inline(src);
assert!(joined.contains("ℒ_(MTP)"), "got: {joined}");
assert!(joined.contains("∑ᵢ₌₀²"), "got: {joined}");
assert!(joined.contains("𝔼_(positions, mask)"), "got: {joined}");
assert!(joined.contains("softmax(z_(torso)"), "got: {joined}");
assert!(joined.contains("‖"), "got: {joined}");
assert!(!joined.contains("boxed"), "got: {joined}");
assert!(!joined.contains('\\'), "got: {joined}");
}
#[test]
fn greek_letters() {
assert_eq!(inline("\\alpha + \\beta = \\Gamma"), "α + β = Γ");
assert_eq!(inline("\\varepsilon \\varphi"), "ε φ");
}
#[test]
fn relations_and_operators() {
assert_eq!(inline("a \\le b \\ne c \\times d"), "a ≤ b ≠ c × d");
assert_eq!(inline("x \\in A \\cup B"), "x ∈ A ∪ B");
assert_eq!(inline("p \\implies q"), "p ⟹ q");
assert_eq!(inline("f: A \\to B"), "f: A → B");
}
#[test]
fn vulgar_and_general_fractions() {
assert_eq!(inline("\\frac{1}{2}"), "½");
assert_eq!(inline("\\frac{3}{4}"), "¾");
assert_eq!(inline("\\frac{dy}{dx}"), "dy/dx");
assert_eq!(inline("\\frac{a+b}{c}"), "(a+b)/c");
assert_eq!(inline("\\frac{x}{y - z}"), "x/(y − z)");
}
#[test]
fn roots() {
assert_eq!(inline("\\sqrt{x}"), "√x");
assert_eq!(inline("\\sqrt{a + b}"), "√(a + b)");
assert_eq!(inline("\\sqrt[3]{x}"), "∛x");
assert_eq!(inline("\\sqrt[4]{x}"), "∜x");
assert_eq!(inline("\\sqrt[n]{x}"), "ⁿ√x");
}
#[test]
fn text_commands_pass_content_through() {
assert_eq!(inline("\\text{if } x > 0"), "if x > 0");
assert_eq!(inline("\\mathrm{d}x"), "dx");
assert_eq!(inline("\\operatorname{softmax}(z)"), "softmax(z)");
// Text mode must not map `-` to minus.
assert_eq!(inline("\\text{x-ray}"), "x-ray");
}
#[test]
fn alphabets() {
assert_eq!(inline("\\mathbb{R}^n"), "ℝⁿ");
assert_eq!(inline("\\mathbb{N} \\mathbb{Z} \\mathbb{Q}"), "ℕ ℤ ℚ");
assert_eq!(inline("\\mathcal{L}"), "ℒ");
assert_eq!(inline("\\mathcal{O}(n)"), "𝒪(n)");
assert_eq!(inline("\\mathfrak{g}"), "𝔤");
assert_eq!(inline("\\mathbf{v}"), "𝐯");
}
#[test]
fn accents_use_combining_marks() {
assert_eq!(inline("\\hat{x}"), "x\u{0302}");
assert_eq!(inline("\\bar{y}"), "y\u{0304}");
assert_eq!(inline("\\vec{v}"), "v\u{20D7}");
assert_eq!(inline("\\dot{q}"), "q\u{0307}");
assert_eq!(inline("\\tilde\\theta"), "θ\u{0303}");
}
#[test]
fn left_right_and_spacing() {
assert_eq!(inline("\\left( \\frac{1}{2} \\right)"), "( ½ )".to_string());
assert_eq!(inline("\\left. x \\right|_0^1"), "x |₀¹");
assert_eq!(inline("\\int f(x)\\,dx"), "∫ f(x) dx");
assert_eq!(inline("a\\!b"), "ab");
assert_eq!(inline("a \\quad b"), "a b");
}
#[test]
fn named_function_operators() {
assert_eq!(inline("\\sin(x) + \\cos(y)"), "sin(x) + cos(y)");
assert_eq!(inline("\\lim_{x \\to 0} f(x)"), "lim_(x → 0) f(x)");
assert_eq!(inline("\\log n"), "log n");
}
#[test]
fn integrals_and_sums_with_bounds() {
assert_eq!(inline("\\int_0^\\infty e^{-x} dx"), "∫₀^∞ e⁻ˣ dx");
assert_eq!(inline("\\sum_{i=1}^{n} a_i"), "∑ᵢ₌₁ⁿ aᵢ");
}
#[test]
fn minus_and_prime_typography() {
assert_eq!(inline("a - b"), "a − b");
assert_eq!(inline("f'(x)"), "f′(x)");
}
#[test]
fn not_negates_known_relations() {
assert_eq!(inline("a \\not= b"), "a ≠ b");
assert_eq!(inline("x \\not\\in S"), "x ∉ S");
assert_eq!(inline("a \\not\\sim b"), "a ∼\u{0338} b");
}
#[test]
fn binomials_and_mod() {
assert_eq!(inline("\\binom{n}{k}"), "C(n, k)");
assert_eq!(inline("a \\equiv b \\pmod{m}"), "a ≡ b (mod m)");
assert_eq!(inline("a \\bmod b"), "a mod b");
}
#[test]
fn row_breaks_join_inline_and_split_display() {
assert_eq!(inline("a \\\\ b"), "a; b");
assert_eq!(display("a \\\\ b"), vec!["a", "b"]);
}
#[test]
fn aligned_environment_strips_markers() {
let lines = display("\\begin{aligned} x &= y + 1 \\\\ y &= 2 \\end{aligned}");
assert_eq!(lines, vec!["x = y + 1", "y = 2"]);
}
#[test]
fn cases_environment_renders_brace_column() {
let lines = display("f(x) = \\begin{cases} x & x > 0 \\\\ 0 & \\text{otherwise} \\end{cases}");
assert_eq!(lines.len(), 2);
assert!(lines[0].starts_with("f(x) = ⎧ x"), "got {lines:?}");
assert!(lines[1].trim_start().starts_with("⎩ 0"), "got {lines:?}");
}
#[test]
fn pmatrix_pads_columns() {
let lines = display("\\begin{pmatrix} 1 & 22 \\\\ 333 & 4 \\end{pmatrix}");
assert_eq!(lines, vec!["⎛1 22⎞", "⎝333 4⎠"]);
}
#[test]
fn bmatrix_single_row_uses_flat_brackets() {
assert_eq!(
display("\\begin{bmatrix} a & b \\end{bmatrix}"),
vec!["[a b]"]
);
}
#[test]
fn vmatrix_uses_bars() {
let lines = display("\\begin{vmatrix} a & b \\\\ c & d \\end{vmatrix}");
assert_eq!(lines, vec!["│a b│", "│c d│"]);
}
#[test]
fn matrix_with_prefix_aligns_as_box() {
// The prefix must stay on the anchor row with the matrix body
// aligned beneath — not glued to the first row only.
let lines = display("A = \\begin{pmatrix} 1 & 2 \\\\ 3 & 4 \\end{pmatrix}");
assert_eq!(lines, vec!["A = ⎛1 2⎞", " ⎝3 4⎠"]);
}
#[test]
fn matrix_with_prefix_and_suffix_flows_on_anchor_row() {
let lines =
display("A = \\begin{pmatrix} 1 & 2 \\\\ 3 & 4 \\end{pmatrix}, \\quad \\det(A) = -2");
assert_eq!(lines, vec!["A = ⎛1 2⎞, det(A) = −2", " ⎝3 4⎠"]);
}
#[test]
fn three_row_matrix_anchors_on_middle_row() {
let lines = display("v = \\begin{pmatrix} 1 \\\\ 2 \\\\ 3 \\end{pmatrix} x");
assert_eq!(lines, vec![" ⎛1⎞", "v = ⎜2⎟ x", " ⎝3⎠"]);
}
#[test]
fn cases_with_prefix_aligns_as_box() {
let lines = display("f(x) = \\begin{cases} x & x > 0 \\\\ 0 & e \\end{cases}");
assert_eq!(lines, vec!["f(x) = ⎧ x x > 0", " ⎩ 0 e"]);
}
#[test]
fn inline_matrix_renders_flat() {
assert_eq!(
inline("\\begin{pmatrix} 1 & 2 \\\\ 3 & 4 \\end{pmatrix}"),
"(1 2; 3 4)"
);
assert_eq!(inline("\\begin{bmatrix} a \\\\ b \\end{bmatrix}"), "[a; b]");
}
#[test]
fn inline_cases_renders_flat() {
assert_eq!(
inline("\\begin{cases} x & x > 0 \\\\ 0 & e \\end{cases}"),
"{x x > 0; 0 e}"
);
}
#[test]
fn two_matrices_on_one_line_share_rows() {
let lines = display(
"\\begin{pmatrix} 1 \\\\ 2 \\end{pmatrix} + \\begin{pmatrix} 3 \\\\ 4 \\end{pmatrix}",
);
assert_eq!(lines, vec!["⎛1⎞ + ⎛3⎞", "⎝2⎠ ⎝4⎠"]);
}
#[test]
fn row_break_then_matrix_does_not_disturb_previous_line() {
let lines = display("a \\\\ B = \\begin{pmatrix} 1 \\\\ 2 \\end{pmatrix}");
assert_eq!(lines, vec!["a", "B = ⎛1⎞", " ⎝2⎠"]);
}
#[test]
fn unknown_environment_renders_rows() {
let lines = display("\\begin{foo} a \\\\ b \\end{foo}");
assert_eq!(lines, vec!["a", "b"]);
}
#[test]
fn nested_environment_resolves_matching_end() {
let lines = display(
"\\begin{aligned} A &= \\begin{pmatrix} 1 \\end{pmatrix} \\\\ B &= 2 \\end{aligned}",
);
assert_eq!(lines, vec!["A = (1)", "B = 2"]);
}
#[test]
fn unknown_commands_keep_their_name() {
assert_eq!(inline("\\foobar x"), "foobar x");
}
#[test]
fn overset_and_stackrel() {
assert_eq!(inline("a \\overset{!}{=} b"), "a = b");
assert_eq!(inline("a \\overset{n}{=} b"), "a =ⁿ b");
}
#[test]
fn malformed_input_does_not_panic() {
for src in [
"",
"{",
"}",
"\\",
"\\frac{a}",
"\\frac",
"\\sqrt[",
"\\begin{aligned} x",
"\\begin",
"\\end{x}",
"^",
"_",
"^{",
"a^",
"{{{{{{",
"\\left",
"\\not",
"$$$",
"\\\\\\",
"&&&&",
] {
let _ = latex_to_unicode_inline(src);
let _ = latex_to_unicode_display(src);
}
}
#[test]
fn deeply_nested_input_is_bounded() {
let mut src = String::new();
for _ in 0..200 {
src.push('{');
}
src.push('x');
for _ in 0..200 {
src.push('}');
}
let _ = latex_to_unicode_inline(&src);
}
#[test]
fn oversized_input_is_rejected() {
let big = "x".repeat(MAX_MATH_SOURCE_LEN + 1);
assert!(latex_to_unicode_inline(&big).is_none());
assert!(latex_to_unicode_display(&big).is_none());
}
#[test]
fn whitespace_only_display_is_empty() {
assert!(display(" \n ").is_empty());
}
#[test]
fn escaped_literals() {
assert_eq!(inline("100\\%"), "100%");
assert_eq!(inline("\\{a, b\\}"), "{a, b}");
assert_eq!(inline("\\$5"), "$5");
}

1305
crates/codegen/xai-grok-markdown/src/latex_delimiters.rs Normal file
View file

File diff suppressed because it is too large Load diff

182
crates/codegen/xai-grok-markdown/src/lib.rs Normal file
View file

@ -0,0 +1,182 @@
//! Streaming markdown renderer for terminal UIs.
//!
//! This crate provides incremental/streaming markdown rendering optimized for
//! displaying LLM responses in terminal UIs. Key features:
//!
//! - **Streaming rendering**: Efficiently render markdown as it arrives chunk by chunk
//! - **Checkpoint-based freezing**: Only re-render the "tail" after stable boundaries
//! - **Syntax highlighting**: Code blocks highlighted via syntect
//! - **Terminal color adaptation**: Automatic downgrade for 256-color/16-color terminals
//! - **LaTeX math rendering**: `$...$`, `$$...$$`, `\(...\)` and `\[...\]` math is
//! converted to a Unicode approximation (`$E=mc^2$` → `E=mc²`) in pretty mode
//!
//! # Example
//!
//! ```ignore
//! use xai_grok_markdown::{StreamingMarkdownRenderer, MarkdownStyle, Syntect};
//!
//! let syntect = Syntect::new(include_bytes!("theme.tmTheme"));
//! let style = MarkdownStyle::default();
//! let mut renderer = StreamingMarkdownRenderer::new(style, true);
//!
//! for token in stream {
//! renderer.push_and_render(&token, Some(&syntect));
//! let view = renderer.view();
//! // display view.lines
//! }
//! ```
mod buffers;
pub mod checkpoint;
mod colors;
mod hyperlinks;
mod latex;
mod latex_delimiters;
mod mermaid;
mod open_code_highlighter;
mod output;
mod parse;
mod render;
mod source_map;
pub mod streaming;
pub mod style;
mod syntax;
mod url_scan;
// Re-export public API
pub use buffers::MarkdownBuffers;
pub use checkpoint::{Checkpoint, CheckpointKind};
pub use colors::{
ColorLevel, adapt_color, adapt_style, detect_color_level, get_color_level, set_color_level_cap,
};
pub use latex_delimiters::{LatexDelimiterNormalizer, normalize_latex_delimiters};
pub use output::{CodeBlockSpan, HyperlinkTarget, MarkdownRenderOutput, MarkdownRenderView};
pub use parse::{MarkdownParser, ParsedMarkdown};
pub use source_map::SourceMap;
pub use streaming::StreamingMarkdownRenderer;
pub use style::{MarkdownStyle, TableBorders};
pub use syntax::Syntect;
// Re-export test helpers when fuzzing
#[cfg(fuzzing)]
pub use syntax::test_syntect;
/// Render markdown to ratatui Lines with full output including checkpoint.
///
/// Runs the parser pass followed by the `url_scan` pass so the returned
/// output's `hyperlinks` mirrors what `StreamingMarkdownRenderer::finish()`
/// produces for the same input (plain-URL detection for the pretty-mode
/// `(url)` suffix and bare URLs in prose).
pub fn render_markdown_ratatui_full(
text: &str,
ms: MarkdownStyle,
pretty: bool,
syntect: Option<&Syntect>,
) -> (MarkdownRenderOutput, Option<Checkpoint>) {
let mut buffers = MarkdownBuffers::new();
render_markdown_ratatui_with_buffers(text, ms, pretty, &mut buffers, syntect)
}
/// Render markdown to ratatui Lines, reusing the provided buffers.
pub fn render_markdown_ratatui_with_buffers(
text: &str,
ms: MarkdownStyle,
pretty: bool,
buffers: &mut MarkdownBuffers,
syntect: Option<&Syntect>,
) -> (MarkdownRenderOutput, Option<Checkpoint>) {
render_markdown_ratatui_with_buffers_width(text, ms, pretty, buffers, syntect, None)
}
/// Render markdown to ratatui Lines, reusing the provided buffers,
/// with an optional maximum table width.
pub fn render_markdown_ratatui_with_buffers_width(
text: &str,
ms: MarkdownStyle,
pretty: bool,
buffers: &mut MarkdownBuffers,
syntect: Option<&Syntect>,
max_table_width: Option<usize>,
) -> (MarkdownRenderOutput, Option<Checkpoint>) {
// Normalize LaTeX delimiters (`\(…\)`/`\[…\]`/`\begin{equation}`) into the
// canonical `$`/`$$` forms before parsing, so the math handlers convert them
// uniformly (incl. inside table cells). All offsets are in normalized space;
// `StreamingMarkdownRenderer` normalizes at ingestion so its stored source
// matches. Streaming tail renders (`render_markdown_ratatui_with_link_id`)
// do NOT re-normalize — they receive already-normalized source.
let normalized = latex_delimiters::normalize_latex_delimiters(text);
let mut parsed = MarkdownParser::new(&normalized, ms, buffers, syntect)
.max_table_width(max_table_width)
.parse();
let next_link_id = parsed.next_link_id;
let (mut output, checkpoint) = parsed.render_ratatui(pretty);
// Mirror `StreamingMarkdownRenderer::finish()`: detect plain URLs
// so a one-shot full render produces the same hyperlinks a
// `push_and_render` + `finish()` sequence would.
let (extra_links, _post_scan_next_id) =
url_scan::detect_plain_urls(&output.lines, &output.hyperlinks, next_link_id);
output.hyperlinks.extend(extra_links);
output
.hyperlinks
.sort_by_key(|h| (h.line_index, h.column_range.start));
(output, checkpoint)
}
/// Render markdown to ratatui Lines and provide `next_link_id` so the
/// streaming renderer can resume link ID assignment across tail re-renders.
///
/// `open_code` threads an optional incremental highlighter for the trailing
/// still-open fenced code block: only the streaming tail re-render passes
/// `Some(cache)`; `finish()` and non-streaming callers pass `None`. Everything
/// other than that one open block (closed code blocks, HTML, math, tables,
/// inline) always goes through the unchanged batch highlighter, so output is
/// byte-for-byte identical to the cache-less path. See [`open_code_highlighter`].
#[allow(clippy::too_many_arguments)]
pub(crate) fn render_markdown_ratatui_with_link_id(
text: &str,
ms: MarkdownStyle,
pretty: bool,
buffers: &mut MarkdownBuffers,
syntect: Option<&Syntect>,
max_table_width: Option<usize>,
link_id_start: u32,
collapse_soft_breaks: bool,
open_code: Option<&mut open_code_highlighter::OpenCodeHighlighter>,
) -> (MarkdownRenderOutput, Option<Checkpoint>, u32) {
let mut parsed = MarkdownParser::new(text, ms, buffers, syntect)
.max_table_width(max_table_width)
.link_id_start(link_id_start)
.collapse_soft_breaks(collapse_soft_breaks)
.open_code(open_code)
.parse();
// NOTE: There can be multiple links in a tail, hence next_link_id is the return.
let next_link_id = parsed.next_link_id;
let (output, checkpoint) = parsed.render_ratatui(pretty);
(output, checkpoint, next_link_id)
}
/// Render markdown to an ANSI-styled string.
pub fn render_markdown(
text: &str,
ms: MarkdownStyle,
pretty: bool,
syntect: Option<&Syntect>,
) -> (String, SourceMap) {
let mut buffers = MarkdownBuffers::new();
let normalized = latex_delimiters::normalize_latex_delimiters(text);
MarkdownParser::new(&normalized, ms, &mut buffers, syntect)
.parse()
.render_ansi(pretty)
}
/// Render markdown to ratatui Lines (simple API).
pub fn render_markdown_ratatui(
text: &str,
ms: MarkdownStyle,
pretty: bool,
syntect: Option<&Syntect>,
) -> (Vec<ratatui::text::Line<'static>>, Vec<usize>) {
let (out, _checkpoint) = render_markdown_ratatui_full(text, ms, pretty, syntect);
(out.lines, out.line_source_map)
}

5237
crates/codegen/xai-grok-markdown/src/mermaid.rs Normal file
View file

File diff suppressed because it is too large Load diff

439
crates/codegen/xai-grok-markdown/src/open_code_highlighter.rs Normal file
View file

@ -0,0 +1,439 @@
//! Streaming-render syntect caches for fenced code blocks in the unfrozen
//! tail. Two complementary strategies behind one entry point
//! ([`OpenCodeHighlighter::highlight_block`]):
//!
//! - **Still-open trailing block** (closing ``` not arrived): persists
//! syntect's *resumable* per-line state ([`ParseState`]/[`HighlightState`])
//! across `rerender_tail` calls so each committed line is highlighted
//! exactly once. Without it, every push re-ran syntect over the whole
//! growing block — O(N²) over the stream (~35 ms/push near the end of a
//! ~1000-line block).
//! - **Closed blocks trapped in the tail** (e.g. inside an open list, which
//! can never checkpoint): memoizes the batch highlight per
//! `(fence_info, body)` so syntect runs once per distinct fence body
//! instead of once per streamed chunk (~50–100 ms per re-run,
//! recorded 4.5 s UI freeze).
//!
//! Both paths are byte-identical to a one-shot batch render. Invalidation is
//! wholesale: the streaming renderer drops this struct on any
//! theme/style/width reset.
//!
//! # Invariants relied upon (open-block path)
//!
//! - **Append-only:** while a block is open the source only grows by appending,
//! and (because nothing freezes) the block's start offset within the tail is
//! stable. Both are guarded defensively here; on any mismatch the persisted
//! state is discarded and rebuilt from scratch.
//! - **One `Event::Text` per pass:** `TextMergeWithOffset` coalesces the block
//! body into a single `Event::Text`, so this is invoked once per block per
//! render pass and only needs to persist *across* passes.
use std::collections::HashMap;
use syntect::highlighting::{
HighlightIterator, HighlightState, Highlighter, Style as SyntectStyle,
};
use syntect::parsing::{ParseState, ScopeStack};
use syntect::util::LinesWithEndings;
use crate::syntax::{Syntect, syntax_highlight_raw};
/// Per-line highlight output: styled `(style, text)` segments for one line.
type HlLine = Vec<(SyntectStyle, String)>;
/// Byte budget for memoized closed-fence bodies; cleared wholesale on
/// overflow. Sized in body bytes (not entries) because pulldown can split a
/// list-indented fence into per-line `Event::Text` fragments — an entry
/// count would overflow on one large fence. If live bodies ever exceed the
/// budget the memo degrades to recomputing each pass (the pre-memo batch
/// behavior), never to unbounded memory or wrong output.
const CLOSED_MEMO_CAP_BYTES: usize = 256 * 1024;
/// Streaming syntect caches for fenced code blocks in the unfrozen tail:
/// incremental state for the single still-open trailing block, plus a memo
/// for closed blocks the tail re-parses every pass (see module docs).
///
/// Owns all the low-level syntect state so the parser/renderer don't have to.
pub(crate) struct OpenCodeHighlighter {
/// Language/info token of the block currently cached. A change means a
/// different syntax (and colors), so the cache must be rebuilt.
fence_info: String,
/// Block start offset within the tail. A change means we are looking at a
/// different block, so the cache must be rebuilt.
start_in_tail: usize,
/// Bytes highlighted up to and including the last committed `\n`.
committed_len: usize,
/// Highlighted, newline-terminated lines (one entry per committed line).
committed_lines: Vec<HlLine>,
/// syntect parse state AFTER the last committed (newline-terminated) line.
parse_state: ParseState,
/// syntect highlight state AFTER the last committed line.
highlight_state: HighlightState,
/// Memo for **closed** fences still in the unfrozen tail
/// (`fence_info -> body -> highlighted lines`). Nested maps keep the hot
/// lookup allocation-free; invalidation is inherited from `self` (the
/// streaming renderer drops this struct on any theme/style/width reset).
closed_memo: HashMap<String, HashMap<String, Vec<HlLine>>>,
/// Total body bytes currently memoized, for the `CLOSED_MEMO_CAP_BYTES`
/// budget check.
closed_memo_bytes: usize,
}
impl OpenCodeHighlighter {
/// Create an empty cache. The `parse_state`/`highlight_state` are seeded
/// with the plain-text syntax purely as placeholders: the empty
/// `fence_info` sentinel guarantees the first real
/// [`highlight`](Self::highlight) call takes the rebuild branch and
/// discards them in favour of the correct syntax.
pub(crate) fn new(syn: &Syntect) -> Self {
let highlighter = Highlighter::new(&syn.theme);
Self {
fence_info: String::new(),
start_in_tail: 0,
committed_len: 0,
committed_lines: Vec::new(),
// Seeded invalid; rebuilt on first highlight (see doc above).
parse_state: ParseState::new(syn.syntax_set.find_syntax_plain_text()),
highlight_state: HighlightState::new(&highlighter, ScopeStack::new()),
closed_memo: HashMap::new(),
closed_memo_bytes: 0,
}
}
/// Highlight a fenced block body from the streaming tail, routing to the
/// right cache: the incremental open-block path when the body reaches the
/// EOF of the tail (block still streaming), the closed-fence memo
/// otherwise. Single entry point so the parser carries no cache policy.
pub(crate) fn highlight_block(
&mut self,
syn: &Syntect,
fence_info: &str,
start_in_tail: usize,
body_reaches_eof: bool,
text: &str,
) -> Option<Vec<HlLine>> {
if body_reaches_eof {
self.highlight(syn, fence_info, start_in_tail, text)
} else {
self.highlight_closed(syn, fence_info, text)
}
}
/// Batch-highlight a **closed** fence body, memoized on
/// `(fence_info, body)`.
///
/// Closed fences trapped in an unfreezable tail (e.g. inside an open
/// list) are re-parsed by every `rerender_tail` pass; the memo makes
/// syntect run once per distinct body. The compute path *is*
/// [`syntax_highlight_raw`], so output is byte-identical by construction.
/// Theme stability follows [`highlight`](Self::highlight): the streaming
/// renderer drops this struct on any style change.
fn highlight_closed(
&mut self,
syn: &Syntect,
fence_info: &str,
text: &str,
) -> Option<Vec<HlLine>> {
if let Some(hit) = self.closed_memo.get(fence_info).and_then(|m| m.get(text)) {
// Hit clone is the same accepted O(lines)/pass residual as the
// open-block return below (see TODO on `highlight`).
return Some(hit.clone());
}
let lines = syntax_highlight_raw(Some(syn), fence_info, text)?;
if self.closed_memo_bytes.saturating_add(text.len()) > CLOSED_MEMO_CAP_BYTES {
self.closed_memo.clear();
self.closed_memo_bytes = 0;
}
let prev = self
.closed_memo
.entry(fence_info.to_owned())
.or_default()
.insert(text.to_owned(), lines.clone());
debug_assert!(prev.is_none(), "miss-checked key cannot already exist");
self.closed_memo_bytes += text.len();
Some(lines)
}
/// Test-only view of memoized closed-fence body bytes.
#[cfg(test)]
pub(crate) fn closed_memo_bytes(&self) -> usize {
self.closed_memo_bytes
}
/// Highlight the open block body `text` (the full body so far, append-only),
/// reusing persisted syntect state where possible.
///
/// Returns one styled line per source line (including the trailing partial
/// line if the body does not end in `\n`), matching what a batch
/// `HighlightLines` run would produce. Returns `None` if the fence has no
/// known syntax or a line fails to parse, so the caller can fall back to the
/// plain/untagged code path exactly like [`syntax_highlight_raw`].
///
/// # Theme stability invariant
///
/// The persisted state and the already-highlighted `committed_lines` bake
/// in the colors of the `syn.theme` seen so far, so the caller MUST pass a
/// [`Syntect`] whose `theme` is stable for the lifetime of a given open
/// block. A theme swap must go through a cache reset (the streaming renderer
/// does this in `set_style`); otherwise committed lines keep their old
/// colors while newly-committed lines use the new theme. The batch path has
/// no such constraint because it re-highlights from scratch every call.
fn highlight(
&mut self,
syn: &Syntect,
fence_info: &str,
start_in_tail: usize,
text: &str,
) -> Option<Vec<HlLine>> {
// Rebuild from scratch when anything that would change the output from
// the very first line changes: the language (different syntax/colors),
// the block position (a different block), or a non-append-only edit to
// the body (the committed prefix no longer matches `text`).
let needs_rebuild = fence_info != self.fence_info
|| start_in_tail != self.start_in_tail
|| !self.committed_prefix_matches(text);
if needs_rebuild {
// `syntax` is only needed to (re)seed the parser, so it is resolved
// here rather than on the warm path where it would be dead work.
let syntax = syn.find_syntax_for_fence_info(fence_info)?;
let highlighter = Highlighter::new(&syn.theme);
fence_info.clone_into(&mut self.fence_info);
self.start_in_tail = start_in_tail;
self.committed_len = 0;
self.committed_lines.clear();
self.parse_state = ParseState::new(syntax);
self.highlight_state = HighlightState::new(&highlighter, ScopeStack::new());
}
// Nothing new since the last committed `\n`: return the cached lines
// without constructing a `Highlighter` at all.
if self.committed_len == text.len() {
return Some(self.committed_lines.clone());
}
// Walk only the not-yet-committed remainder.
let highlighter = Highlighter::new(&syn.theme);
let mut tentative: Option<HlLine> = None;
for line in LinesWithEndings::from(&text[self.committed_len..]) {
if line.ends_with('\n') {
// A newline-terminated line is final: highlight once and
// permanently advance the persisted state. On a (practically
// unreachable) parse error, invalidate the cache so the next
// pass rebuilds from scratch instead of resuming from a
// now-inconsistent `parse_state` — matching the stateless
// batch fallback.
let ops = match self.parse_state.parse_line(line, &syn.syntax_set) {
Ok(ops) => ops,
Err(_) => {
self.fence_info.clear();
return None;
}
};
let highlighted =
HighlightIterator::new(&mut self.highlight_state, &ops, line, &highlighter)
.map(|(s, t)| (s, t.to_string()))
.collect();
self.committed_lines.push(highlighted);
self.committed_len += line.len();
} else {
// The trailing line has no `\n` yet — it is still streaming and
// may be extended by the next push. Highlight it on CLONES so
// the committed state stays anchored at the last `\n`.
let mut parse_state = self.parse_state.clone();
let mut highlight_state = self.highlight_state.clone();
let ops = match parse_state.parse_line(line, &syn.syntax_set) {
Ok(ops) => ops,
Err(_) => {
self.fence_info.clear();
return None;
}
};
tentative = Some(
HighlightIterator::new(&mut highlight_state, &ops, line, &highlighter)
.map(|(s, t)| (s, t.to_string()))
.collect(),
);
}
}
// TODO: this clone keeps the open-block RETURN at O(lines)/pass
// = O(lines^2)/stream. It only copies precomputed style spans (the
// expensive syntect parse/highlight CPU is already O(N) total), and the
// surrounding tail render + url_scan are likewise O(N)/pass, so this is
// tracked as an accepted residual — not a regression. Removing it needs
// a borrowed return threaded through `Replace` + the render pipeline.
let mut out = self.committed_lines.clone();
if let Some(last) = tentative {
out.push(last);
}
Some(out)
}
/// Whether the committed prefix is still a prefix of `text` (append-only
/// safety check). Allocation-free: walks the stored styled segments, whose
/// texts concatenate back to the original first `committed_len` bytes.
fn committed_prefix_matches(&self, text: &str) -> bool {
if self.committed_len > text.len() {
return false;
}
let bytes = text.as_bytes();
let mut pos = 0;
for line in &self.committed_lines {
for (_, piece) in line {
let end = pos + piece.len();
if bytes.get(pos..end) != Some(piece.as_bytes()) {
return false;
}
pos = end;
}
}
pos == self.committed_len
}
}
#[cfg(test)]
mod tests {
use syntect::util::LinesWithEndings;
use super::*;
use crate::syntax::test_syntect;
/// Batch reference highlight, mirroring `parse::syntax_highlight_raw`.
fn batch(syn: &Syntect, fence: &str, text: &str) -> Vec<HlLine> {
let mut hl = syn
.highlight_lines_for_fence_info(fence)
.expect("syntax for fence");
LinesWithEndings::from(text)
.map(|line| {
hl.highlight_line(line, &syn.syntax_set)
.expect("highlight line")
.into_iter()
.map(|(s, t)| (s, t.to_string()))
.collect()
})
.collect()
}
#[test]
fn append_only_growth_matches_fresh_full_highlight() {
let syn = test_syntect();
let full = "foo: 1\nbar:\n - a\n - b\nbaz: true\n";
let mut cache = OpenCodeHighlighter::new(syn);
// Grow one byte at a time; every prefix must equal a one-shot batch
// highlight of that same prefix (incremental == batch, byte-for-byte).
for end in 1..=full.len() {
if !full.is_char_boundary(end) {
continue;
}
let got = cache.highlight(syn, "yaml", 0, &full[..end]).expect("hl");
assert_eq!(got, batch(syn, "yaml", &full[..end]), "prefix len {end}");
}
}
#[test]
fn fence_change_invalidates() {
let syn = test_syntect();
let text = "let x = 1;\nfn main() {}\n";
let mut cache = OpenCodeHighlighter::new(syn);
// Prime with yaml, then re-key to rust: output must match a fresh rust
// batch, proving the persisted yaml state was discarded.
let _ = cache.highlight(syn, "yaml", 0, text).expect("hl yaml");
let got = cache.highlight(syn, "rust", 0, text).expect("hl rust");
assert_eq!(got, batch(syn, "rust", text));
}
#[test]
fn start_offset_change_invalidates() {
let syn = test_syntect();
let mut cache = OpenCodeHighlighter::new(syn);
let a = "alpha: 1\nbeta: 2\n";
let _ = cache.highlight(syn, "yaml", 0, a).expect("hl a");
// Same language, different block position and body: the new body must
// be highlighted fresh (no stale committed lines from the old block).
let b = "gamma: 3\ndelta: 4\n";
let got = cache.highlight(syn, "yaml", 42, b).expect("hl b");
assert_eq!(got, batch(syn, "yaml", b));
}
#[test]
fn unknown_fence_returns_none() {
let syn = test_syntect();
let mut cache = OpenCodeHighlighter::new(syn);
assert!(
cache
.highlight(syn, "definitely-not-a-language-xyz", 0, "data\n")
.is_none(),
);
}
// ── highlight_closed (closed-fence memo) ─────────────────────────
#[test]
fn closed_memo_matches_batch_and_is_idempotent() {
let syn = test_syntect();
let mut cache = OpenCodeHighlighter::new(syn);
let body = "fn answer(x: u64) -> u64 {\n x.wrapping_mul(42)\n}\n";
// First call computes; must equal the batch reference exactly.
let first = cache.highlight_closed(syn, "rust", body).expect("hl");
assert_eq!(first, batch(syn, "rust", body));
assert_eq!(cache.closed_memo_bytes(), body.len());
// Second call is a memo hit: identical output, no new entry.
let second = cache.highlight_closed(syn, "rust", body).expect("hl");
assert_eq!(second, first);
assert_eq!(cache.closed_memo_bytes(), body.len());
}
#[test]
fn closed_memo_distinguishes_fence_info_and_body() {
let syn = test_syntect();
let mut cache = OpenCodeHighlighter::new(syn);
let body_a = "key: value\n";
let body_b = "other: thing\n";
let yaml_a = cache.highlight_closed(syn, "yaml", body_a).expect("hl");
let yaml_b = cache.highlight_closed(syn, "yaml", body_b).expect("hl");
let rust_a = cache.highlight_closed(syn, "rust", body_a).expect("hl");
assert_eq!(yaml_a, batch(syn, "yaml", body_a));
assert_eq!(yaml_b, batch(syn, "yaml", body_b));
assert_eq!(rust_a, batch(syn, "rust", body_a));
}
#[test]
fn closed_memo_does_not_disturb_open_block_state() {
let syn = test_syntect();
let mut cache = OpenCodeHighlighter::new(syn);
// Interleave closed-memo calls with open-block incremental growth
// (a tail with one closed fence above an open one); open-block
// output must stay batch-identical throughout.
let closed = "name: pinned\n";
let full = "a = 1\nb = 2\nc = 3\n";
for end in 1..=full.len() {
if !full.is_char_boundary(end) {
continue;
}
let _ = cache.highlight_closed(syn, "yaml", closed).expect("memo");
let got = cache.highlight(syn, "python", 7, &full[..end]).expect("hl");
assert_eq!(got, batch(syn, "python", &full[..end]), "prefix len {end}");
}
assert_eq!(cache.closed_memo_bytes(), closed.len());
}
#[test]
fn closed_memo_cap_overflow_keeps_output_correct() {
let syn = test_syntect();
let mut cache = OpenCodeHighlighter::new(syn);
// Bodies sized so a handful of distinct ones cross the byte budget
// and trigger the wholesale clear; output must stay batch-identical
// before, at, and after the eviction.
let filler = "x".repeat(CLOSED_MEMO_CAP_BYTES / 4);
for i in 0..6 {
let body = format!("key_{i}: \"{filler}\"\n");
let got = cache.highlight_closed(syn, "yaml", &body).expect("hl");
assert_eq!(got, batch(syn, "yaml", &body), "iteration {i}");
}
assert!(cache.closed_memo_bytes() <= CLOSED_MEMO_CAP_BYTES);
}
}

602
crates/codegen/xai-grok-markdown/src/output.rs Normal file
View file

@ -0,0 +1,602 @@
//! Render output types for markdown.
use std::ops::Range;
use ratatui::text::Line;
use crate::buffers::CodeBlockMeta;
/// A hyperlink target extracted from rendered markdown.
///
/// Each instance maps a contiguous cell range on one rendered line to a URL.
/// When a link wraps across lines, multiple `HyperlinkTarget`s share the same
/// `id` and `url` -- the `id` enables OSC 8 hover-grouping across wrapped lines.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct HyperlinkTarget {
/// Index of the rendered line this target appears on.
pub line_index: usize,
/// Column range (in display cells) of the link text on that line.
pub column_range: Range<usize>,
/// The destination URL.
pub url: String,
/// Stable identifier for grouping link fragments that belong to the
/// same logical link (e.g., a link whose text wraps across lines).
pub id: u32,
}
/// A fenced code block discovered while rendering markdown.
///
/// One `CodeBlockSpan` is produced per **closed** fenced code block, in
/// document order. An unterminated (still-open) fence at the end of the input
/// produces no span: `pulldown-cmark` synthesizes a block end at end-of-input,
/// so closure is detected structurally (a closing fence must follow the body)
/// rather than from the end event alone.
///
/// This is a generic, reusable description of a fenced block — it is not
/// specific to any one info string (e.g. `mermaid`).
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CodeBlockSpan {
/// The fence info string, e.g. `"mermaid"` or `"rust"`.
///
/// Empty for a fence opened with no info (just ` ``` `). Reported verbatim
/// as `pulldown-cmark` yields it (the full info string, not just the first
/// word).
pub info: String,
/// The fence body content — the clean, container-stripped code/diagram
/// source.
///
/// This is `pulldown-cmark`'s merged body text, so container markers are
/// removed (a blockquote `>` / list indentation does **not** leak in) and
/// CRLF line endings are normalized to `\n`. It ends with the body's
/// trailing newline and is empty for an empty-body fence. Prefer this over
/// slicing [`source_byte_range`](Self::source_byte_range) when you need the
/// logical body (e.g. a Mermaid diagram nested in a blockquote).
pub body: String,
/// Range of **pre-wrap** rendered body lines for this block, as indices
/// into [`MarkdownRenderOutput::lines`] / [`MarkdownRenderView::lines`].
///
/// Covers only the body — the delimiter ` ``` ` lines are excluded — so it
/// is independent of whether the renderer hides those delimiters in pretty
/// mode. Empty (`start == end`) for a fence with an empty body.
pub output_line_range: Range<usize>,
/// Byte range of the fence body in the **raw** source text.
///
/// Spans from the first body byte to the last, with the delimiter fence
/// lines excluded; empty (`start == end`) for an empty body. Unlike
/// [`body`](Self::body) this is a raw slice, so for a fence nested in a
/// blockquote or list it covers the source between the delimiters and may
/// include container markers/indentation (and `\r` for CRLF) on
/// continuation lines. Use [`body`](Self::body) for the clean content.
pub source_byte_range: Range<usize>,
}
/// Output from rendering markdown to ratatui Lines.
///
/// Contains all the information needed to display rendered markdown and
/// support copy operations back to source text.
#[derive(Debug, Clone, Default)]
pub struct MarkdownRenderOutput {
/// Rendered lines ready for display.
pub lines: Vec<Line<'static>>,
/// Maps each rendered line index to its source line number.
/// `line_source_map[rendered_line_idx]` = source line number (0-indexed).
pub line_source_map: Vec<usize>,
/// Maps a cell range on a rendered line to a URL. Links that
/// wrap across lines produce multiple entries with the same `id` and `url`.
pub hyperlinks: Vec<HyperlinkTarget>,
/// Fenced code blocks discovered during rendering, in document order.
/// One entry per closed fenced block; see [`CodeBlockSpan`].
pub code_blocks: Vec<CodeBlockSpan>,
}
impl MarkdownRenderOutput {
/// Create a new empty output.
pub fn new() -> Self {
Self::default()
}
/// Clear all content, keeping allocated capacity.
pub fn clear(&mut self) {
self.lines.clear();
self.line_source_map.clear();
self.hyperlinks.clear();
self.code_blocks.clear();
}
/// Get a borrowed view of this output.
pub fn as_view(&self) -> MarkdownRenderView<'_> {
MarkdownRenderView {
lines: &self.lines,
line_source_map: &self.line_source_map,
hyperlinks: &self.hyperlinks,
code_blocks: &self.code_blocks,
}
}
}
/// Borrowed view of rendered markdown output.
///
/// This is a zero-copy reference to rendered content, used by the streaming
/// renderer to avoid cloning frozen content on every render.
#[derive(Debug, Clone, Copy)]
pub struct MarkdownRenderView<'a> {
/// Rendered lines ready for display.
pub lines: &'a [Line<'static>],
/// Maps each rendered line index to its source line number.
pub line_source_map: &'a [usize],
/// Hyperlink targets extracted from the rendered markdown.
pub hyperlinks: &'a [HyperlinkTarget],
/// Fenced code blocks discovered during rendering, in document order.
/// One entry per closed fenced block; see [`CodeBlockSpan`].
pub code_blocks: &'a [CodeBlockSpan],
}
impl<'a> MarkdownRenderView<'a> {
/// Get the number of lines.
pub fn line_count(&self) -> usize {
self.lines.len()
}
}
/// Map parse-time code-block metadata onto the rendered output.
///
/// Runs after `render_ratatui` has produced `line_source_map`, turning each
/// captured [`CodeBlockMeta`] into a public [`CodeBlockSpan`]. The
/// pre-wrap body line range is derived from `line_source_map`: a fence body
/// occupies source lines `[src_first, src_last]`, and because the renderer
/// emits exactly one output line per body source line (and never maps a
/// non-body line into that source-line range), the matching output lines form
/// one contiguous run. `line_source_map` is non-decreasing, so the run is
/// located with two `partition_point`s.
///
/// Cost is O(text_len + lines·log) per render: the metas are in ascending body
/// order, so newline counts come from a single monotonic forward cursor over
/// `text` rather than rescanning from byte 0 for every meta (which would be
/// O(metas·text_len) — quadratic in the number of fences on the streaming hot
/// path).
pub(crate) fn build_code_block_spans(
text: &str,
line_source_map: &[usize],
metas: Vec<CodeBlockMeta>,
) -> Vec<CodeBlockSpan> {
if metas.is_empty() {
return Vec::new();
}
let bytes = text.as_bytes();
// Monotonic newline cursor. Each query advances from the previous position
// (metas ascend by body offset), so the whole pass is O(text_len). '\n' is
// single-byte ASCII, so byte counting is UTF-8-safe at any offset.
let mut cursor_pos = 0usize;
let mut cursor_newlines = 0usize;
let mut newlines_before = |pos: usize| -> usize {
let pos = pos.min(bytes.len());
debug_assert!(
pos >= cursor_pos,
"metas must be processed in ascending body order",
);
while cursor_pos < pos {
if bytes[cursor_pos] == b'\n' {
cursor_newlines += 1;
}
cursor_pos += 1;
}
cursor_newlines
};
metas
.into_iter()
.map(|meta| {
let range = meta.body_source_range;
let src_first = newlines_before(range.start);
let output_line_range = if range.end <= range.start {
// Empty body: no rendered body lines. Anchor an empty range at
// the first output line that does not precede the body.
let start = line_source_map.partition_point(|&src| src < src_first);
start..start
} else {
// `range.end - 1` is the last body byte; its source line is the
// inclusive last body source line, robust to a trailing newline.
let src_last = newlines_before(range.end - 1);
let start = line_source_map.partition_point(|&src| src < src_first);
let end = line_source_map.partition_point(|&src| src <= src_last);
start..end
};
CodeBlockSpan {
info: meta.info,
body: meta.body,
output_line_range,
source_byte_range: range,
}
})
.collect()
}
#[cfg(test)]
mod code_block_span_tests {
use ratatui::text::Line;
use crate::style::test_style::STYLE;
use crate::{CodeBlockSpan, StreamingMarkdownRenderer, render_markdown_ratatui_full};
fn lines_text(lines: &[Line<'static>]) -> Vec<String> {
lines
.iter()
.map(|l| l.spans.iter().map(|s| s.content.as_ref()).collect())
.collect()
}
/// Plain text of the rendered body lines a span points at.
fn body_lines(lines: &[Line<'static>], span: &CodeBlockSpan) -> Vec<String> {
lines_text(&lines[span.output_line_range.clone()])
}
/// Source bytes a span's `source_byte_range` selects.
fn body_source<'a>(src: &'a str, span: &CodeBlockSpan) -> &'a str {
&src[span.source_byte_range.clone()]
}
fn blocks(src: &str, pretty: bool) -> (Vec<Line<'static>>, Vec<CodeBlockSpan>) {
let (out, _) = render_markdown_ratatui_full(src, STYLE, pretty, None);
(out.lines, out.code_blocks)
}
#[test]
fn closed_fence_top_level_pretty_and_raw() {
// A non-rendered language (`text`): its rendered body lines are the
// verbatim source, so the span's `output_line_range` maps back to them.
// (A `mermaid` fence renders to diagram art instead — see
// `mermaid_fence_renders_inline_but_span_keeps_clean_source`.)
let src = "```text\nflowchart TD\n A --> B\n```\n";
for pretty in [true, false] {
let (lines, cbs) = blocks(src, pretty);
assert_eq!(cbs.len(), 1, "pretty={pretty}");
assert_eq!(cbs[0].info, "text");
// Body line range excludes the delimiter fences in both modes.
assert_eq!(
body_lines(&lines, &cbs[0]),
vec!["flowchart TD", " A --> B"],
"pretty={pretty}",
);
// Byte range and clean body are mode-independent; for a top-level
// fence both equal the verbatim fence body.
assert_eq!(body_source(src, &cbs[0]), "flowchart TD\n A --> B\n");
assert_eq!(cbs[0].body, "flowchart TD\n A --> B\n");
}
}
#[test]
fn open_fence_produces_no_span() {
// Unterminated fence at EOF: pulldown still emits a block end, but no
// span must be produced (malformed/partial input).
for src in [
"```mermaid\nflowchart TD\n",
"```mermaid\nflowchart TD",
"```mermaid",
"intro\n\n```rust\nlet x = 1;",
] {
for pretty in [true, false] {
let (_, cbs) = blocks(src, pretty);
assert!(
cbs.is_empty(),
"open fence {src:?} (pretty={pretty}) should yield no span, got {cbs:?}",
);
}
}
}
#[test]
fn multiple_interspersed_blocks_in_order() {
let src = "Intro\n\n```rust\nfn a() {}\n```\n\nMid prose\n\n```text\nA-->B\n```\n\nEnd\n";
for pretty in [true, false] {
let (lines, cbs) = blocks(src, pretty);
assert_eq!(cbs.len(), 2, "pretty={pretty}");
assert_eq!(cbs[0].info, "rust");
assert_eq!(cbs[1].info, "text");
assert_eq!(body_source(src, &cbs[0]), "fn a() {}\n");
assert_eq!(body_source(src, &cbs[1]), "A-->B\n");
assert_eq!(body_lines(&lines, &cbs[0]), vec!["fn a() {}"]);
assert_eq!(body_lines(&lines, &cbs[1]), vec!["A-->B"]);
// Document order ⇒ disjoint, increasing line ranges.
assert!(cbs[0].output_line_range.end <= cbs[1].output_line_range.start);
}
}
#[test]
fn fence_nested_in_list() {
// Multi-line body so the list's base indent stripping is exercised on a
// continuation line (" A --> B" → " A --> B").
let src = "- item\n ```mermaid\n flowchart TD\n A --> B\n ```\n- next\n";
for pretty in [true, false] {
let (_lines, cbs) = blocks(src, pretty);
assert_eq!(cbs.len(), 1, "pretty={pretty}");
assert_eq!(cbs[0].info, "mermaid");
// `body` is the clean, de-prefixed source: the list base indent is
// stripped but inner relative indentation is preserved.
assert_eq!(cbs[0].body, "flowchart TD\n A --> B\n", "pretty={pretty}");
// The raw byte range, by contrast, also strips the per-line base
// indent here (pulldown's text-event range starts after it).
assert_eq!(
body_source(src, &cbs[0]),
"flowchart TD\n A --> B\n",
"pretty={pretty}",
);
assert!(!cbs[0].output_line_range.is_empty());
}
}
#[test]
fn fence_nested_in_blockquote() {
// The motivating case for the structural closure rule: the closing
// fence line is "> ```" (not a bare fence), and the body must come out
// de-prefixed (no leaked "> " / "│ ").
let src = "> ```mermaid\n> flowchart TD\n> A --> B\n> ```\n";
for pretty in [true, false] {
let (_lines, cbs) = blocks(src, pretty);
assert_eq!(cbs.len(), 1, "pretty={pretty}");
assert_eq!(cbs[0].info, "mermaid");
assert_eq!(cbs[0].body, "flowchart TD\n A --> B\n", "pretty={pretty}");
}
}
#[test]
fn open_blockquote_fence_produces_no_span() {
// Unterminated fence inside a blockquote ⇒ no span.
let src = "> ```mermaid\n> flowchart TD\n";
for pretty in [true, false] {
let (_, cbs) = blocks(src, pretty);
assert!(cbs.is_empty(), "pretty={pretty} got {cbs:?}");
}
}
#[test]
fn indented_code_block_is_not_a_fence() {
// A 4-space indented code block is not a fenced block ⇒ no span, even
// when its literal content looks like a fence.
let src = "para\n\n ```mermaid\n A-->B\n ```\n";
for pretty in [true, false] {
let (_, cbs) = blocks(src, pretty);
assert!(cbs.is_empty(), "pretty={pretty} got {cbs:?}");
}
}
#[test]
fn empty_body_closed_fence() {
let src = "```mermaid\n```\n";
// Pretty: both fence lines are hidden ⇒ no output lines ⇒ the empty
// anchor lands at 0..0 (exact, not merely is_empty()).
let (_, cbs) = blocks(src, true);
assert_eq!(cbs.len(), 1);
assert_eq!(cbs[0].info, "mermaid");
assert_eq!(cbs[0].output_line_range, 0..0);
assert_eq!(body_source(src, &cbs[0]), "");
assert_eq!(cbs[0].body, "");
// Raw: both fence lines are shown ⇒ the empty body is anchored between
// them at 1..1.
let (_, cbs_raw) = blocks(src, false);
assert_eq!(cbs_raw.len(), 1);
assert_eq!(cbs_raw[0].output_line_range, 1..1);
assert_eq!(cbs_raw[0].body, "");
}
#[test]
fn fence_without_info_string() {
let src = "```\nplain code\n```\n";
let (lines, cbs) = blocks(src, true);
assert_eq!(cbs.len(), 1);
assert_eq!(cbs[0].info, "");
assert_eq!(body_lines(&lines, &cbs[0]), vec!["plain code"]);
}
#[test]
fn tilde_fence_and_no_trailing_newline() {
// Tilde delimiters and a closed fence at EOF without a final newline.
for src in ["~~~text\nA-->B\n~~~\n", "```text\nfoo\n```"] {
let (lines, cbs) = blocks(src, true);
assert_eq!(cbs.len(), 1, "{src:?}");
assert_eq!(cbs[0].info, "text");
assert_eq!(body_lines(&lines, &cbs[0]).len(), 1);
}
}
#[test]
fn blank_line_inside_body_is_counted() {
let src = "```text\nfoo\n\nbar\n```\n";
let (lines, cbs) = blocks(src, true);
assert_eq!(cbs.len(), 1);
assert_eq!(body_lines(&lines, &cbs[0]), vec!["foo", "", "bar"]);
assert_eq!(body_source(src, &cbs[0]), "foo\n\nbar\n");
assert_eq!(cbs[0].body, "foo\n\nbar\n");
}
#[test]
fn crlf_body_is_normalized_but_byte_range_retains_cr() {
// CRLF: pulldown normalizes the body content to `\n`, while the raw
// byte range still slices the `\r`. Line counting (over `\n`) is
// unaffected.
let src = "```text\r\nA-->B\r\n```\r\n";
let (lines, cbs) = blocks(src, true);
assert_eq!(cbs.len(), 1);
assert_eq!(cbs[0].body, "A-->B\n");
assert_eq!(body_source(src, &cbs[0]), "A-->B\r\n");
// One rendered body line (the renderer keeps the raw `\r`; `body` is the
// normalized source of truth).
assert_eq!(body_lines(&lines, &cbs[0]).len(), 1);
}
#[test]
fn multibyte_body_slices_safely() {
// Multi-byte UTF-8 in the body: `source_byte_range` must land on char
// boundaries (no slice panic) and `body` is the exact content.
let src = "```text\nA --> \u{65e5}\u{672c}\u{8a9e}\nC --> \u{1f980}\n```\n";
let (lines, cbs) = blocks(src, true);
assert_eq!(cbs.len(), 1);
assert_eq!(
cbs[0].body,
"A --> \u{65e5}\u{672c}\u{8a9e}\nC --> \u{1f980}\n"
);
assert_eq!(
body_source(src, &cbs[0]),
"A --> \u{65e5}\u{672c}\u{8a9e}\nC --> \u{1f980}\n",
);
assert_eq!(body_lines(&lines, &cbs[0]).len(), 2);
}
#[test]
fn long_fence_with_inner_backticks() {
// A 4-backtick fence whose body contains a ``` line must not close
// early — one span whose body includes the inner fence text.
let src = "````mermaid\n```\ninner\n```\n````\n";
let (_lines, cbs) = blocks(src, true);
assert_eq!(cbs.len(), 1);
assert_eq!(cbs[0].info, "mermaid");
assert_eq!(cbs[0].body, "```\ninner\n```\n");
}
#[test]
fn tab_in_body_is_preserved_by_crate() {
// The markdown crate preserves a literal tab in the body (the pager
// expands tabs before rendering; the crate itself does not).
let src = "```mermaid\n\tA --> B\n```\n";
let (_lines, cbs) = blocks(src, true);
assert_eq!(cbs.len(), 1);
assert_eq!(cbs[0].body, "\tA --> B\n");
}
#[test]
fn rendered_body_lines_match_span_body() {
// The rendered (pre-wrap) body lines reconstruct to the span body
// (sans the trailing newline a line-join drops). Both pretty and raw.
let src = "```text\nflowchart TD\n A --> B\n B --> C\n```\n";
for pretty in [true, false] {
let (lines, cbs) = blocks(src, pretty);
let joined = body_lines(&lines, &cbs[0]).join("\n");
assert_eq!(
joined, "flowchart TD\n A --> B\n B --> C",
"pretty={pretty}"
);
assert_eq!(format!("{joined}\n"), cbs[0].body, "pretty={pretty}");
}
}
#[test]
fn mermaid_fence_renders_inline_but_span_keeps_clean_source() {
// A closed ```mermaid fence is rendered inline by the markdown crate
// (its body lines are replaced with diagram art), yet its CodeBlockSpan
// still exposes the clean SOURCE via `body` — what the pager feeds the
// PNG engine — and an `output_line_range` that spans the rendered
// diagram, where the pager anchors its affordance row. This contract is
// what the pager's Mermaid affordance row relies on.
let src = "```mermaid\nflowchart TD\n A --> B\n```\n";
let (lines, cbs) = blocks(src, true);
assert_eq!(cbs.len(), 1);
assert_eq!(cbs[0].info, "mermaid");
// `body` is the verbatim diagram source, independent of rendering.
assert_eq!(cbs[0].body, "flowchart TD\n A --> B\n");
// The fence is rendered inline: the spanned output lines are the diagram
// art, not the verbatim source.
assert!(!cbs[0].output_line_range.is_empty());
let rendered = body_lines(&lines, &cbs[0]).join("\n");
assert_ne!(rendered, "flowchart TD\n A --> B");
}
#[test]
fn streaming_span_stable_once_frozen() {
let full = "Intro\n\n```mermaid\nA-->B\nC-->D\n```\n\nAfter the block.\n\nMore prose.\n";
let mut renderer = StreamingMarkdownRenderer::new(STYLE, true);
let mut frozen_span: Option<CodeBlockSpan> = None;
for ch in full.chars() {
renderer.push_and_render(&ch.to_string(), None);
let view = renderer.view();
let frozen_lines = renderer.frozen_lines_count();
if let Some(cb) = view.code_blocks.iter().find(|c| c.info == "mermaid") {
// Only assert stability once the block is within frozen content.
if cb.output_line_range.end <= frozen_lines {
match &frozen_span {
None => frozen_span = Some(cb.clone()),
Some(prev) => {
assert_eq!(prev, cb, "frozen mermaid span changed across pushes",)
}
}
}
}
}
let frozen_span = frozen_span.expect("mermaid block should freeze mid-stream");
// Streaming output (after finish) must match a one-shot full render.
let finished = renderer.finish(None);
let streamed = finished
.code_blocks
.iter()
.find(|c| c.info == "mermaid")
.expect("finish() keeps the mermaid span");
assert_eq!(&frozen_span, streamed);
let (full_out, _) = render_markdown_ratatui_full(full, STYLE, true, None);
let full_cb = full_out
.code_blocks
.iter()
.find(|c| c.info == "mermaid")
.expect("full render finds the mermaid span");
assert_eq!(&frozen_span, full_cb);
}
#[test]
fn streaming_open_fence_has_no_frozen_span() {
// While the fence is still open, any transient span must remain in the
// unfrozen tail (never within the frozen prefix).
let mut renderer = StreamingMarkdownRenderer::new(STYLE, true);
for chunk in ["intro\n\n", "```mermaid\n", "flowchart TD\n", "A --> B\n"] {
renderer.push_and_render(chunk, None);
let frozen_lines = renderer.frozen_lines_count();
let view = renderer.view();
for cb in view.code_blocks {
assert!(
cb.output_line_range.end > frozen_lines,
"open fence span must not be frozen: {cb:?} frozen_lines={frozen_lines}",
);
}
}
}
#[test]
fn streaming_chunked_matches_full_render() {
let full = "# Title\n\n```mermaid\nA-->B\n```\n\ntext\n\n```rust\nfn f() {}\n```\n\nbye\n";
for pretty in [true, false] {
let mut renderer = StreamingMarkdownRenderer::new(STYLE, pretty);
// Irregular chunk boundaries to exercise tail rebasing.
let bytes = full.as_bytes();
let mut pos = 0;
for step in [4usize, 9, 1, 13, 7].iter().cycle() {
if pos >= bytes.len() {
break;
}
let mut end = (pos + step).min(bytes.len());
while end < bytes.len() && !full.is_char_boundary(end) {
end += 1;
}
renderer.push_and_render(&full[pos..end], None);
pos = end;
}
let view = renderer.finish(None);
let streamed: Vec<_> = view.code_blocks.to_vec();
let (full_out, _) = render_markdown_ratatui_full(full, STYLE, pretty, None);
assert_eq!(
streamed, full_out.code_blocks,
"pretty={pretty}: chunked stream must match full render",
);
}
}
}

2057
crates/codegen/xai-grok-markdown/src/parse.rs Normal file
View file

File diff suppressed because it is too large Load diff

2753
crates/codegen/xai-grok-markdown/src/render.rs Normal file
View file

File diff suppressed because it is too large Load diff

141
crates/codegen/xai-grok-markdown/src/source_map.rs Normal file
View file

@ -0,0 +1,141 @@
//! Source mapping for rendered markdown back to original source.
//!
//! Used for copy-paste operations: when the user selects rendered text,
//! we can look up the corresponding original markdown source.
use std::ops::Range;
/// Maps rendered byte positions back to source byte positions.
///
/// Direction: rendered (new) → source (old)
#[derive(Debug, Clone, Default)]
pub struct SourceMap {
/// Segments: (rendered_range, source_range)
segments: Vec<(Range<usize>, Range<usize>)>,
}
impl SourceMap {
/// Create an empty source map.
pub fn new() -> Self {
Self::default()
}
/// Add a mapping from rendered position to source position.
///
/// The rendered text and source text must have the same length.
pub fn add(&mut self, rendered_start: usize, source_range: Range<usize>) {
let len = source_range.end - source_range.start;
if len > 0 {
self.segments
.push((rendered_start..rendered_start + len, source_range));
}
}
/// Given a rendered byte range, return the corresponding source range.
///
/// Returns None if the range doesn't map cleanly (e.g., spans multiple
/// non-contiguous source regions).
pub fn to_source(&self, rendered: Range<usize>) -> Option<Range<usize>> {
let mut source_start = None;
let mut source_end = None;
for (r_range, s_range) in &self.segments {
if r_range.end <= rendered.start || r_range.start >= rendered.end {
continue;
}
let overlap_start = rendered.start.max(r_range.start);
let overlap_end = rendered.end.min(r_range.end);
let offset_start = overlap_start - r_range.start;
let offset_end = overlap_end - r_range.start;
let s_start = s_range.start + offset_start;
let s_end = s_range.start + offset_end;
source_start = Some(source_start.map_or(s_start, |v: usize| v.min(s_start)));
source_end = Some(source_end.map_or(s_end, |v: usize| v.max(s_end)));
}
match (source_start, source_end) {
(Some(s), Some(e)) => Some(s..e),
_ => None,
}
}
/// Check if the source map is empty.
pub fn is_empty(&self) -> bool {
self.segments.is_empty()
}
/// Get the number of segments.
pub fn len(&self) -> usize {
self.segments.len()
}
/// Extend with entries from another source map, applying offsets.
///
/// Used when combining frozen content with newly rendered tail content.
pub fn extend_with_offsets(
&mut self,
other: &Self,
rendered_offset: usize,
source_offset: usize,
) {
for (r_range, s_range) in &other.segments {
self.segments.push((
(r_range.start + rendered_offset)..(r_range.end + rendered_offset),
(s_range.start + source_offset)..(s_range.end + source_offset),
));
}
}
/// Get read-only access to segments for inspection.
pub fn segments(&self) -> &[(Range<usize>, Range<usize>)] {
&self.segments
}
/// Truncate to keep only the first `n` segments.
pub fn truncate(&mut self, n: usize) {
self.segments.truncate(n);
}
/// Clear all segments.
pub fn clear(&mut self) {
self.segments.clear();
}
}
// ## Restoring Byte-Level Source Maps (if ever needed)
//
// The ratatui rendering path currently only tracks line-level source mapping
// (`line_source_map`), which is sufficient for copy/selection operations.
// Byte-level `SourceMap` was removed for simplicity and ~6% speedup.
//
// To restore byte-level source maps:
//
// 1. Add field to MarkdownRenderOutput and MarkdownRenderView:
// ```
// pub source_map: SourceMap,
// ```
//
// 2. In render_ratatui(), add tracking variables:
// ```
// let mut source_map = SourceMap::new();
// let mut rendered_offset: usize = 0;
// ```
//
// 3. For each text segment emitted, record the mapping:
// ```
// source_map.add(rendered_offset, source_start..source_end);
// rendered_offset += emitted_text.len();
// ```
//
// 4. In streaming.rs, update FrozenState to track:
// ```
// source_map_len: usize,
// rendered_bytes: usize,
// ```
//
// 5. Use SourceMap::extend_with_offsets() to merge tail source maps.
//
// See git history for the removed implementation.

2910
crates/codegen/xai-grok-markdown/src/streaming.rs Normal file
View file

File diff suppressed because it is too large Load diff

303
crates/codegen/xai-grok-markdown/src/style.rs Normal file
View file

@ -0,0 +1,303 @@
//! Markdown styling types.
//!
//! This module provides the `MarkdownStyle` struct which defines colors and
//! effects for all markdown elements.
use anstyle::{Effects, Style};
use crate::colors::adapt_style;
/// Table border characters for rendering tables in pretty mode.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct TableBorders {
chars: [char; 11],
}
impl TableBorders {
const H: usize = 0;
const V: usize = 1;
const TL: usize = 2;
const TR: usize = 3;
const BL: usize = 4;
const BR: usize = 5;
const T_T: usize = 6;
const T_B: usize = 7;
const T_L: usize = 8;
const T_R: usize = 9;
const X: usize = 10;
pub const BOX: Self = Self {
chars: ['─', '│', '┌', '┐', '└', '┘', '┬', '┴', '├', '┤', '┼'],
};
pub const ASCII: Self = Self {
chars: ['-', '|', '+', '+', '+', '+', '+', '+', '+', '+', '+'],
};
pub const DOUBLE: Self = Self {
chars: ['═', '║', '╔', '╗', '╚', '╝', '╦', '╩', '╠', '╣', '╬'],
};
pub const fn new(chars: [char; 11]) -> Self {
Self { chars }
}
// Short names (used in table formatting)
pub const fn h(&self) -> char {
self.chars[Self::H]
}
pub const fn v(&self) -> char {
self.chars[Self::V]
}
pub const fn c_tl(&self) -> char {
self.chars[Self::TL]
}
pub const fn c_tr(&self) -> char {
self.chars[Self::TR]
}
pub const fn c_bl(&self) -> char {
self.chars[Self::BL]
}
pub const fn c_br(&self) -> char {
self.chars[Self::BR]
}
pub const fn t_t(&self) -> char {
self.chars[Self::T_T]
}
pub const fn t_b(&self) -> char {
self.chars[Self::T_B]
}
pub const fn t_l(&self) -> char {
self.chars[Self::T_L]
}
pub const fn t_r(&self) -> char {
self.chars[Self::T_R]
}
pub const fn x(&self) -> char {
self.chars[Self::X]
}
// Long names (for readability)
pub const fn horizontal(&self) -> char {
self.chars[Self::H]
}
pub const fn vertical(&self) -> char {
self.chars[Self::V]
}
pub const fn top_left(&self) -> char {
self.chars[Self::TL]
}
pub const fn top_right(&self) -> char {
self.chars[Self::TR]
}
pub const fn bottom_left(&self) -> char {
self.chars[Self::BL]
}
pub const fn bottom_right(&self) -> char {
self.chars[Self::BR]
}
pub const fn t_top(&self) -> char {
self.chars[Self::T_T]
}
pub const fn t_bottom(&self) -> char {
self.chars[Self::T_B]
}
pub const fn t_left(&self) -> char {
self.chars[Self::T_L]
}
pub const fn t_right(&self) -> char {
self.chars[Self::T_R]
}
pub const fn cross(&self) -> char {
self.chars[Self::X]
}
}
impl Default for TableBorders {
fn default() -> Self {
Self::BOX
}
}
/// Style configuration for markdown rendering.
///
/// Each field controls the styling for a specific markdown element.
/// The `_inner` variants are applied to the content, while `_outer` variants
/// are applied to the syntax markers (which are hidden in pretty mode).
#[derive(Copy, Clone, Default, Debug, PartialEq, Eq, Hash)]
pub struct MarkdownStyle {
pub heading_inner: [Style; 6],
pub heading_outer: [Style; 6],
pub strong_inner: Style,
pub strong_outer: Style,
pub emphasis_inner: Style,
pub emphasis_outer: Style,
pub strikethrough_inner: Style,
pub strikethrough_outer: Style,
pub inline_code_inner: Style,
pub inline_code_outer: Style,
pub blockquote_outer: Style,
pub task_checked: Style,
pub task_unchecked: Style,
pub list_item: Style,
pub rule: Style,
pub link_outer: Style,
pub link_text: Style,
pub link_url: Style,
pub link_title: Style,
pub code_outer: Style,
pub code_language: Style,
pub code_untagged: Style,
pub code_background: Style,
pub table_outer: Style,
/// Default foreground for plain body text (paragraphs with no formatting).
/// When set, the renderer applies this to text spans that would otherwise
/// inherit the terminal's default foreground.
pub text: Style,
/// Style for rendered LaTeX math (inline `$...$`/`\(...\)` content after
/// Unicode conversion, and display math block lines). In raw mode the
/// style applies to the unconverted TeX source.
pub math: Style,
}
impl MarkdownStyle {
/// Adapt all styles for the terminal's color capabilities.
///
/// This downgrades RGB colors to 256-color or 16-color as needed.
pub fn adapt(self) -> Self {
Self {
heading_inner: [
adapt_style(self.heading_inner[0]),
adapt_style(self.heading_inner[1]),
adapt_style(self.heading_inner[2]),
adapt_style(self.heading_inner[3]),
adapt_style(self.heading_inner[4]),
adapt_style(self.heading_inner[5]),
],
heading_outer: [
adapt_style(self.heading_outer[0]),
adapt_style(self.heading_outer[1]),
adapt_style(self.heading_outer[2]),
adapt_style(self.heading_outer[3]),
adapt_style(self.heading_outer[4]),
adapt_style(self.heading_outer[5]),
],
strong_inner: adapt_style(self.strong_inner),
strong_outer: adapt_style(self.strong_outer),
emphasis_inner: adapt_style(self.emphasis_inner),
emphasis_outer: adapt_style(self.emphasis_outer),
strikethrough_inner: adapt_style(self.strikethrough_inner),
strikethrough_outer: adapt_style(self.strikethrough_outer),
inline_code_inner: adapt_style(self.inline_code_inner),
inline_code_outer: adapt_style(self.inline_code_outer),
blockquote_outer: adapt_style(self.blockquote_outer),
task_checked: adapt_style(self.task_checked),
task_unchecked: adapt_style(self.task_unchecked),
list_item: adapt_style(self.list_item),
rule: adapt_style(self.rule),
link_outer: adapt_style(self.link_outer),
link_text: adapt_style(self.link_text),
link_url: adapt_style(self.link_url),
link_title: adapt_style(self.link_title),
code_outer: adapt_style(self.code_outer),
code_language: adapt_style(self.code_language),
code_untagged: adapt_style(self.code_untagged),
code_background: adapt_style(self.code_background),
table_outer: adapt_style(self.table_outer),
text: adapt_style(self.text),
math: adapt_style(self.math),
}
}
}
/// Check if ALL active styles have HIDDEN effect.
/// Used in pretty mode to determine if text should be skipped.
pub(crate) fn all_hidden(styles: impl IntoIterator<Item = Option<Style>>) -> bool {
let mut has_any = false;
let mut all_are_hidden = true;
for style in styles {
has_any = true;
match style {
Some(s) if s.get_effects().contains(Effects::HIDDEN) => {}
_ => {
all_are_hidden = false;
}
}
}
has_any && all_are_hidden
}
/// Merge multiple styles into one for rendering.
/// Strips HIDDEN from final output - it's a semantic marker, not a visual style.
pub(crate) fn merge_styles(styles: impl IntoIterator<Item = Option<Style>>) -> Style {
let mut out = Style::new();
let mut prev = Style::new();
for style in styles {
if out.get_effects().contains(Effects::HIDDEN) {
out = prev;
} else {
prev = out;
}
let Some(style) = style else {
continue;
};
if !style.get_effects().is_plain() {
out = out.effects(out.get_effects() | style.get_effects());
}
if style.get_effects().contains(Effects::DIMMED) {
out = out.effects(out.get_effects().remove(Effects::BOLD));
}
if style.get_effects().contains(Effects::BOLD) {
out = out.effects(out.get_effects().remove(Effects::DIMMED));
}
if let Some(color) = style.get_fg_color() {
out = out.fg_color(Some(color));
}
if let Some(color) = style.get_bg_color() {
out = out.bg_color(Some(color));
}
if let Some(color) = style.get_underline_color() {
out = out.underline_color(Some(color));
}
}
out.effects(out.get_effects().remove(Effects::HIDDEN))
}
// Simple default style for testing (no colors, just effects)
#[cfg(any(test, fuzzing))]
pub mod test_style {
use super::MarkdownStyle;
use anstyle::Style;
/// A minimal style for testing with no colors.
pub const STYLE: MarkdownStyle = MarkdownStyle {
heading_inner: [Style::new().bold(); 6],
heading_outer: [Style::new().dimmed().hidden(); 6],
strong_inner: Style::new().bold(),
strong_outer: Style::new().dimmed().hidden(),
emphasis_inner: Style::new().italic(),
emphasis_outer: Style::new().dimmed().hidden(),
strikethrough_inner: Style::new().strikethrough(),
strikethrough_outer: Style::new().dimmed().hidden(),
inline_code_inner: Style::new().bold(),
inline_code_outer: Style::new().dimmed().hidden(),
blockquote_outer: Style::new().dimmed(),
task_checked: Style::new(),
task_unchecked: Style::new().dimmed(),
list_item: Style::new().dimmed(),
rule: Style::new(),
link_outer: Style::new(),
link_text: Style::new().bold(),
link_url: Style::new().dimmed(),
link_title: Style::new(),
code_outer: Style::new().dimmed().hidden(),
code_language: Style::new().hidden(),
code_untagged: Style::new(),
code_background: Style::new(),
table_outer: Style::new().bold(),
text: Style::new(),
math: Style::new().italic(),
};
}

211
crates/codegen/xai-grok-markdown/src/syntax.rs Normal file
View file

@ -0,0 +1,211 @@
//! Syntax highlighting support using syntect.
//!
//! This module provides the `Syntect` struct which holds the syntax definitions
//! and theme for code block highlighting.
use std::io::Cursor;
use std::path::Path;
use syntect::{
easy::HighlightLines,
highlighting::{Theme as SyntectTheme, ThemeSet},
parsing::{SyntaxReference, SyntaxSet},
};
/// Syntax highlighting configuration.
///
/// Holds the theme and syntax definitions for code highlighting.
/// Create one instance and pass it to the markdown renderer.
pub struct Syntect {
/// The color theme for syntax highlighting.
pub theme: SyntectTheme,
/// The syntax definitions (supports 250+ languages via two-face).
pub syntax_set: SyntaxSet,
}
impl Syntect {
/// Create a new Syntect instance from theme bytes.
///
/// The theme bytes should be a TextMate `.tmTheme` file.
/// Uses two-face's extended syntax set with 250+ languages.
///
/// # Example
///
/// ```ignore
/// let syntect = Syntect::new(include_bytes!("assets/tokyo-night.tmTheme"));
/// ```
pub fn new(theme_bytes: &[u8]) -> Self {
let mut cursor = Cursor::new(theme_bytes);
let theme = ThemeSet::load_from_reader(&mut cursor).expect("Failed to load theme");
// Use two-face's extended syntax set which includes 250+ languages from bat
let syntax_set = two_face::syntax::extra_newlines();
Self { theme, syntax_set }
}
/// Find a syntax definition by file path extension.
pub fn find_syntax_by_file_path(&self, file_path: &Path) -> Option<&SyntaxReference> {
let ext = file_path.extension()?.to_str()?;
self.syntax_set.find_syntax_by_extension(ext)
}
/// Find a syntax definition by language token (e.g., "rust", "python").
pub fn find_syntax_by_token(&self, token: &str) -> Option<&SyntaxReference> {
self.syntax_set.find_syntax_by_token(token)
}
/// Create a highlighter for the given file path.
pub fn highlight_lines_by_file_path(&self, file_path: &Path) -> Option<HighlightLines<'_>> {
Some(HighlightLines::new(
self.find_syntax_by_file_path(file_path)?,
&self.theme,
))
}
/// Create a highlighter for the given language token.
pub fn highlight_lines_for_token(&self, token: &str) -> Option<HighlightLines<'_>> {
Some(HighlightLines::new(
self.find_syntax_by_token(token)?,
&self.theme,
))
}
/// Highlighter for a fenced code block *info* string: a normal language token
/// (e.g. `rust`, `python`), or a **line-range citation** of the form
/// `lineStart:lineEnd:path/to/file.ext` where the syntax is resolved the same
/// way as [`Syntect::highlight_lines_by_file_path`] (see
/// [`Syntect::find_syntax_by_file_path`]).
///
/// If the string matches the citation form but no syntax is found for the
/// path, this falls back to [`Syntect::find_syntax_by_token`] with the full
/// `fence_info` string, so plain ` ```lang` blocks keep working and odd
/// citations degrade like the pre-citation code path.
pub fn highlight_lines_for_fence_info(&self, fence_info: &str) -> Option<HighlightLines<'_>> {
Some(HighlightLines::new(
self.find_syntax_for_fence_info(fence_info)?,
&self.theme,
))
}
/// Resolve the [`SyntaxReference`] for a fenced code block *info* string,
/// using the SAME rules as [`Syntect::highlight_lines_for_fence_info`]:
/// a `lineStart:lineEnd:path` citation resolves by file path, otherwise
/// (or if the path has no known syntax) it falls back to a language token.
///
/// Exposed so the incremental open-code highlighter can build its own
/// resumable `ParseState`/`HighlightState` against exactly the syntax the
/// batch `HighlightLines` path would have used — keeping the two
/// byte-identical.
pub(crate) fn find_syntax_for_fence_info(&self, fence_info: &str) -> Option<&SyntaxReference> {
if let Some((_, _, path)) = parse_line_citation_fence_info(fence_info)
&& let Some(s) = self.find_syntax_by_file_path(Path::new(path))
{
return Some(s);
}
self.find_syntax_by_token(fence_info)
}
}
/// ```text
/// lineStart:lineEnd:path/to/file.ext
/// ```
///
/// The path is the segment after the **second** colon; it is then parsed with
/// [`Path::new`]. Paths with extra colons in the first two segments (e.g. some
/// Windows `C:...` forms) are not supported; use a repo-relative or
/// forward-slash form.
fn parse_line_citation_fence_info(info: &str) -> Option<(&str, &str, &str)> {
let mut it = info.splitn(3, ':');
let start = it.next()?;
let end = it.next()?;
let path = it.next()?;
if start.is_empty() || !start.chars().all(|c| c.is_ascii_digit()) {
return None;
}
if end.is_empty() || !end.chars().all(|c| c.is_ascii_digit()) {
return None;
}
if path.is_empty() {
return None;
}
Some((start, end, path))
}
/// Syntax highlight code, returning raw styled segments per line.
///
/// `fence_info` is the fenced code block *info* string (language tag or
/// `lineStart:lineEnd:path` citation form); see
/// [`Syntect::highlight_lines_for_fence_info`]. Lives here (not in `parse`)
/// so both the parser and the streaming highlighter caches depend one-way on
/// `syntax`.
pub(crate) fn syntax_highlight_raw(
syntect: Option<&Syntect>,
fence_info: &str,
text: &str,
) -> Option<Vec<Vec<(syntect::highlighting::Style, String)>>> {
use syntect::util::LinesWithEndings;
let syn = syntect?;
let mut hl = syn.highlight_lines_for_fence_info(fence_info)?;
let mut lines = Vec::new();
for line in LinesWithEndings::from(text) {
let highlighted = hl.highlight_line(line, &syn.syntax_set).ok()?;
lines.push(
highlighted
.into_iter()
.map(|(s, t)| (s, t.to_string()))
.collect(),
);
}
Some(lines)
}
/// Get a shared Syntect instance for tests.
///
/// This loads the tokyo-night theme bundled with the crate.
/// Uses a static OnceLock for efficiency in test runs.
#[cfg(any(test, fuzzing))]
#[allow(dead_code)]
pub fn test_syntect() -> &'static Syntect {
use std::sync::OnceLock;
static TEST_SYNTECT: OnceLock<Syntect> = OnceLock::new();
TEST_SYNTECT.get_or_init(|| Syntect::new(include_bytes!("../assets/tokyo-night.tmTheme")))
}
#[cfg(test)]
mod tests {
use super::parse_line_citation_fence_info;
#[test]
fn line_citation_fence_parses_start_end_path() {
assert_eq!(
parse_line_citation_fence_info("37:65:crates/example/src/tools/read.rs"),
Some(("37", "65", "crates/example/src/tools/read.rs"))
);
}
#[test]
fn line_citation_rejects_non_numeric_line() {
assert_eq!(parse_line_citation_fence_info("37:ab:file.rs"), None);
}
#[test]
fn line_citation_rejects_plain_lang_token() {
assert_eq!(parse_line_citation_fence_info("rust"), None);
assert_eq!(parse_line_citation_fence_info(""), None);
}
#[test]
fn highlight_lines_for_fence_info_resolves_citation_path_to_rust() {
let s = super::test_syntect();
assert!(
s.highlight_lines_for_fence_info("37:65:crates/codegen/xai-grok-markdown/src/parse.rs")
.is_some()
);
}
#[test]
fn highlight_lines_for_fence_info_still_accepts_rust_token() {
let s = super::test_syntect();
assert!(s.highlight_lines_for_fence_info("rust").is_some());
}
}

429
crates/codegen/xai-grok-markdown/src/url_scan.rs Normal file
View file

@ -0,0 +1,429 @@
//! Plain-URL detection over rendered display ratatui Lines.
use linkify::{LinkFinder, LinkKind};
use ratatui::text::Line;
use crate::buffers::unicode_display_width;
use crate::output::HyperlinkTarget;
/// Scan `lines` for plain URLs and return new `HyperlinkTarget` entries
/// that don't overlap any existing target in `existing`.
///
/// `next_id` is the first id to assign; the returned `u32` is the
/// post-scan counter, suitable for stuffing back into
/// `FrozenState::next_link_id`.
pub(crate) fn detect_plain_urls(
lines: &[Line<'_>],
existing: &[HyperlinkTarget],
next_id: u32,
) -> (Vec<HyperlinkTarget>, u32) {
detect_plain_urls_with_offset(lines, 0, existing, next_id)
}
/// Like [`detect_plain_urls`] but scans `lines` whose first element
/// represents document line `line_index_offset` (caller passes a tail
/// slice of `self.output.lines` and the index of its first element).
///
/// Lines fully inside `0..line_index_offset` are assumed to be in
/// `existing` already and are not re-scanned. The dedup overlap check
/// still works correctly because emitted targets use document-absolute
/// `line_index = line_index_offset + i`, matching the indices already
/// present in `existing`.
pub(crate) fn detect_plain_urls_with_offset(
lines: &[Line<'_>],
line_index_offset: usize,
existing: &[HyperlinkTarget],
next_id: u32,
) -> (Vec<HyperlinkTarget>, u32) {
let mut result = Vec::new();
let mut current_id = next_id;
let mut finder = LinkFinder::new();
finder.kinds(&[LinkKind::Url]);
for (i, line) in lines.iter().enumerate() {
let line_index = line_index_offset + i;
let mut display_col: usize = 0;
for span in &line.spans {
let span_text: &str = span.content.as_ref();
for link in finder.links(span_text) {
let before = &span_text[..link.start()];
let matched = &span_text[link.start()..link.end()];
let col_start = display_col + unicode_display_width(before);
let col_end = col_start + unicode_display_width(matched);
let url = link.as_str().to_string();
// Dedup: skip if any existing or already-added target overlaps
// on the same line. Overlap: cand.start < ex.end && ex.start < cand.end.
let overlaps = existing.iter().chain(result.iter()).any(|h| {
h.line_index == line_index
&& col_start < h.column_range.end
&& h.column_range.start < col_end
});
if !overlaps {
result.push(HyperlinkTarget {
line_index,
column_range: col_start..col_end,
url,
id: current_id,
});
current_id += 1;
}
}
display_col += unicode_display_width(span_text);
}
}
(result, current_id)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::StreamingMarkdownRenderer;
use crate::style::test_style;
/// Helper: render markdown via StreamingMarkdownRenderer::finish() and
/// return the hyperlinks from the finalized output.
fn finish_and_get_hyperlinks(text: &str) -> Vec<HyperlinkTarget> {
let mut renderer = StreamingMarkdownRenderer::new(test_style::STYLE, true);
renderer.push_and_render(text, None);
let view = renderer.finish(None);
view.hyperlinks.to_vec()
}
fn line_to_string(line: &Line<'static>) -> String {
line.spans.iter().map(|s| s.content.as_ref()).collect()
}
#[test]
fn plain_url_in_prose_produces_target() {
let text = "See https://example.com for details.\n";
let hyperlinks = finish_and_get_hyperlinks(text);
assert_eq!(hyperlinks.len(), 1, "exactly one hyperlink expected");
let h = &hyperlinks[0];
assert_eq!(h.url, "https://example.com");
// Verify column range covers only the URL
let mut renderer = StreamingMarkdownRenderer::new(test_style::STYLE, true);
renderer.push_and_render(text, None);
let view = renderer.finish(None);
let rendered = line_to_string(&view.lines[h.line_index]);
let slice: String = rendered
.chars()
.skip(h.column_range.start)
.take(h.column_range.len())
.collect();
assert_eq!(slice, "https://example.com");
}
#[test]
fn multiple_urls_one_line_distinct_ids() {
let text = "See https://a.example and https://b.example.\n";
let hyperlinks = finish_and_get_hyperlinks(text);
assert_eq!(hyperlinks.len(), 2, "two hyperlinks expected");
assert_ne!(hyperlinks[0].id, hyperlinks[1].id, "ids must differ");
assert_eq!(hyperlinks[0].url, "https://a.example");
assert_eq!(hyperlinks[1].url, "https://b.example");
// Column ranges must be disjoint
assert!(
hyperlinks[0].column_range.end <= hyperlinks[1].column_range.start,
"column ranges must be disjoint, got {:?} vs {:?}",
hyperlinks[0].column_range,
hyperlinks[1].column_range,
);
}
#[test]
fn markdown_link_with_url_text_does_not_double_link() {
let text = "Visit [https://example.com](https://example.com).\n";
let hyperlinks = finish_and_get_hyperlinks(text);
// NOTE: one might expect 1 target, but in pretty mode
// `[url](url)` renders as `url (url)`, showing the URL in two
// distinct positions: once as the link text (covered by the parser's
// HyperlinkTarget) and once in the `(url)` suffix at a disjoint
// column range. Dedup prevents a *third* entry at the same column
// range as the parser-produced target; the second entry (at the
// suffix position) is correctly detected as a separate target.
assert_eq!(
hyperlinks.len(),
2,
"expected 2 hyperlinks (parser link text + URL in pretty-mode suffix), got {}",
hyperlinks.len()
);
// Both should reference the same URL.
assert!(hyperlinks.iter().all(|h| h.url == "https://example.com"));
// Column ranges must be disjoint (dedup working correctly).
assert!(
hyperlinks[0].column_range.end <= hyperlinks[1].column_range.start
|| hyperlinks[1].column_range.end <= hyperlinks[0].column_range.start,
"column ranges must be disjoint, got {:?} and {:?}",
hyperlinks[0].column_range,
hyperlinks[1].column_range,
);
}
#[test]
fn autolink_does_not_double_link() {
let text = "Visit <https://example.com>.\n";
let hyperlinks = finish_and_get_hyperlinks(text);
// All entries should share the same URL — autolinks may produce
// multiple HyperlinkTarget fragments sharing the same id.
let autolink_count = hyperlinks
.iter()
.filter(|h| h.url == "https://example.com")
.count();
assert!(autolink_count >= 1, "expected at least one autolink target");
// The total count should match the autolink fragments only — no
// extra plain-URL duplicates.
assert_eq!(
hyperlinks.len(),
autolink_count,
"plain-URL scan should not add duplicates on top of autolink targets"
);
}
#[test]
fn trailing_period_excluded_from_url() {
let text = "See https://example.com.\n";
let hyperlinks = finish_and_get_hyperlinks(text);
assert_eq!(hyperlinks.len(), 1);
assert_eq!(
hyperlinks[0].url, "https://example.com",
"trailing dot should be excluded by linkify"
);
}
#[test]
fn cjk_neighbors_preserve_correct_columns() {
use crate::buffers::unicode_display_width;
let text = "日本語 https://example.com 日本語\n";
let hyperlinks = finish_and_get_hyperlinks(text);
assert_eq!(hyperlinks.len(), 1);
let h = &hyperlinks[0];
assert_eq!(h.url, "https://example.com");
// "日本語 " has 3 CJK chars (2 cells each) + 1 space = 7 display cells
let prefix = "日本語 ";
let expected_start = unicode_display_width(prefix);
assert_eq!(expected_start, 7, "prefix should be 7 display cells");
assert_eq!(h.column_range.start, expected_start);
let url_width = unicode_display_width("https://example.com");
assert_eq!(h.column_range.end, expected_start + url_width);
}
#[test]
fn empty_document_returns_empty() {
let hyperlinks = finish_and_get_hyperlinks("");
assert!(
hyperlinks.is_empty(),
"empty document should produce no hyperlinks"
);
}
/// Behavior pin: URL inside inline code.
///
/// This test pins the *current* behavior — if linkify matches inside
/// the code-styled span, the URL becomes a HyperlinkTarget. If we
/// later decide to skip code-styled spans, this test will fail loudly
/// and force the change to be intentional.
#[test]
fn url_inside_inline_code_documented_behavior() {
let text = "Use `https://example.com` carefully.\n";
let hyperlinks = finish_and_get_hyperlinks(text);
// Pin observed behavior: linkify finds the URL inside the
// code-styled span, producing a HyperlinkTarget.
assert!(
!hyperlinks.is_empty(),
"behavior pin: URL inside inline code currently produces a HyperlinkTarget"
);
assert_eq!(hyperlinks[0].url, "https://example.com");
}
/// Behavior pin: URL inside a fenced code block.
///
/// Same rationale as `url_inside_inline_code_documented_behavior` —
/// this pins current behavior, not product spec.
#[test]
fn url_inside_code_fence_documented_behavior() {
let text = "```\nsee https://example.com\n```\n";
let hyperlinks = finish_and_get_hyperlinks(text);
// Pin observed behavior: linkify finds the URL in the code block's
// rendered text. Whether this is desirable is a product decision;
// this test ensures any change is intentional.
let has_url = hyperlinks.iter().any(|h| h.url == "https://example.com");
assert!(
has_url,
"behavior pin: URL inside fenced code block currently produces a HyperlinkTarget"
);
}
/// URL detection must run from `render()` too, not only `finish()`.
/// Otherwise width changes or other state resets (e.g. via
/// `set_max_table_width`) drop the URL hyperlinks that pretty-mode
/// rendering adds for the `(url)` suffix of markdown links.
///
/// Also pins the OSC 8 grouping invariant: the link-text and URL
/// hyperlinks must have DISTINCT ids and DISJOINT column ranges, so
/// terminals group them as two separate hyperlinks (not one merged
/// underline across the brackets).
#[test]
fn render_detects_pretty_mode_url_suffix() {
let text = "[link](https://example.com/some/long/path)\n";
let mut renderer = StreamingMarkdownRenderer::new(test_style::STYLE, true);
renderer.push_and_render(text, None);
// No finish() call: after render() alone, both the parser-produced
// (link text) and the url_scan-produced (URL in `(url)` suffix)
// hyperlinks must be present.
let view = renderer.view();
assert_eq!(
view.hyperlinks.len(),
2,
"render() must produce both the link-text and URL-suffix hyperlinks; \
got {:?}",
view.hyperlinks,
);
assert!(
view.hyperlinks
.iter()
.all(|h| h.url == "https://example.com/some/long/path")
);
assert_ne!(
view.hyperlinks[0].id, view.hyperlinks[1].id,
"link-text and URL-suffix hyperlinks must have distinct OSC 8 ids",
);
let (a, b) = (&view.hyperlinks[0], &view.hyperlinks[1]);
assert!(
a.column_range.end <= b.column_range.start
|| b.column_range.end <= a.column_range.start,
"column ranges must be disjoint, got {:?} and {:?}",
a.column_range,
b.column_range,
);
}
/// Snapshot helper used by survival tests below.
fn snapshot(view: &crate::output::MarkdownRenderView<'_>) -> Vec<HyperlinkTarget> {
let mut snap: Vec<HyperlinkTarget> = view.hyperlinks.to_vec();
snap.sort_by_key(|h| (h.line_index, h.column_range.start));
snap
}
fn assert_url_suffix_preserved(
before: &[HyperlinkTarget],
after: &[HyperlinkTarget],
url: &str,
) {
let before_suffix = before
.iter()
.find(|h| h.url == url && h.column_range.start > 5)
.expect("URL-suffix hyperlink must be present BEFORE reset");
let after_suffix = after
.iter()
.find(|h| h.url == url && h.column_range.start > 5)
.expect("URL-suffix hyperlink must be present AFTER reset");
assert_eq!(
before_suffix.column_range, after_suffix.column_range,
"URL-suffix column range must be stable across the reset",
);
assert_eq!(
before_suffix.line_index, after_suffix.line_index,
"URL-suffix line index must be stable across the reset",
);
}
/// After `finish()`, re-rendering (e.g. triggered by a width change
/// via `set_max_table_width`) must NOT drop the URL hyperlinks that
/// pretty-mode adds for the `(url)` suffix.
///
/// Snapshots the full hyperlink list before/after the reset and
/// asserts that the URL-suffix entry survives with its column range
/// intact (the OSC 8 id may be re-assigned by the post-reset
/// re-render — that's expected — but the location must not move).
#[test]
fn url_hyperlinks_survive_re_render_after_finish() {
let url = "https://example.com/some/long/path";
let text = format!("[link]({url})\n");
let mut renderer = StreamingMarkdownRenderer::new(test_style::STYLE, true);
renderer.push_and_render(&text, None);
renderer.finish(None);
let before = snapshot(&renderer.view());
// Simulate a width change which resets renderer state.
renderer.set_max_table_width(Some(40));
renderer.render(None);
let after = snapshot(&renderer.view());
assert_eq!(
before.len(),
after.len(),
"hyperlink count must be stable across the reset; before={before:?} after={after:?}",
);
assert_url_suffix_preserved(&before, &after, url);
}
/// Identical contract to `url_hyperlinks_survive_re_render_after_finish`
/// but exercising the `set_pretty` reset path (production:
/// `MarkdownContent::set_raw_mode` toggle).
#[test]
fn url_hyperlinks_survive_re_render_after_set_pretty_toggle() {
let url = "https://example.com/some/long/path";
let text = format!("[link]({url})\n");
let mut renderer = StreamingMarkdownRenderer::new(test_style::STYLE, true);
renderer.push_and_render(&text, None);
renderer.finish(None);
let before = snapshot(&renderer.view());
// Toggle pretty off then back on — both transitions reset state.
renderer.set_pretty(false);
renderer.set_pretty(true);
renderer.render(None);
let after = snapshot(&renderer.view());
assert_eq!(
before.len(),
after.len(),
"hyperlink count must be stable across the set_pretty toggle",
);
assert_url_suffix_preserved(&before, &after, url);
}
/// Identical contract to `url_hyperlinks_survive_re_render_after_finish`
/// but exercising the `set_style` reset path (production: theme change
/// via `MarkdownContent::ensure_wrapped` when theme cache kind shifts).
#[test]
fn url_hyperlinks_survive_re_render_after_set_style() {
let url = "https://example.com/some/long/path";
let text = format!("[link]({url})\n");
let mut renderer = StreamingMarkdownRenderer::new(test_style::STYLE, true);
renderer.push_and_render(&text, None);
renderer.finish(None);
let before = snapshot(&renderer.view());
// `set_style` unconditionally resets state, even with same style.
renderer.set_style(test_style::STYLE);
renderer.render(None);
let after = snapshot(&renderer.view());
assert_eq!(
before.len(),
after.len(),
"hyperlink count must be stable across the set_style reset",
);
assert_url_suffix_preserved(&before, &after, url);
}
}