From 7ca5d5bff5fb65a8edd9a38d75ad1835ca6f2f3a Mon Sep 17 00:00:00 2001 From: William Valentin Date: Wed, 18 Feb 2026 10:50:34 -0800 Subject: [PATCH] feat: improve channel message chunking boundary quality --- docs/plans/state.json | 16 +++++++-- src/channels/utils.test.ts | 17 +++++++++ src/channels/utils.ts | 70 +++++++++++++++++++++++++++++++------- 3 files changed, 88 insertions(+), 15 deletions(-) diff --git a/docs/plans/state.json b/docs/plans/state.json index 15131ad..0ee1bec 100644 --- a/docs/plans/state.json +++ b/docs/plans/state.json @@ -5363,10 +5363,22 @@ "docs/plans/state.json" ], "test_status": "pnpm test:run src/cli/setup/summary.test.ts src/cli/setup/channels.test.ts src/cli/setup/integration.test.ts + pnpm typecheck passing" + }, + "streaming-chunking-quality-tier-b5": { + "status": "completed", + "date": "2026-02-18", + "updated": "2026-02-18", + "summary": "Implemented Tier B5 chunking-quality improvements by upgrading shared channel message splitting to prefer natural boundaries (paragraph/newline/sentence/space), while preserving markdown code-fence integrity across multi-part chunks. Added regression coverage for sentence-boundary preference and code-fence balancing.", + "files_modified": [ + "src/channels/utils.ts", + "src/channels/utils.test.ts", + "docs/plans/state.json" + ], + "test_status": "pnpm test:run src/channels/utils.test.ts src/channels/telegram/adapter.test.ts src/channels/discord/adapter.test.ts src/channels/slack/adapter.test.ts src/channels/whatsapp/adapter.test.ts + pnpm typecheck passing" } }, "overall_progress": { - "total_test_count": 1918, + "total_test_count": 1924, "all_tests_passing": true, "p0_completion": "3/3 (100%)", "p1_completion": "4/4 (100%)", @@ -5386,7 +5398,7 @@ "gmail_auth_cli": "flynn gmail-auth command implemented with OAuth2 flow, doctor check, config routed to Telegram", "native_audio_support": "completed — smart routing for native audio (Gemini/OpenAI/GitHub) vs Whisper transcription fallback", "remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening", - "next_up": "Implement Tier B5 streaming chunking quality parity checks" + "next_up": "Implement Tier B2 minimal companion app client" }, "soul_md_and_cron_create": { "date": "2026-02-11", diff --git a/src/channels/utils.test.ts b/src/channels/utils.test.ts index 7c22204..85f92a1 100644 --- a/src/channels/utils.test.ts +++ b/src/channels/utils.test.ts @@ -91,6 +91,23 @@ describe('splitMessage', () => { expect(result[0]).toBe('ab\ncdefghij'); expect(result[1]).toBe('klmnopqrst'); }); + + it('splits on sentence boundaries when available', () => { + const text = 'First sentence is here. Second sentence follows with more detail.'; + const result = splitMessage(text, 36); + expect(result[0]).toBe('First sentence is here.'); + expect(result.length).toBeGreaterThan(1); + }); + + it('balances markdown code fences across chunks', () => { + const text = '```ts\\nconst value = 1;\\nconsole.log(value);\\n```\\nAfter code.'; + const result = splitMessage(text, 22); + expect(result.length).toBeGreaterThan(1); + for (const chunk of result) { + const fenceCount = (chunk.match(/```/g) ?? []).length; + expect(fenceCount % 2).toBe(0); + } + }); }); describe('normalizeResetCommandText', () => { diff --git a/src/channels/utils.ts b/src/channels/utils.ts index 45da27d..09ea7e1 100644 --- a/src/channels/utils.ts +++ b/src/channels/utils.ts @@ -6,32 +6,76 @@ import type { PairingManager } from './pairing.js'; /** * Split a long message into chunks that respect a platform's character limit. - * Prefers splitting at newlines, then spaces, then hard-cuts. + * Prefers splitting at natural boundaries: + * paragraph breaks, newlines, sentence endings, spaces, then hard-cuts. + * When code fences are split, chunks are balanced by closing/re-opening fences. */ export function splitMessage(text: string, maxLength: number): string[] { - const chunks: string[] = []; - let remaining = text; + const rawChunks: string[] = []; + let remaining = text.trim(); while (remaining.length > 0) { if (remaining.length <= maxLength) { - chunks.push(remaining); + rawChunks.push(remaining); break; } - // Try to split at a newline within the allowed window - let splitIndex = remaining.lastIndexOf('\n', maxLength); - if (splitIndex === -1 || splitIndex < maxLength / 2) { - splitIndex = remaining.lastIndexOf(' ', maxLength); - } - if (splitIndex === -1 || splitIndex < maxLength / 2) { - splitIndex = maxLength; + const minAcceptable = Math.floor(maxLength * 0.45); + const window = remaining.slice(0, maxLength); + const paragraphSplit = window.lastIndexOf('\n\n'); + const newlineSplit = window.lastIndexOf('\n'); + const sentenceSplit = window.search(/[\.\!\?;:](?!.*[\.\!\?;:])/); + const spaceSplit = window.lastIndexOf(' '); + const sentenceIndex = sentenceSplit >= 0 ? sentenceSplit + 1 : -1; + + const preferred = [paragraphSplit + 1, newlineSplit, sentenceIndex, spaceSplit]; + let splitIndex = maxLength; + for (const candidate of preferred) { + if (Number.isInteger(candidate) && candidate > 0 && candidate >= minAcceptable) { + splitIndex = candidate; + break; + } } - chunks.push(remaining.slice(0, splitIndex)); + rawChunks.push(remaining.slice(0, splitIndex)); remaining = remaining.slice(splitIndex).trimStart(); } - return chunks; + return rebalanceMarkdownCodeFences(rawChunks); +} + +function rebalanceMarkdownCodeFences(chunks: string[]): string[] { + const balanced: string[] = []; + let codeFenceOpen = false; + let rawParity = 0; + + for (let i = 0; i < chunks.length; i++) { + const rawChunk = chunks[i]; + let chunk = rawChunk; + + if (codeFenceOpen && !chunk.startsWith('```')) { + chunk = `\`\`\`\n${chunk}`; + } + + const rawFenceCount = (rawChunk.match(/```/g) ?? []).length; + rawParity = (rawParity + rawFenceCount) % 2; + const hasNextChunk = i < chunks.length - 1; + + if (rawParity === 1 && hasNextChunk) { + chunk = `${chunk}\n\`\`\``; + codeFenceOpen = true; + } else { + codeFenceOpen = false; + } + + balanced.push(chunk); + } + + if (rawParity === 1 && balanced.length > 0) { + balanced[balanced.length - 1] = `${balanced[balanced.length - 1]}\n\`\`\``; + } + + return balanced; } /**