feat: improve channel message chunking boundary quality
This commit is contained in:
+14
-2
@@ -5363,10 +5363,22 @@
|
|||||||
"docs/plans/state.json"
|
"docs/plans/state.json"
|
||||||
],
|
],
|
||||||
"test_status": "pnpm test:run src/cli/setup/summary.test.ts src/cli/setup/channels.test.ts src/cli/setup/integration.test.ts + pnpm typecheck passing"
|
"test_status": "pnpm test:run src/cli/setup/summary.test.ts src/cli/setup/channels.test.ts src/cli/setup/integration.test.ts + pnpm typecheck passing"
|
||||||
|
},
|
||||||
|
"streaming-chunking-quality-tier-b5": {
|
||||||
|
"status": "completed",
|
||||||
|
"date": "2026-02-18",
|
||||||
|
"updated": "2026-02-18",
|
||||||
|
"summary": "Implemented Tier B5 chunking-quality improvements by upgrading shared channel message splitting to prefer natural boundaries (paragraph/newline/sentence/space), while preserving markdown code-fence integrity across multi-part chunks. Added regression coverage for sentence-boundary preference and code-fence balancing.",
|
||||||
|
"files_modified": [
|
||||||
|
"src/channels/utils.ts",
|
||||||
|
"src/channels/utils.test.ts",
|
||||||
|
"docs/plans/state.json"
|
||||||
|
],
|
||||||
|
"test_status": "pnpm test:run src/channels/utils.test.ts src/channels/telegram/adapter.test.ts src/channels/discord/adapter.test.ts src/channels/slack/adapter.test.ts src/channels/whatsapp/adapter.test.ts + pnpm typecheck passing"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"overall_progress": {
|
"overall_progress": {
|
||||||
"total_test_count": 1918,
|
"total_test_count": 1924,
|
||||||
"all_tests_passing": true,
|
"all_tests_passing": true,
|
||||||
"p0_completion": "3/3 (100%)",
|
"p0_completion": "3/3 (100%)",
|
||||||
"p1_completion": "4/4 (100%)",
|
"p1_completion": "4/4 (100%)",
|
||||||
@@ -5386,7 +5398,7 @@
|
|||||||
"gmail_auth_cli": "flynn gmail-auth command implemented with OAuth2 flow, doctor check, config routed to Telegram",
|
"gmail_auth_cli": "flynn gmail-auth command implemented with OAuth2 flow, doctor check, config routed to Telegram",
|
||||||
"native_audio_support": "completed — smart routing for native audio (Gemini/OpenAI/GitHub) vs Whisper transcription fallback",
|
"native_audio_support": "completed — smart routing for native audio (Gemini/OpenAI/GitHub) vs Whisper transcription fallback",
|
||||||
"remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening",
|
"remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening",
|
||||||
"next_up": "Implement Tier B5 streaming chunking quality parity checks"
|
"next_up": "Implement Tier B2 minimal companion app client"
|
||||||
},
|
},
|
||||||
"soul_md_and_cron_create": {
|
"soul_md_and_cron_create": {
|
||||||
"date": "2026-02-11",
|
"date": "2026-02-11",
|
||||||
|
|||||||
@@ -91,6 +91,23 @@ describe('splitMessage', () => {
|
|||||||
expect(result[0]).toBe('ab\ncdefghij');
|
expect(result[0]).toBe('ab\ncdefghij');
|
||||||
expect(result[1]).toBe('klmnopqrst');
|
expect(result[1]).toBe('klmnopqrst');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('splits on sentence boundaries when available', () => {
|
||||||
|
const text = 'First sentence is here. Second sentence follows with more detail.';
|
||||||
|
const result = splitMessage(text, 36);
|
||||||
|
expect(result[0]).toBe('First sentence is here.');
|
||||||
|
expect(result.length).toBeGreaterThan(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('balances markdown code fences across chunks', () => {
|
||||||
|
const text = '```ts\\nconst value = 1;\\nconsole.log(value);\\n```\\nAfter code.';
|
||||||
|
const result = splitMessage(text, 22);
|
||||||
|
expect(result.length).toBeGreaterThan(1);
|
||||||
|
for (const chunk of result) {
|
||||||
|
const fenceCount = (chunk.match(/```/g) ?? []).length;
|
||||||
|
expect(fenceCount % 2).toBe(0);
|
||||||
|
}
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('normalizeResetCommandText', () => {
|
describe('normalizeResetCommandText', () => {
|
||||||
|
|||||||
+57
-13
@@ -6,32 +6,76 @@ import type { PairingManager } from './pairing.js';
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* Split a long message into chunks that respect a platform's character limit.
|
* Split a long message into chunks that respect a platform's character limit.
|
||||||
* Prefers splitting at newlines, then spaces, then hard-cuts.
|
* Prefers splitting at natural boundaries:
|
||||||
|
* paragraph breaks, newlines, sentence endings, spaces, then hard-cuts.
|
||||||
|
* When code fences are split, chunks are balanced by closing/re-opening fences.
|
||||||
*/
|
*/
|
||||||
export function splitMessage(text: string, maxLength: number): string[] {
|
export function splitMessage(text: string, maxLength: number): string[] {
|
||||||
const chunks: string[] = [];
|
const rawChunks: string[] = [];
|
||||||
let remaining = text;
|
let remaining = text.trim();
|
||||||
|
|
||||||
while (remaining.length > 0) {
|
while (remaining.length > 0) {
|
||||||
if (remaining.length <= maxLength) {
|
if (remaining.length <= maxLength) {
|
||||||
chunks.push(remaining);
|
rawChunks.push(remaining);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try to split at a newline within the allowed window
|
const minAcceptable = Math.floor(maxLength * 0.45);
|
||||||
let splitIndex = remaining.lastIndexOf('\n', maxLength);
|
const window = remaining.slice(0, maxLength);
|
||||||
if (splitIndex === -1 || splitIndex < maxLength / 2) {
|
const paragraphSplit = window.lastIndexOf('\n\n');
|
||||||
splitIndex = remaining.lastIndexOf(' ', maxLength);
|
const newlineSplit = window.lastIndexOf('\n');
|
||||||
}
|
const sentenceSplit = window.search(/[\.\!\?;:](?!.*[\.\!\?;:])/);
|
||||||
if (splitIndex === -1 || splitIndex < maxLength / 2) {
|
const spaceSplit = window.lastIndexOf(' ');
|
||||||
splitIndex = maxLength;
|
const sentenceIndex = sentenceSplit >= 0 ? sentenceSplit + 1 : -1;
|
||||||
|
|
||||||
|
const preferred = [paragraphSplit + 1, newlineSplit, sentenceIndex, spaceSplit];
|
||||||
|
let splitIndex = maxLength;
|
||||||
|
for (const candidate of preferred) {
|
||||||
|
if (Number.isInteger(candidate) && candidate > 0 && candidate >= minAcceptable) {
|
||||||
|
splitIndex = candidate;
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
chunks.push(remaining.slice(0, splitIndex));
|
rawChunks.push(remaining.slice(0, splitIndex));
|
||||||
remaining = remaining.slice(splitIndex).trimStart();
|
remaining = remaining.slice(splitIndex).trimStart();
|
||||||
}
|
}
|
||||||
|
|
||||||
return chunks;
|
return rebalanceMarkdownCodeFences(rawChunks);
|
||||||
|
}
|
||||||
|
|
||||||
|
function rebalanceMarkdownCodeFences(chunks: string[]): string[] {
|
||||||
|
const balanced: string[] = [];
|
||||||
|
let codeFenceOpen = false;
|
||||||
|
let rawParity = 0;
|
||||||
|
|
||||||
|
for (let i = 0; i < chunks.length; i++) {
|
||||||
|
const rawChunk = chunks[i];
|
||||||
|
let chunk = rawChunk;
|
||||||
|
|
||||||
|
if (codeFenceOpen && !chunk.startsWith('```')) {
|
||||||
|
chunk = `\`\`\`\n${chunk}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const rawFenceCount = (rawChunk.match(/```/g) ?? []).length;
|
||||||
|
rawParity = (rawParity + rawFenceCount) % 2;
|
||||||
|
const hasNextChunk = i < chunks.length - 1;
|
||||||
|
|
||||||
|
if (rawParity === 1 && hasNextChunk) {
|
||||||
|
chunk = `${chunk}\n\`\`\``;
|
||||||
|
codeFenceOpen = true;
|
||||||
|
} else {
|
||||||
|
codeFenceOpen = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
balanced.push(chunk);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rawParity === 1 && balanced.length > 0) {
|
||||||
|
balanced[balanced.length - 1] = `${balanced[balanced.length - 1]}\n\`\`\``;
|
||||||
|
}
|
||||||
|
|
||||||
|
return balanced;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user