feat(audit): add phase-0 baseline summary tooling

Diagrams reviewed: docs/architecture/AGENT_DIAGRAM.md, docs/architecture/GATEWAY_SESSIONS_AND_QUEUE.md, docs/api/PROTOCOL.md (no changes required).
This commit is contained in:
William Valentin
2026-02-25 09:14:38 -08:00
parent 2311826092
commit 0b8f7c7299
6 changed files with 861 additions and 1 deletions
@@ -119,6 +119,8 @@ Extend in-memory gateway metrics with baseline counters:
## Ticket 0.4 — Baseline Summary Tooling ## Ticket 0.4 — Baseline Summary Tooling
Status: completed (2026-02-25)
### Scope ### Scope
Add or extend report tooling to summarize phase-0 telemetry slices: Add or extend report tooling to summarize phase-0 telemetry slices:
+17 -1
View File
@@ -48,6 +48,21 @@
], ],
"test_status": "pnpm test:run src/gateway/metrics.test.ts src/daemon/routing.test.ts src/gateway/handlers/agent.test.ts passing" "test_status": "pnpm test:run src/gateway/metrics.test.ts src/daemon/routing.test.ts src/gateway/handlers/agent.test.ts passing"
}, },
"phase0-ticket-0.4-baseline-summary-tooling": {
"status": "completed",
"date": "2026-02-25",
"updated": "2026-02-25",
"summary": "Implemented Phase 0 Ticket 0.4 by adding a phase-0 baseline summarizer for run outcomes, cancel latency, and reaction decision rates, with a CLI script for markdown/json outputs.",
"files_modified": [
"src/audit/phase0BaselineSummary.ts",
"src/audit/phase0BaselineSummary.test.ts",
"scripts/summarize-phase0-baseline.ts",
"package.json",
"docs/plans/2026-02-25-phase0-instrumentation-ticket-checklist.md",
"docs/plans/state.json"
],
"test_status": "pnpm test:run src/audit/phase0BaselineSummary.test.ts passing"
},
"phase0-instrumentation-ticket-checklist": { "phase0-instrumentation-ticket-checklist": {
"status": "completed", "status": "completed",
"date": "2026-02-25", "date": "2026-02-25",
@@ -6661,7 +6676,8 @@
"deeper_surfaces_phase0_ticket_01": "completed — audit schema/logger now capture run lifecycle and reaction decision baseline events (`run.state`, `run.cancel`, `reaction.match`, `reaction.skip`) with regression test coverage", "deeper_surfaces_phase0_ticket_01": "completed — audit schema/logger now capture run lifecycle and reaction decision baseline events (`run.state`, `run.cancel`, `reaction.match`, `reaction.skip`) with regression test coverage",
"deeper_surfaces_phase0_ticket_02": "completed — gateway + daemon routing emit run lifecycle/cancel telemetry and reaction match/skip audit events with filter summaries and cancellation latency, plus focused tests", "deeper_surfaces_phase0_ticket_02": "completed — gateway + daemon routing emit run lifecycle/cancel telemetry and reaction match/skip audit events with filter summaries and cancellation latency, plus focused tests",
"deeper_surfaces_phase0_ticket_03": "completed — gateway metrics now track run-state outcomes, cancel latency samples, and reaction decision counters with routing/gateway emitters", "deeper_surfaces_phase0_ticket_03": "completed — gateway metrics now track run-state outcomes, cancel latency samples, and reaction decision counters with routing/gateway emitters",
"next_up": "Implement Ticket 0.4 from docs/plans/2026-02-25-phase0-instrumentation-ticket-checklist.md", "deeper_surfaces_phase0_ticket_04": "completed — added phase-0 baseline summary tooling for run outcomes, cancel latency, and reaction decisions with markdown/json CLI output",
"next_up": "Implement Ticket 0.5 from docs/plans/2026-02-25-phase0-instrumentation-ticket-checklist.md",
"pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default", "pi_embedded_canary_spike": "completed — added optional pi_embedded backend adapter, canary-safe no-tools routing guard, backend success/fallback latency audit events, and docs/diagram updates while native remains default",
"pi_embedded_evaluation_phase": "completed — final decision rollback (applied in runtime config): Window A failed latency/fallback gates (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B remained sample-insufficient; controlled probes verified guard coverage (pi_no_tools_mode/capability_query/attachments_present each hit once)", "pi_embedded_evaluation_phase": "completed — final decision rollback (applied in runtime config): Window A failed latency/fallback gates (p50 +259ms, p95 +5695ms, fallback 25%, categories: pi_module_interface/empty_assistant_text); Window B remained sample-insufficient; controlled probes verified guard coverage (pi_no_tools_mode/capability_query/attachments_present each hit once)",
"pi_embedded_manual_mode": "completed — added persisted runtime backend controls for manual Pi activation/deactivation (`/runtime` preferred, `/backend` alias; `status`, `activate pi`, `deactivate pi`, `use config`) while keeping config-driven default routing", "pi_embedded_manual_mode": "completed — added persisted runtime backend controls for manual Pi activation/deactivation (`/runtime` preferred, `/backend` alias; `status`, `activate pi`, `deactivate pi`, `use config`) while keeping config-driven default routing",
+1
View File
@@ -21,6 +21,7 @@
"config:profiles:generate": "node scripts/generate-config-profiles.mjs", "config:profiles:generate": "node scripts/generate-config-profiles.mjs",
"config:profiles:check": "node scripts/generate-config-profiles.mjs --check", "config:profiles:check": "node scripts/generate-config-profiles.mjs --check",
"audit:backend-canary": "node --import tsx/esm scripts/summarize-backend-canary.ts", "audit:backend-canary": "node --import tsx/esm scripts/summarize-backend-canary.ts",
"audit:phase0-baseline": "node --import tsx/esm scripts/summarize-phase0-baseline.ts",
"audit:backend-canary:probes": "node --import tsx/esm scripts/run-pi-canary-guard-probes.ts" "audit:backend-canary:probes": "node --import tsx/esm scripts/run-pi-canary-guard-probes.ts"
}, },
"keywords": [ "keywords": [
+169
View File
@@ -0,0 +1,169 @@
#!/usr/bin/env node
import { writeFile } from 'node:fs/promises';
import { parseArgs } from 'node:util';
import { queryAuditLogs } from '../src/audit/export.js';
import {
renderPhase0BaselineMarkdown,
summarizePhase0Baseline,
type Phase0BaselineSummaryOptions,
} from '../src/audit/phase0BaselineSummary.js';
const DEFAULT_EVENT_TYPES = ['run.state', 'run.cancel', 'reaction.match', 'reaction.skip'] as const;
function usage(): string {
return [
'Usage: node --import tsx/esm scripts/summarize-phase0-baseline.ts --audit <path> [options]',
'',
'Options:',
' --audit <path> Path to audit.log (required)',
' --since <ISO-8601|epoch_ms> Start time filter',
' --until <ISO-8601|epoch_ms> End time filter',
' --session <id[,id...]> Restrict to session IDs',
' --channel <name[,name...]> Restrict to channels',
' --sender <id[,id...]> Restrict to senders',
' --source <gateway|channel[,..]> Restrict to sources',
' --max-sessions <number> Limit session rows in output (default: 20)',
' --max-channels <number> Limit channel rows in output (default: 20)',
' --max-skip-reasons <number> Limit skip reason rows in output (default: 10)',
' --format <markdown|json> Output format (default: markdown)',
' --out <path> Write output to file instead of stdout',
].join('\n');
}
function parseTime(value: string | undefined, flag: string): number | undefined {
if (!value) {
return undefined;
}
if (/^\d+$/.test(value)) {
const asNumber = Number(value);
if (Number.isFinite(asNumber)) {
return asNumber;
}
}
const parsed = Date.parse(value);
if (!Number.isFinite(parsed)) {
throw new Error(`Invalid ${flag} value "${value}". Use ISO-8601 or epoch milliseconds.`);
}
return parsed;
}
function parseCsv(value: string | undefined): string[] | undefined {
if (!value) {
return undefined;
}
const values = value
.split(',')
.map((item) => item.trim())
.filter((item) => item.length > 0);
return values.length > 0 ? values : undefined;
}
function parseOptionalNumber(raw: string | undefined, flag: string): number | undefined {
if (!raw) {
return undefined;
}
const parsed = Number(raw);
if (!Number.isFinite(parsed)) {
throw new Error(`Invalid ${flag} value "${raw}". Expected a number.`);
}
return parsed;
}
function parseSources(raw: string | undefined): Array<'gateway' | 'channel'> | undefined {
const values = parseCsv(raw);
if (!values) {
return undefined;
}
const parsed: Array<'gateway' | 'channel'> = [];
for (const value of values) {
if (value === 'gateway' || value === 'channel') {
parsed.push(value);
continue;
}
throw new Error(`Invalid source "${value}".`);
}
return parsed;
}
async function main(): Promise<void> {
const { values } = parseArgs({
options: {
audit: { type: 'string' },
since: { type: 'string' },
until: { type: 'string' },
session: { type: 'string' },
channel: { type: 'string' },
sender: { type: 'string' },
source: { type: 'string' },
'max-sessions': { type: 'string' },
'max-channels': { type: 'string' },
'max-skip-reasons': { type: 'string' },
format: { type: 'string' },
out: { type: 'string' },
help: { type: 'boolean', short: 'h' },
},
strict: true,
allowPositionals: false,
});
if (values.help) {
process.stdout.write(`${usage()}\n`);
return;
}
if (!values.audit) {
throw new Error('--audit is required.');
}
const format = values.format ?? 'markdown';
if (format !== 'markdown' && format !== 'json') {
throw new Error(`Invalid --format value "${format}".`);
}
const summaryOptions: Phase0BaselineSummaryOptions = {
sessionIds: parseCsv(values.session),
channels: parseCsv(values.channel),
senders: parseCsv(values.sender),
sources: parseSources(values.source),
maxSessions: parseOptionalNumber(values['max-sessions'], '--max-sessions') ?? 20,
maxChannels: parseOptionalNumber(values['max-channels'], '--max-channels') ?? 20,
maxSkipReasons: parseOptionalNumber(values['max-skip-reasons'], '--max-skip-reasons') ?? 10,
};
const startTime = parseTime(values.since, '--since');
const endTime = parseTime(values.until, '--until');
const events = await queryAuditLogs(values.audit, {
start_time: startTime,
end_time: endTime,
event_types: [...DEFAULT_EVENT_TYPES],
});
const summary = summarizePhase0Baseline(events, summaryOptions);
const output = format === 'json'
? JSON.stringify({
generated_at: new Date().toISOString(),
event_count: events.length,
filters: {
since_ms: startTime,
until_ms: endTime,
},
options: summaryOptions,
summary,
}, null, 2)
: renderPhase0BaselineMarkdown(summary, summaryOptions);
if (values.out) {
await writeFile(values.out, `${output}\n`, 'utf-8');
} else {
process.stdout.write(`${output}\n`);
}
}
main().catch((error) => {
const message = error instanceof Error ? error.message : String(error);
process.stderr.write(`${message}\n\n${usage()}\n`);
process.exitCode = 1;
});
+195
View File
@@ -0,0 +1,195 @@
import { describe, expect, it } from 'vitest';
import type { AuditEvent } from './types.js';
import { renderPhase0BaselineMarkdown, summarizePhase0Baseline } from './phase0BaselineSummary.js';
function makeEvent(
timestamp: number,
event_type: AuditEvent['event_type'],
event: Record<string, unknown>,
): AuditEvent {
return {
timestamp,
level: 'info',
event_type,
event,
};
}
describe('summarizePhase0Baseline', () => {
it('summarizes run outcomes, cancel latency, and reaction decisions', () => {
const events: AuditEvent[] = [
makeEvent(1000, 'run.state', {
session_id: 'telegram:s1',
channel: 'telegram',
sender: 'u1',
source: 'channel',
state: 'start',
}),
makeEvent(1200, 'run.state', {
session_id: 'telegram:s1',
channel: 'telegram',
sender: 'u1',
source: 'channel',
state: 'complete',
}),
makeEvent(2000, 'run.state', {
session_id: 'discord:s2',
channel: 'discord',
sender: 'u2',
source: 'gateway',
state: 'start',
}),
makeEvent(2400, 'run.state', {
session_id: 'discord:s2',
channel: 'discord',
sender: 'u2',
source: 'gateway',
state: 'error',
}),
makeEvent(3000, 'run.state', {
session_id: 'telegram:s3',
channel: 'telegram',
sender: 'u3',
source: 'channel',
state: 'start',
}),
makeEvent(3200, 'run.state', {
session_id: 'telegram:s3',
channel: 'telegram',
sender: 'u3',
source: 'channel',
state: 'cancelled',
}),
makeEvent(3300, 'run.state', {
session_id: 'telegram:s3',
channel: 'telegram',
sender: 'u3',
source: 'channel',
state: 'cancel_requested',
}),
makeEvent(3500, 'run.cancel', {
session_id: 'telegram:s3',
channel: 'telegram',
sender: 'u3',
source: 'channel',
requested: true,
acknowledged: true,
latency_ms: 120,
}),
makeEvent(3600, 'run.cancel', {
session_id: 'discord:s2',
channel: 'discord',
sender: 'u2',
source: 'gateway',
requested: true,
acknowledged: false,
latency_ms: 300,
}),
makeEvent(3700, 'reaction.match', {
session_id: 'telegram:s1',
channel: 'telegram',
sender: 'u1',
source: 'channel',
rule_name: 'boss-email',
}),
makeEvent(3800, 'reaction.skip', {
session_id: 'telegram:s1',
channel: 'telegram',
sender: 'u1',
source: 'channel',
reason: 'no_match',
candidate_count: 1,
}),
makeEvent(3900, 'reaction.skip', {
session_id: 'discord:s2',
channel: 'discord',
sender: 'u2',
source: 'gateway',
reason: 'no_rules',
candidate_count: 0,
}),
];
const summary = summarizePhase0Baseline(events);
expect(summary.event_counts.run_state).toBe(7);
expect(summary.run_outcomes.overall.total_outcomes).toBe(3);
expect(summary.run_outcomes.overall.complete).toBe(1);
expect(summary.run_outcomes.overall.cancelled).toBe(1);
expect(summary.run_outcomes.overall.error).toBe(1);
expect(summary.run_outcomes.overall.cancel_requested).toBe(1);
expect(summary.run_outcomes.overall.start).toBe(3);
const telegram = summary.run_outcomes.by_channel.find((row) => row.key === 'telegram');
expect(telegram?.stats.total_outcomes).toBe(2);
const discord = summary.run_outcomes.by_channel.find((row) => row.key === 'discord');
expect(discord?.stats.total_outcomes).toBe(1);
const cancelStats = summary.cancel_latency_ms;
expect(cancelStats?.count).toBe(2);
expect(cancelStats?.p50_ms).toBe(210);
expect(cancelStats?.p95_ms).toBe(291);
expect(summary.reactions.matched).toBe(1);
expect(summary.reactions.skipped).toBe(2);
expect(summary.reactions.match_rate_pct).toBe(33.33);
expect(summary.reactions.skip_reasons).toEqual([
{ reason: 'no_match', count: 1, pct: 50 },
{ reason: 'no_rules', count: 1, pct: 50 },
]);
});
it('filters by channel', () => {
const events: AuditEvent[] = [
makeEvent(1000, 'run.state', {
session_id: 'telegram:s1',
channel: 'telegram',
sender: 'u1',
source: 'channel',
state: 'complete',
}),
makeEvent(1100, 'run.state', {
session_id: 'discord:s2',
channel: 'discord',
sender: 'u2',
source: 'gateway',
state: 'error',
}),
];
const summary = summarizePhase0Baseline(events, { channels: ['telegram'] });
expect(summary.run_outcomes.overall.total_outcomes).toBe(1);
expect(summary.run_outcomes.by_channel).toHaveLength(1);
expect(summary.run_outcomes.by_channel[0]?.key).toBe('telegram');
});
});
describe('renderPhase0BaselineMarkdown', () => {
it('renders key sections', () => {
const events: AuditEvent[] = [
makeEvent(1000, 'run.state', {
session_id: 'telegram:s1',
channel: 'telegram',
sender: 'u1',
source: 'channel',
state: 'complete',
}),
makeEvent(1200, 'reaction.skip', {
session_id: 'telegram:s1',
channel: 'telegram',
sender: 'u1',
source: 'channel',
reason: 'no_match',
candidate_count: 1,
}),
];
const summary = summarizePhase0Baseline(events);
const markdown = renderPhase0BaselineMarkdown(summary);
expect(markdown).toContain('Phase 0 Baseline Telemetry Summary');
expect(markdown).toContain('Run Outcomes (Overall)');
expect(markdown).toContain('Reaction Decisions');
expect(markdown).toContain('no_match');
});
});
+477
View File
@@ -0,0 +1,477 @@
import type { AuditEvent } from './types.js';
export type AuditSource = 'gateway' | 'channel';
export type RunState = 'start' | 'complete' | 'cancel_requested' | 'cancelled' | 'error';
export interface Phase0BaselineSummaryOptions {
sessionIds?: string[];
channels?: string[];
senders?: string[];
sources?: AuditSource[];
maxSessions?: number;
maxChannels?: number;
maxSkipReasons?: number;
}
export interface RunOutcomeStats {
total_outcomes: number;
complete: number;
cancelled: number;
error: number;
cancel_requested: number;
start: number;
completion_rate_pct: number | null;
cancel_rate_pct: number | null;
error_rate_pct: number | null;
}
export interface RunOutcomeGroup {
key: string;
stats: RunOutcomeStats;
}
export interface CancelLatencyStats {
count: number;
avg_ms: number;
p50_ms: number;
p95_ms: number;
min_ms: number;
max_ms: number;
}
export interface ReactionSkipReasonStats {
reason: string;
count: number;
pct: number;
}
export interface ReactionSummary {
matched: number;
skipped: number;
total: number;
match_rate_pct: number | null;
skip_rate_pct: number | null;
skip_reasons: ReactionSkipReasonStats[];
}
export interface Phase0BaselineSummary {
event_counts: {
run_state: number;
run_cancel: number;
reaction_match: number;
reaction_skip: number;
};
run_outcomes: {
overall: RunOutcomeStats;
by_channel: RunOutcomeGroup[];
by_session: RunOutcomeGroup[];
};
cancel_latency_ms: CancelLatencyStats | null;
reactions: ReactionSummary;
}
interface RunCounter {
start: number;
complete: number;
cancel_requested: number;
cancelled: number;
error: number;
}
function createRunCounter(): RunCounter {
return {
start: 0,
complete: 0,
cancel_requested: 0,
cancelled: 0,
error: 0,
};
}
function toRecord(value: unknown): Record<string, unknown> {
return (value && typeof value === 'object') ? value as Record<string, unknown> : {};
}
function readString(value: unknown): string | undefined {
return typeof value === 'string' ? value : undefined;
}
function readNumber(value: unknown): number | undefined {
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
}
function isRunState(value: unknown): value is RunState {
return value === 'start'
|| value === 'complete'
|| value === 'cancel_requested'
|| value === 'cancelled'
|| value === 'error';
}
function isAuditSource(value: unknown): value is AuditSource {
return value === 'gateway' || value === 'channel';
}
function toPct(part: number, whole: number): number | null {
if (whole <= 0) {
return null;
}
return Math.round((part / whole) * 10000) / 100;
}
function percentile(sortedAscending: number[], pct: number): number {
if (sortedAscending.length === 0) {
return 0;
}
if (sortedAscending.length === 1) {
return sortedAscending[0];
}
const clampedPct = Math.max(0, Math.min(100, pct));
const position = (clampedPct / 100) * (sortedAscending.length - 1);
const lowerIndex = Math.floor(position);
const upperIndex = Math.ceil(position);
if (lowerIndex === upperIndex) {
return sortedAscending[lowerIndex] ?? 0;
}
const lower = sortedAscending[lowerIndex] ?? 0;
const upper = sortedAscending[upperIndex] ?? 0;
const weight = position - lowerIndex;
return lower + ((upper - lower) * weight);
}
function computeLatencyStats(samples: number[]): CancelLatencyStats | null {
if (samples.length === 0) {
return null;
}
const sorted = [...samples].sort((a, b) => a - b);
const total = sorted.reduce((sum, value) => sum + value, 0);
return {
count: sorted.length,
avg_ms: Math.round(total / sorted.length),
p50_ms: Math.round(percentile(sorted, 50)),
p95_ms: Math.round(percentile(sorted, 95)),
min_ms: sorted[0] ?? 0,
max_ms: sorted[sorted.length - 1] ?? 0,
};
}
function buildRunOutcomeStats(counter: RunCounter): RunOutcomeStats {
const totalOutcomes = counter.complete + counter.cancelled + counter.error;
return {
total_outcomes: totalOutcomes,
complete: counter.complete,
cancelled: counter.cancelled,
error: counter.error,
cancel_requested: counter.cancel_requested,
start: counter.start,
completion_rate_pct: toPct(counter.complete, totalOutcomes),
cancel_rate_pct: toPct(counter.cancelled, totalOutcomes),
error_rate_pct: toPct(counter.error, totalOutcomes),
};
}
function sortGroups(groups: Map<string, RunCounter>, limit?: number): RunOutcomeGroup[] {
const rows = [...groups.entries()]
.map(([key, counter]) => ({ key, stats: buildRunOutcomeStats(counter) }))
.sort((a, b) => {
const delta = b.stats.total_outcomes - a.stats.total_outcomes;
if (delta !== 0) {
return delta;
}
return a.key.localeCompare(b.key);
});
if (typeof limit === 'number' && Number.isFinite(limit) && limit > 0) {
return rows.slice(0, Math.floor(limit));
}
return rows;
}
function sortSkipReasons(reasons: Map<string, number>, limit?: number): ReactionSkipReasonStats[] {
const total = [...reasons.values()].reduce((sum, value) => sum + value, 0);
const rows = [...reasons.entries()]
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
.map(([reason, count]) => ({
reason,
count,
pct: total > 0 ? Math.round((count / total) * 10000) / 100 : 0,
}));
if (typeof limit === 'number' && Number.isFinite(limit) && limit > 0) {
return rows.slice(0, Math.floor(limit));
}
return rows;
}
function shouldInclude(
payload: Record<string, unknown>,
filters: {
session: Set<string>;
channel: Set<string>;
sender: Set<string>;
source: Set<string>;
},
): { sessionId: string | undefined; channel: string | undefined; sender: string | undefined; source: AuditSource | undefined; include: boolean } {
const sessionId = readString(payload.session_id);
const channel = readString(payload.channel);
const sender = readString(payload.sender);
const sourceRaw = readString(payload.source);
const source = isAuditSource(sourceRaw) ? sourceRaw : undefined;
if (!channel || !sender || !source) {
return { sessionId, channel, sender, source, include: false };
}
if (filters.session.size > 0 && (!sessionId || !filters.session.has(sessionId))) {
return { sessionId, channel, sender, source, include: false };
}
if (filters.channel.size > 0 && !filters.channel.has(channel)) {
return { sessionId, channel, sender, source, include: false };
}
if (filters.sender.size > 0 && !filters.sender.has(sender)) {
return { sessionId, channel, sender, source, include: false };
}
if (filters.source.size > 0 && !filters.source.has(source)) {
return { sessionId, channel, sender, source, include: false };
}
return { sessionId, channel, sender, source, include: true };
}
export function summarizePhase0Baseline(
events: AuditEvent[],
options: Phase0BaselineSummaryOptions = {},
): Phase0BaselineSummary {
const sessionFilter = new Set(options.sessionIds ?? []);
const channelFilter = new Set(options.channels ?? []);
const senderFilter = new Set(options.senders ?? []);
const sourceFilter = new Set(options.sources ?? []);
const filters = {
session: sessionFilter,
channel: channelFilter,
sender: senderFilter,
source: sourceFilter,
};
const runTotals = createRunCounter();
const runByChannel = new Map<string, RunCounter>();
const runBySession = new Map<string, RunCounter>();
const cancelLatencies: number[] = [];
const skipReasons = new Map<string, number>();
const eventCounts = {
run_state: 0,
run_cancel: 0,
reaction_match: 0,
reaction_skip: 0,
};
let reactionMatched = 0;
let reactionSkipped = 0;
for (const event of events) {
const payload = toRecord(event.event);
if (event.event_type === 'run.state') {
const state = readString(payload.state);
if (!isRunState(state)) {
continue;
}
const filterResult = shouldInclude(payload, filters);
if (!filterResult.include) {
continue;
}
eventCounts.run_state += 1;
runTotals[state] += 1;
const channelCounter = runByChannel.get(filterResult.channel!) ?? createRunCounter();
channelCounter[state] += 1;
runByChannel.set(filterResult.channel!, channelCounter);
if (filterResult.sessionId) {
const sessionCounter = runBySession.get(filterResult.sessionId) ?? createRunCounter();
sessionCounter[state] += 1;
runBySession.set(filterResult.sessionId, sessionCounter);
}
continue;
}
if (event.event_type === 'run.cancel') {
const filterResult = shouldInclude(payload, filters);
if (!filterResult.include) {
continue;
}
eventCounts.run_cancel += 1;
const latency = readNumber(payload.latency_ms);
if (typeof latency === 'number' && latency >= 0) {
cancelLatencies.push(latency);
}
continue;
}
if (event.event_type === 'reaction.match') {
const filterResult = shouldInclude(payload, filters);
if (!filterResult.include) {
continue;
}
eventCounts.reaction_match += 1;
reactionMatched += 1;
continue;
}
if (event.event_type === 'reaction.skip') {
const filterResult = shouldInclude(payload, filters);
if (!filterResult.include) {
continue;
}
eventCounts.reaction_skip += 1;
reactionSkipped += 1;
const reason = readString(payload.reason) ?? 'unknown';
skipReasons.set(reason, (skipReasons.get(reason) ?? 0) + 1);
}
}
const totalReactions = reactionMatched + reactionSkipped;
return {
event_counts: eventCounts,
run_outcomes: {
overall: buildRunOutcomeStats(runTotals),
by_channel: sortGroups(runByChannel, options.maxChannels),
by_session: sortGroups(runBySession, options.maxSessions),
},
cancel_latency_ms: computeLatencyStats(cancelLatencies),
reactions: {
matched: reactionMatched,
skipped: reactionSkipped,
total: totalReactions,
match_rate_pct: toPct(reactionMatched, totalReactions),
skip_rate_pct: toPct(reactionSkipped, totalReactions),
skip_reasons: sortSkipReasons(skipReasons, options.maxSkipReasons),
},
};
}
function formatPct(value: number | null): string {
return value === null ? 'n/a' : `${value.toFixed(2)}%`;
}
export function renderPhase0BaselineMarkdown(
summary: Phase0BaselineSummary,
options: Phase0BaselineSummaryOptions = {},
): string {
const lines: string[] = [];
lines.push('# Phase 0 Baseline Telemetry Summary');
lines.push('');
lines.push(`- Run state events: ${summary.event_counts.run_state}`);
lines.push(`- Run cancel events: ${summary.event_counts.run_cancel}`);
lines.push(`- Reaction matches: ${summary.event_counts.reaction_match}`);
lines.push(`- Reaction skips: ${summary.event_counts.reaction_skip}`);
lines.push('');
if (options.sessionIds?.length) {
lines.push(`- Sessions: ${options.sessionIds.join(', ')}`);
}
if (options.channels?.length) {
lines.push(`- Channels: ${options.channels.join(', ')}`);
}
if (options.senders?.length) {
lines.push(`- Senders: ${options.senders.join(', ')}`);
}
if (options.sources?.length) {
lines.push(`- Sources: ${options.sources.join(', ')}`);
}
if (
(options.sessionIds?.length ?? 0) > 0
|| (options.channels?.length ?? 0) > 0
|| (options.senders?.length ?? 0) > 0
|| (options.sources?.length ?? 0) > 0
) {
lines.push('');
}
lines.push('## Run Outcomes (Overall)');
lines.push('');
lines.push(`- Total outcomes: ${summary.run_outcomes.overall.total_outcomes}`);
lines.push(`- Complete: ${summary.run_outcomes.overall.complete} (${formatPct(summary.run_outcomes.overall.completion_rate_pct)})`);
lines.push(`- Cancelled: ${summary.run_outcomes.overall.cancelled} (${formatPct(summary.run_outcomes.overall.cancel_rate_pct)})`);
lines.push(`- Errors: ${summary.run_outcomes.overall.error} (${formatPct(summary.run_outcomes.overall.error_rate_pct)})`);
lines.push(`- Cancel requested: ${summary.run_outcomes.overall.cancel_requested}`);
lines.push(`- Starts: ${summary.run_outcomes.overall.start}`);
lines.push('');
lines.push('## Run Outcomes by Channel');
lines.push('');
lines.push('| Channel | Outcomes | Complete | Cancelled | Error | Complete % | Cancel % | Error % | Cancel Req | Starts |');
lines.push('| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |');
if (summary.run_outcomes.by_channel.length === 0) {
lines.push('| _none_ | 0 | 0 | 0 | 0 | n/a | n/a | n/a | 0 | 0 |');
} else {
for (const row of summary.run_outcomes.by_channel) {
lines.push(
`| ${row.key} | ${row.stats.total_outcomes} | ${row.stats.complete} | ${row.stats.cancelled} | ${row.stats.error} | `
+ `${formatPct(row.stats.completion_rate_pct)} | ${formatPct(row.stats.cancel_rate_pct)} | ${formatPct(row.stats.error_rate_pct)} | `
+ `${row.stats.cancel_requested} | ${row.stats.start} |`,
);
}
}
lines.push('');
lines.push('## Run Outcomes by Session');
lines.push('');
lines.push('| Session | Outcomes | Complete | Cancelled | Error | Complete % | Cancel % | Error % | Cancel Req | Starts |');
lines.push('| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |');
if (summary.run_outcomes.by_session.length === 0) {
lines.push('| _none_ | 0 | 0 | 0 | 0 | n/a | n/a | n/a | 0 | 0 |');
} else {
for (const row of summary.run_outcomes.by_session) {
lines.push(
`| ${row.key} | ${row.stats.total_outcomes} | ${row.stats.complete} | ${row.stats.cancelled} | ${row.stats.error} | `
+ `${formatPct(row.stats.completion_rate_pct)} | ${formatPct(row.stats.cancel_rate_pct)} | ${formatPct(row.stats.error_rate_pct)} | `
+ `${row.stats.cancel_requested} | ${row.stats.start} |`,
);
}
}
lines.push('');
lines.push('## Cancel Latency');
lines.push('');
if (!summary.cancel_latency_ms) {
lines.push('- No cancel latency samples.');
} else {
const stats = summary.cancel_latency_ms;
lines.push(`- Count: ${stats.count}`);
lines.push(`- Avg: ${stats.avg_ms}ms`);
lines.push(`- P50: ${stats.p50_ms}ms`);
lines.push(`- P95: ${stats.p95_ms}ms`);
lines.push(`- Min: ${stats.min_ms}ms`);
lines.push(`- Max: ${stats.max_ms}ms`);
}
lines.push('');
lines.push('## Reaction Decisions');
lines.push('');
lines.push(`- Matched: ${summary.reactions.matched} (${formatPct(summary.reactions.match_rate_pct)})`);
lines.push(`- Skipped: ${summary.reactions.skipped} (${formatPct(summary.reactions.skip_rate_pct)})`);
lines.push('');
lines.push('### Skip Reasons');
lines.push('');
lines.push('| Reason | Count | Percent |');
lines.push('| --- | ---: | ---: |');
if (summary.reactions.skip_reasons.length === 0) {
lines.push('| _none_ | 0 | 0.00% |');
} else {
for (const reason of summary.reactions.skip_reasons) {
lines.push(`| ${reason.reason} | ${reason.count} | ${reason.pct.toFixed(2)}% |`);
}
}
lines.push('');
return lines.join('\n');
}