From 93621bbe6e36dea0fa6c30819b4be394fd1b0c79 Mon Sep 17 00:00:00 2001 From: William Valentin Date: Mon, 16 Feb 2026 14:11:50 -0800 Subject: [PATCH] feat(analytics): add top tools and topics to session analytics --- docs/api/PROTOCOL.md | 6 +++ docs/plans/state.json | 8 ++-- src/daemon/index.ts | 6 +++ src/gateway/handlers/handlers.test.ts | 8 ++++ src/gateway/handlers/system.ts | 2 + src/session/index.ts | 8 +++- src/session/store.test.ts | 19 +++++++-- src/session/store.ts | 59 +++++++++++++++++++++++++++ src/tools/executor.test.ts | 25 ++++++++++++ src/tools/executor.ts | 40 ++++++++++++++++++ 10 files changed, 173 insertions(+), 8 deletions(-) diff --git a/docs/api/PROTOCOL.md b/docs/api/PROTOCOL.md index 0f1100e..2470d74 100644 --- a/docs/api/PROTOCOL.md +++ b/docs/api/PROTOCOL.md @@ -332,6 +332,12 @@ Useful for operator dashboards and trend checks (sessions/day, message volume, t "topSessions": [ { "sessionId": "telegram:123456", "messages": 42, "lastActivity": 1739700300 } ], + "topTools": [ + { "toolName": "web.search", "executions": 37 } + ], + "topTopics": [ + { "topic": "kubernetes", "occurrences": 22 } + ], "averageMessagesPerSession": 16.29, "totalSessions": 14, "totalMessages": 228 diff --git a/docs/plans/state.json b/docs/plans/state.json index 04d4080..f7839a1 100644 --- a/docs/plans/state.json +++ b/docs/plans/state.json @@ -7,7 +7,7 @@ "status": "completed", "date": "2026-02-16", "updated": "2026-02-16", - "summary": "Added first-class automation presets and scheduling upgrades: `automation.daily_briefing` now auto-registers an opinionated cron job for morning briefings, and backup scheduling now supports cron expressions via `backup.schedule` plus optional `backup.run_on_start` while preserving interval fallback. Added `BackupScheduler` with `backup.notify` channel alerts, configurable `backup.failure_threshold`, and recovery notifications (`backup.notify_recovery`) so backup failures/recoveries proactively notify operators. Extended heartbeat monitoring with `process_memory`, `backup`, and `provider_errors` checks (with thresholds) so high RSS usage, backup failure streaks, and model-provider error spikes proactively trigger health alerts. Added timezone-safe daily briefing dedupe via `automation.daily_briefing.dedupe_per_local_day` and cron-level `once_per_local_day` so morning briefings do not send twice on the same local day. Added `minio.share` tool to upload local artifacts and return temporary MinIO share links using existing backup MinIO credentials. Added `system.sessionAnalytics` RPC with SQLite-backed aggregates (daily sessions/messages, average messages per session, top active sessions) for operator usage tracking.", + "summary": "Added first-class automation presets and scheduling upgrades: `automation.daily_briefing` now auto-registers an opinionated cron job for morning briefings, and backup scheduling now supports cron expressions via `backup.schedule` plus optional `backup.run_on_start` while preserving interval fallback. Added `BackupScheduler` with `backup.notify` channel alerts, configurable `backup.failure_threshold`, and recovery notifications (`backup.notify_recovery`) so backup failures/recoveries proactively notify operators. Extended heartbeat monitoring with `process_memory`, `backup`, and `provider_errors` checks (with thresholds) so high RSS usage, backup failure streaks, and model-provider error spikes proactively trigger health alerts. Added timezone-safe daily briefing dedupe via `automation.daily_briefing.dedupe_per_local_day` and cron-level `once_per_local_day` so morning briefings do not send twice on the same local day. Added `minio.share` tool to upload local artifacts and return temporary MinIO share links using existing backup MinIO credentials. Added `system.sessionAnalytics` RPC with SQLite-backed aggregates including daily sessions/messages, average messages per session, top active sessions, most-used tools, and common topics for operator usage tracking.", "files_modified": [ "src/config/schema.ts", "src/config/schema.test.ts", @@ -32,6 +32,8 @@ "src/session/store.test.ts", "src/session/manager.ts", "src/session/index.ts", + "src/tools/executor.ts", + "src/tools/executor.test.ts", "src/gateway/handlers/system.ts", "src/gateway/handlers/handlers.test.ts", "src/gateway/server.ts", @@ -46,7 +48,7 @@ "config/default.yaml", "README.md" ], - "test_status": "pnpm test:run src/automation/presets.test.ts src/automation/cron.test.ts src/automation/heartbeat.test.ts src/backup/scheduler.test.ts src/backup/status.test.ts src/tools/builtin/minio-share.test.ts src/tools/policy.test.ts src/config/schema.test.ts src/daemon/channels.test.ts src/gateway/handlers/services.test.ts src/session/store.test.ts src/gateway/handlers/handlers.test.ts + pnpm typecheck passing" + "test_status": "pnpm test:run src/automation/presets.test.ts src/automation/cron.test.ts src/automation/heartbeat.test.ts src/backup/scheduler.test.ts src/backup/status.test.ts src/tools/builtin/minio-share.test.ts src/tools/policy.test.ts src/config/schema.test.ts src/daemon/channels.test.ts src/gateway/handlers/services.test.ts src/session/store.test.ts src/tools/executor.test.ts src/gateway/handlers/handlers.test.ts + pnpm typecheck passing" }, "backup-session-summary-audit-trail": { "status": "completed", @@ -3332,7 +3334,7 @@ } }, "overall_progress": { - "total_test_count": 1851, + "total_test_count": 1852, "all_tests_passing": true, "p0_completion": "3/3 (100%)", "p1_completion": "4/4 (100%)", diff --git a/src/daemon/index.ts b/src/daemon/index.ts index 19270f4..2f9c48a 100644 --- a/src/daemon/index.ts +++ b/src/daemon/index.ts @@ -107,6 +107,12 @@ export async function startDaemon(config: Config, options?: StartDaemonOptions): // ── Core Services ── const hookEngine = new HookEngine(config.hooks); const { toolRegistry, toolExecutor, browserManager } = initTools({ config, lifecycle, hookEngine }); + toolExecutor.setExecutionObserver((event) => { + if (!event.sessionId) { + return; + } + sessionStore.recordToolExecution(event.sessionId, event.toolName, event.success, event.timestampSeconds); + }); const { memoryStore, memoryDir } = await initMemory({ config, dataDir, lifecycle, toolRegistry }); const mcpManager = await initMcp(config, lifecycle, toolRegistry); const { skillRegistry, skillInstaller } = initSkills(config, lifecycle); diff --git a/src/gateway/handlers/handlers.test.ts b/src/gateway/handlers/handlers.test.ts index 300dc97..b3b95de 100644 --- a/src/gateway/handlers/handlers.test.ts +++ b/src/gateway/handlers/handlers.test.ts @@ -344,12 +344,16 @@ describe('system.sessionAnalytics handler', () => { const r = result.result as { daily: unknown[]; topSessions: unknown[]; + topTools: unknown[]; + topTopics: unknown[]; averageMessagesPerSession: number; totalSessions: number; totalMessages: number; }; expect(r.daily).toEqual([]); expect(r.topSessions).toEqual([]); + expect(r.topTools).toEqual([]); + expect(r.topTopics).toEqual([]); expect(r.averageMessagesPerSession).toBe(0); expect(r.totalSessions).toBe(0); expect(r.totalMessages).toBe(0); @@ -359,6 +363,8 @@ describe('system.sessionAnalytics handler', () => { const getSessionAnalytics = vi.fn(() => ({ daily: [{ day: '2026-02-16', sessions: 2, messages: 8 }], topSessions: [{ sessionId: 'telegram:1', messages: 5, lastActivity: 1708080000 }], + topTools: [{ toolName: 'web.search', executions: 4 }], + topTopics: [{ topic: 'kubernetes', occurrences: 3 }], averageMessagesPerSession: 4, totalSessions: 2, totalMessages: 8, @@ -383,6 +389,8 @@ describe('system.sessionAnalytics handler', () => { expect(getSessionAnalytics).toHaveBeenCalledWith({ days: 7, topLimit: 5 }); expect(getPath(result.result, 'totalSessions')).toBe(2); expect(getPath(result.result, 'daily')).toEqual([{ day: '2026-02-16', sessions: 2, messages: 8 }]); + expect(getPath(result.result, 'topTools')).toEqual([{ toolName: 'web.search', executions: 4 }]); + expect(getPath(result.result, 'topTopics')).toEqual([{ topic: 'kubernetes', occurrences: 3 }]); }); }); diff --git a/src/gateway/handlers/system.ts b/src/gateway/handlers/system.ts index b635370..5e88cd2 100644 --- a/src/gateway/handlers/system.ts +++ b/src/gateway/handlers/system.ts @@ -214,6 +214,8 @@ export function createSystemHandlers(deps: SystemHandlerDeps) { return makeResponse(request.id, { daily: [], topSessions: [], + topTools: [], + topTopics: [], averageMessagesPerSession: 0, totalSessions: 0, totalMessages: 0, diff --git a/src/session/index.ts b/src/session/index.ts index 5e34edd..364844f 100644 --- a/src/session/index.ts +++ b/src/session/index.ts @@ -1,5 +1,11 @@ export { SessionStore, parseDuration } from './store.js'; -export type { SessionAnalyticsSnapshot, SessionDailyAnalyticsRow, SessionTopAnalyticsRow } from './store.js'; +export type { + SessionAnalyticsSnapshot, + SessionDailyAnalyticsRow, + SessionTopAnalyticsRow, + SessionTopToolAnalyticsRow, + SessionTopTopicAnalyticsRow, +} from './store.js'; export { SessionManager, ManagedSession, type Session } from './manager.js'; export { SessionIndexer, tokenize } from './indexer.js'; export type { HistoryMetadata, HistoryIndexerConfig } from './indexer.js'; diff --git a/src/session/store.test.ts b/src/session/store.test.ts index 44c247d..2df0f65 100644 --- a/src/session/store.test.ts +++ b/src/session/store.test.ts @@ -79,10 +79,13 @@ describe('SessionStore', () => { const nowSec = Math.floor(nowMs / 1000); const yesterdaySec = nowSec - 86_400; - store.addMessage('session-a', { role: 'user', content: 'A1', timestamp: nowMs }); - store.addMessage('session-a', { role: 'assistant', content: 'A2', timestamp: nowMs + 1000 }); - store.addMessage('session-b', { role: 'user', content: 'B1', timestamp: nowMs }); - store.addMessage('session-c', { role: 'user', content: 'C1', timestamp: yesterdaySec * 1000 }); + store.addMessage('session-a', { role: 'user', content: 'A1', timestamp: nowMs }, { keywords: ['kubernetes'], topics: ['kubernetes'] }); + store.addMessage('session-a', { role: 'assistant', content: 'A2', timestamp: nowMs + 1000 }, { keywords: ['deploy'], topics: ['deploy'] }); + store.addMessage('session-b', { role: 'user', content: 'B1', timestamp: nowMs }, { keywords: ['kubernetes'], topics: ['kubernetes'] }); + store.addMessage('session-c', { role: 'user', content: 'C1', timestamp: yesterdaySec * 1000 }, { keywords: ['incident'], topics: ['incident'] }); + store.recordToolExecution('session-a', 'web.search', true, nowSec); + store.recordToolExecution('session-a', 'web.search', true, nowSec); + store.recordToolExecution('session-b', 'file.read', true, nowSec); const snapshot = store.getSessionAnalytics({ sinceTimestamp: yesterdaySec, @@ -94,6 +97,14 @@ describe('SessionStore', () => { expect(snapshot.averageMessagesPerSession).toBeCloseTo(1.33, 2); expect(snapshot.topSessions).toHaveLength(2); expect(snapshot.topSessions[0]?.sessionId).toBe('session-a'); + expect(snapshot.topTools).toEqual([ + { toolName: 'web.search', executions: 2 }, + { toolName: 'file.read', executions: 1 }, + ]); + expect(snapshot.topTopics).toEqual([ + { topic: 'kubernetes', occurrences: 2 }, + { topic: 'deploy', occurrences: 1 }, + ]); expect(snapshot.daily.length).toBeGreaterThan(0); }); diff --git a/src/session/store.ts b/src/session/store.ts index 47eeeca..0f1c9b6 100644 --- a/src/session/store.ts +++ b/src/session/store.ts @@ -24,9 +24,21 @@ export interface SessionTopAnalyticsRow { lastActivity: number; } +export interface SessionTopToolAnalyticsRow { + toolName: string; + executions: number; +} + +export interface SessionTopTopicAnalyticsRow { + topic: string; + occurrences: number; +} + export interface SessionAnalyticsSnapshot { daily: SessionDailyAnalyticsRow[]; topSessions: SessionTopAnalyticsRow[]; + topTools: SessionTopToolAnalyticsRow[]; + topTopics: SessionTopTopicAnalyticsRow[]; averageMessagesPerSession: number; totalSessions: number; totalMessages: number; @@ -64,6 +76,15 @@ export class SessionStore { PRIMARY KEY (session_id, key) ); CREATE INDEX IF NOT EXISTS idx_session_config_session ON session_config(session_id); + CREATE TABLE IF NOT EXISTS tool_executions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + tool_name TEXT NOT NULL, + created_at INTEGER NOT NULL DEFAULT (unixepoch()), + success INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_tool_executions_session ON tool_executions(session_id); + CREATE INDEX IF NOT EXISTS idx_tool_executions_created_at ON tool_executions(created_at); `); const messageColumns = this.db.prepare('PRAGMA table_info(messages)').all() as Array<{ name: string }>; @@ -117,6 +138,7 @@ export class SessionStore { const transaction = this.db.transaction(() => { this.db.prepare('DELETE FROM messages WHERE session_id = ?').run(sessionId); this.db.prepare('DELETE FROM session_config WHERE session_id = ?').run(sessionId); + this.db.prepare('DELETE FROM tool_executions WHERE session_id = ?').run(sessionId); }); transaction(); } @@ -139,10 +161,12 @@ export class SessionStore { const deleteMessages = this.db.prepare('DELETE FROM messages WHERE session_id = ?'); const deleteConfig = this.db.prepare('DELETE FROM session_config WHERE session_id = ?'); + const deleteToolExecutions = this.db.prepare('DELETE FROM tool_executions WHERE session_id = ?'); const transaction = this.db.transaction(() => { for (const { session_id } of stale) { deleteMessages.run(session_id); deleteConfig.run(session_id); + deleteToolExecutions.run(session_id); } }); transaction(); @@ -288,6 +312,12 @@ export class SessionStore { this.db.prepare('UPDATE messages SET metadata = ? WHERE id = ?').run(JSON.stringify(metadata), messageId); } + recordToolExecution(sessionId: string, toolName: string, success: boolean, createdAtSeconds?: number): void { + this.db.prepare( + 'INSERT INTO tool_executions (session_id, tool_name, created_at, success) VALUES (?, ?, ?, ?)', + ).run(sessionId, toolName, createdAtSeconds ?? Math.floor(Date.now() / 1000), success ? 1 : 0); + } + getSessionAnalytics(opts: { sinceTimestamp: number; topLimit?: number }): SessionAnalyticsSnapshot { const since = opts.sinceTimestamp; const topLimit = opts.topLimit ?? 10; @@ -315,6 +345,17 @@ export class SessionStore { LIMIT ? `).all(since, topLimit) as Array<{ session_id: string; messages: number; last_activity: number }>; + const topToolsRows = this.db.prepare(` + SELECT + tool_name, + COUNT(*) AS executions + FROM tool_executions + WHERE created_at >= ? + GROUP BY tool_name + ORDER BY executions DESC, tool_name ASC + LIMIT ? + `).all(since, topLimit) as Array<{ tool_name: string; executions: number }>; + const totalMessagesRow = this.db.prepare(` SELECT COUNT(*) AS total_messages FROM messages @@ -333,6 +374,19 @@ export class SessionStore { ? Math.round((totalMessages / totalSessions) * 100) / 100 : 0; + const topicCounts = new Map(); + const rowsForTopics = this.getAllMessagesWithMetadata().filter((row) => row.createdAt >= since); + for (const row of rowsForTopics) { + const topics = row.metadata?.topics ?? row.metadata?.keywords ?? []; + for (const topic of topics) { + topicCounts.set(topic, (topicCounts.get(topic) ?? 0) + 1); + } + } + const topTopics = Array.from(topicCounts.entries()) + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .slice(0, topLimit) + .map(([topic, occurrences]) => ({ topic, occurrences })); + return { daily: dailyRows.map((row) => ({ day: row.day, @@ -344,6 +398,11 @@ export class SessionStore { messages: row.messages, lastActivity: row.last_activity, })), + topTools: topToolsRows.map((row) => ({ + toolName: row.tool_name, + executions: row.executions, + })), + topTopics, averageMessagesPerSession, totalSessions, totalMessages, diff --git a/src/tools/executor.test.ts b/src/tools/executor.test.ts index 3595985..9bb8fa0 100644 --- a/src/tools/executor.test.ts +++ b/src/tools/executor.test.ts @@ -96,6 +96,31 @@ describe('ToolExecutor', () => { expect(result.output).toBe('hello'); }); + it('notifies execution observer on success and failure', async () => { + const registry = new ToolRegistry(); + registry.register(echoTool); + registry.register(failTool); + const hooks = new HookEngine({ confirm: [], log: [], silent: [] }); + const executor = new ToolExecutor(registry, hooks); + const observer = vi.fn(); + executor.setExecutionObserver(observer); + + await executor.execute('test.echo', { text: 'hello' }, { sessionId: 'ws:1' }); + await executor.execute('test.fail', {}, { sessionId: 'ws:1' }); + + expect(observer).toHaveBeenCalledTimes(2); + expect(observer.mock.calls[0]?.[0]).toMatchObject({ + toolName: 'test.echo', + sessionId: 'ws:1', + success: true, + }); + expect(observer.mock.calls[1]?.[0]).toMatchObject({ + toolName: 'test.fail', + sessionId: 'ws:1', + success: false, + }); + }); + it('returns error for unknown tool', async () => { const registry = new ToolRegistry(); const hooks = new HookEngine({ confirm: [], log: [], silent: [] }); diff --git a/src/tools/executor.ts b/src/tools/executor.ts index 78fa2ee..5fd8b96 100644 --- a/src/tools/executor.ts +++ b/src/tools/executor.ts @@ -15,12 +15,20 @@ export interface ToolExecutorConfig { maxOutputBytes?: number; } +export interface ToolExecutionObserverEvent { + toolName: string; + sessionId?: string; + success: boolean; + timestampSeconds: number; +} + export class ToolExecutor { private registry: ToolRegistry; private hooks: HookEngine; private defaultTimeoutMs: number; private maxOutputBytes: number; private sandboxManager?: SandboxManager; + private executionObserver?: (event: ToolExecutionObserverEvent) => void; constructor(registry: ToolRegistry, hooks: HookEngine, config?: ToolExecutorConfig) { this.registry = registry; @@ -33,6 +41,10 @@ export class ToolExecutor { this.sandboxManager = manager; } + setExecutionObserver(observer?: (event: ToolExecutionObserverEvent) => void): void { + this.executionObserver = observer; + } + private isElevationActive(context?: ToolPolicyContext): boolean { const untilMs = context?.elevatedHostUntilMs; return typeof untilMs === 'number' && Number.isFinite(untilMs) && untilMs > Date.now(); @@ -271,6 +283,13 @@ export class ToolExecutor { session_id: context?.sessionId, }); + this.notifyExecutionObserver({ + toolName, + sessionId: context?.sessionId, + success: result.success, + timestampSeconds: Math.floor(Date.now() / 1000), + }); + return result; } catch (error) { const duration = Date.now() - startTime; @@ -289,6 +308,13 @@ export class ToolExecutor { redactions_applied: argsRedaction.redactions + errorRedaction.redactions, }); + this.notifyExecutionObserver({ + toolName, + sessionId: context?.sessionId, + success: false, + timestampSeconds: Math.floor(Date.now() / 1000), + }); + return { success: false, output: '', @@ -301,6 +327,20 @@ export class ToolExecutor { } } + private notifyExecutionObserver(event: ToolExecutionObserverEvent): void { + if (!this.executionObserver) { + return; + } + try { + this.executionObserver(event); + } catch (error) { + console.warn( + 'ToolExecutor: execution observer failed:', + error instanceof Error ? error.message : String(error), + ); + } + } + private resolveAllowedSecretScopes(context?: ToolPolicyContext): string[] { if (context?.allowedSecretScopes) { return context.allowedSecretScopes;