feat(analytics): add top tools and topics to session analytics
This commit is contained in:
@@ -107,6 +107,12 @@ export async function startDaemon(config: Config, options?: StartDaemonOptions):
|
||||
// ── Core Services ──
|
||||
const hookEngine = new HookEngine(config.hooks);
|
||||
const { toolRegistry, toolExecutor, browserManager } = initTools({ config, lifecycle, hookEngine });
|
||||
toolExecutor.setExecutionObserver((event) => {
|
||||
if (!event.sessionId) {
|
||||
return;
|
||||
}
|
||||
sessionStore.recordToolExecution(event.sessionId, event.toolName, event.success, event.timestampSeconds);
|
||||
});
|
||||
const { memoryStore, memoryDir } = await initMemory({ config, dataDir, lifecycle, toolRegistry });
|
||||
const mcpManager = await initMcp(config, lifecycle, toolRegistry);
|
||||
const { skillRegistry, skillInstaller } = initSkills(config, lifecycle);
|
||||
|
||||
@@ -344,12 +344,16 @@ describe('system.sessionAnalytics handler', () => {
|
||||
const r = result.result as {
|
||||
daily: unknown[];
|
||||
topSessions: unknown[];
|
||||
topTools: unknown[];
|
||||
topTopics: unknown[];
|
||||
averageMessagesPerSession: number;
|
||||
totalSessions: number;
|
||||
totalMessages: number;
|
||||
};
|
||||
expect(r.daily).toEqual([]);
|
||||
expect(r.topSessions).toEqual([]);
|
||||
expect(r.topTools).toEqual([]);
|
||||
expect(r.topTopics).toEqual([]);
|
||||
expect(r.averageMessagesPerSession).toBe(0);
|
||||
expect(r.totalSessions).toBe(0);
|
||||
expect(r.totalMessages).toBe(0);
|
||||
@@ -359,6 +363,8 @@ describe('system.sessionAnalytics handler', () => {
|
||||
const getSessionAnalytics = vi.fn(() => ({
|
||||
daily: [{ day: '2026-02-16', sessions: 2, messages: 8 }],
|
||||
topSessions: [{ sessionId: 'telegram:1', messages: 5, lastActivity: 1708080000 }],
|
||||
topTools: [{ toolName: 'web.search', executions: 4 }],
|
||||
topTopics: [{ topic: 'kubernetes', occurrences: 3 }],
|
||||
averageMessagesPerSession: 4,
|
||||
totalSessions: 2,
|
||||
totalMessages: 8,
|
||||
@@ -383,6 +389,8 @@ describe('system.sessionAnalytics handler', () => {
|
||||
expect(getSessionAnalytics).toHaveBeenCalledWith({ days: 7, topLimit: 5 });
|
||||
expect(getPath(result.result, 'totalSessions')).toBe(2);
|
||||
expect(getPath(result.result, 'daily')).toEqual([{ day: '2026-02-16', sessions: 2, messages: 8 }]);
|
||||
expect(getPath(result.result, 'topTools')).toEqual([{ toolName: 'web.search', executions: 4 }]);
|
||||
expect(getPath(result.result, 'topTopics')).toEqual([{ topic: 'kubernetes', occurrences: 3 }]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -214,6 +214,8 @@ export function createSystemHandlers(deps: SystemHandlerDeps) {
|
||||
return makeResponse(request.id, {
|
||||
daily: [],
|
||||
topSessions: [],
|
||||
topTools: [],
|
||||
topTopics: [],
|
||||
averageMessagesPerSession: 0,
|
||||
totalSessions: 0,
|
||||
totalMessages: 0,
|
||||
|
||||
@@ -1,5 +1,11 @@
|
||||
export { SessionStore, parseDuration } from './store.js';
|
||||
export type { SessionAnalyticsSnapshot, SessionDailyAnalyticsRow, SessionTopAnalyticsRow } from './store.js';
|
||||
export type {
|
||||
SessionAnalyticsSnapshot,
|
||||
SessionDailyAnalyticsRow,
|
||||
SessionTopAnalyticsRow,
|
||||
SessionTopToolAnalyticsRow,
|
||||
SessionTopTopicAnalyticsRow,
|
||||
} from './store.js';
|
||||
export { SessionManager, ManagedSession, type Session } from './manager.js';
|
||||
export { SessionIndexer, tokenize } from './indexer.js';
|
||||
export type { HistoryMetadata, HistoryIndexerConfig } from './indexer.js';
|
||||
|
||||
@@ -79,10 +79,13 @@ describe('SessionStore', () => {
|
||||
const nowSec = Math.floor(nowMs / 1000);
|
||||
const yesterdaySec = nowSec - 86_400;
|
||||
|
||||
store.addMessage('session-a', { role: 'user', content: 'A1', timestamp: nowMs });
|
||||
store.addMessage('session-a', { role: 'assistant', content: 'A2', timestamp: nowMs + 1000 });
|
||||
store.addMessage('session-b', { role: 'user', content: 'B1', timestamp: nowMs });
|
||||
store.addMessage('session-c', { role: 'user', content: 'C1', timestamp: yesterdaySec * 1000 });
|
||||
store.addMessage('session-a', { role: 'user', content: 'A1', timestamp: nowMs }, { keywords: ['kubernetes'], topics: ['kubernetes'] });
|
||||
store.addMessage('session-a', { role: 'assistant', content: 'A2', timestamp: nowMs + 1000 }, { keywords: ['deploy'], topics: ['deploy'] });
|
||||
store.addMessage('session-b', { role: 'user', content: 'B1', timestamp: nowMs }, { keywords: ['kubernetes'], topics: ['kubernetes'] });
|
||||
store.addMessage('session-c', { role: 'user', content: 'C1', timestamp: yesterdaySec * 1000 }, { keywords: ['incident'], topics: ['incident'] });
|
||||
store.recordToolExecution('session-a', 'web.search', true, nowSec);
|
||||
store.recordToolExecution('session-a', 'web.search', true, nowSec);
|
||||
store.recordToolExecution('session-b', 'file.read', true, nowSec);
|
||||
|
||||
const snapshot = store.getSessionAnalytics({
|
||||
sinceTimestamp: yesterdaySec,
|
||||
@@ -94,6 +97,14 @@ describe('SessionStore', () => {
|
||||
expect(snapshot.averageMessagesPerSession).toBeCloseTo(1.33, 2);
|
||||
expect(snapshot.topSessions).toHaveLength(2);
|
||||
expect(snapshot.topSessions[0]?.sessionId).toBe('session-a');
|
||||
expect(snapshot.topTools).toEqual([
|
||||
{ toolName: 'web.search', executions: 2 },
|
||||
{ toolName: 'file.read', executions: 1 },
|
||||
]);
|
||||
expect(snapshot.topTopics).toEqual([
|
||||
{ topic: 'kubernetes', occurrences: 2 },
|
||||
{ topic: 'deploy', occurrences: 1 },
|
||||
]);
|
||||
expect(snapshot.daily.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
|
||||
@@ -24,9 +24,21 @@ export interface SessionTopAnalyticsRow {
|
||||
lastActivity: number;
|
||||
}
|
||||
|
||||
export interface SessionTopToolAnalyticsRow {
|
||||
toolName: string;
|
||||
executions: number;
|
||||
}
|
||||
|
||||
export interface SessionTopTopicAnalyticsRow {
|
||||
topic: string;
|
||||
occurrences: number;
|
||||
}
|
||||
|
||||
export interface SessionAnalyticsSnapshot {
|
||||
daily: SessionDailyAnalyticsRow[];
|
||||
topSessions: SessionTopAnalyticsRow[];
|
||||
topTools: SessionTopToolAnalyticsRow[];
|
||||
topTopics: SessionTopTopicAnalyticsRow[];
|
||||
averageMessagesPerSession: number;
|
||||
totalSessions: number;
|
||||
totalMessages: number;
|
||||
@@ -64,6 +76,15 @@ export class SessionStore {
|
||||
PRIMARY KEY (session_id, key)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_session_config_session ON session_config(session_id);
|
||||
CREATE TABLE IF NOT EXISTS tool_executions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT NOT NULL,
|
||||
tool_name TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL DEFAULT (unixepoch()),
|
||||
success INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_tool_executions_session ON tool_executions(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_tool_executions_created_at ON tool_executions(created_at);
|
||||
`);
|
||||
|
||||
const messageColumns = this.db.prepare('PRAGMA table_info(messages)').all() as Array<{ name: string }>;
|
||||
@@ -117,6 +138,7 @@ export class SessionStore {
|
||||
const transaction = this.db.transaction(() => {
|
||||
this.db.prepare('DELETE FROM messages WHERE session_id = ?').run(sessionId);
|
||||
this.db.prepare('DELETE FROM session_config WHERE session_id = ?').run(sessionId);
|
||||
this.db.prepare('DELETE FROM tool_executions WHERE session_id = ?').run(sessionId);
|
||||
});
|
||||
transaction();
|
||||
}
|
||||
@@ -139,10 +161,12 @@ export class SessionStore {
|
||||
|
||||
const deleteMessages = this.db.prepare('DELETE FROM messages WHERE session_id = ?');
|
||||
const deleteConfig = this.db.prepare('DELETE FROM session_config WHERE session_id = ?');
|
||||
const deleteToolExecutions = this.db.prepare('DELETE FROM tool_executions WHERE session_id = ?');
|
||||
const transaction = this.db.transaction(() => {
|
||||
for (const { session_id } of stale) {
|
||||
deleteMessages.run(session_id);
|
||||
deleteConfig.run(session_id);
|
||||
deleteToolExecutions.run(session_id);
|
||||
}
|
||||
});
|
||||
transaction();
|
||||
@@ -288,6 +312,12 @@ export class SessionStore {
|
||||
this.db.prepare('UPDATE messages SET metadata = ? WHERE id = ?').run(JSON.stringify(metadata), messageId);
|
||||
}
|
||||
|
||||
recordToolExecution(sessionId: string, toolName: string, success: boolean, createdAtSeconds?: number): void {
|
||||
this.db.prepare(
|
||||
'INSERT INTO tool_executions (session_id, tool_name, created_at, success) VALUES (?, ?, ?, ?)',
|
||||
).run(sessionId, toolName, createdAtSeconds ?? Math.floor(Date.now() / 1000), success ? 1 : 0);
|
||||
}
|
||||
|
||||
getSessionAnalytics(opts: { sinceTimestamp: number; topLimit?: number }): SessionAnalyticsSnapshot {
|
||||
const since = opts.sinceTimestamp;
|
||||
const topLimit = opts.topLimit ?? 10;
|
||||
@@ -315,6 +345,17 @@ export class SessionStore {
|
||||
LIMIT ?
|
||||
`).all(since, topLimit) as Array<{ session_id: string; messages: number; last_activity: number }>;
|
||||
|
||||
const topToolsRows = this.db.prepare(`
|
||||
SELECT
|
||||
tool_name,
|
||||
COUNT(*) AS executions
|
||||
FROM tool_executions
|
||||
WHERE created_at >= ?
|
||||
GROUP BY tool_name
|
||||
ORDER BY executions DESC, tool_name ASC
|
||||
LIMIT ?
|
||||
`).all(since, topLimit) as Array<{ tool_name: string; executions: number }>;
|
||||
|
||||
const totalMessagesRow = this.db.prepare(`
|
||||
SELECT COUNT(*) AS total_messages
|
||||
FROM messages
|
||||
@@ -333,6 +374,19 @@ export class SessionStore {
|
||||
? Math.round((totalMessages / totalSessions) * 100) / 100
|
||||
: 0;
|
||||
|
||||
const topicCounts = new Map<string, number>();
|
||||
const rowsForTopics = this.getAllMessagesWithMetadata().filter((row) => row.createdAt >= since);
|
||||
for (const row of rowsForTopics) {
|
||||
const topics = row.metadata?.topics ?? row.metadata?.keywords ?? [];
|
||||
for (const topic of topics) {
|
||||
topicCounts.set(topic, (topicCounts.get(topic) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
const topTopics = Array.from(topicCounts.entries())
|
||||
.sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
||||
.slice(0, topLimit)
|
||||
.map(([topic, occurrences]) => ({ topic, occurrences }));
|
||||
|
||||
return {
|
||||
daily: dailyRows.map((row) => ({
|
||||
day: row.day,
|
||||
@@ -344,6 +398,11 @@ export class SessionStore {
|
||||
messages: row.messages,
|
||||
lastActivity: row.last_activity,
|
||||
})),
|
||||
topTools: topToolsRows.map((row) => ({
|
||||
toolName: row.tool_name,
|
||||
executions: row.executions,
|
||||
})),
|
||||
topTopics,
|
||||
averageMessagesPerSession,
|
||||
totalSessions,
|
||||
totalMessages,
|
||||
|
||||
@@ -96,6 +96,31 @@ describe('ToolExecutor', () => {
|
||||
expect(result.output).toBe('hello');
|
||||
});
|
||||
|
||||
it('notifies execution observer on success and failure', async () => {
|
||||
const registry = new ToolRegistry();
|
||||
registry.register(echoTool);
|
||||
registry.register(failTool);
|
||||
const hooks = new HookEngine({ confirm: [], log: [], silent: [] });
|
||||
const executor = new ToolExecutor(registry, hooks);
|
||||
const observer = vi.fn();
|
||||
executor.setExecutionObserver(observer);
|
||||
|
||||
await executor.execute('test.echo', { text: 'hello' }, { sessionId: 'ws:1' });
|
||||
await executor.execute('test.fail', {}, { sessionId: 'ws:1' });
|
||||
|
||||
expect(observer).toHaveBeenCalledTimes(2);
|
||||
expect(observer.mock.calls[0]?.[0]).toMatchObject({
|
||||
toolName: 'test.echo',
|
||||
sessionId: 'ws:1',
|
||||
success: true,
|
||||
});
|
||||
expect(observer.mock.calls[1]?.[0]).toMatchObject({
|
||||
toolName: 'test.fail',
|
||||
sessionId: 'ws:1',
|
||||
success: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns error for unknown tool', async () => {
|
||||
const registry = new ToolRegistry();
|
||||
const hooks = new HookEngine({ confirm: [], log: [], silent: [] });
|
||||
|
||||
@@ -15,12 +15,20 @@ export interface ToolExecutorConfig {
|
||||
maxOutputBytes?: number;
|
||||
}
|
||||
|
||||
export interface ToolExecutionObserverEvent {
|
||||
toolName: string;
|
||||
sessionId?: string;
|
||||
success: boolean;
|
||||
timestampSeconds: number;
|
||||
}
|
||||
|
||||
export class ToolExecutor {
|
||||
private registry: ToolRegistry;
|
||||
private hooks: HookEngine;
|
||||
private defaultTimeoutMs: number;
|
||||
private maxOutputBytes: number;
|
||||
private sandboxManager?: SandboxManager;
|
||||
private executionObserver?: (event: ToolExecutionObserverEvent) => void;
|
||||
|
||||
constructor(registry: ToolRegistry, hooks: HookEngine, config?: ToolExecutorConfig) {
|
||||
this.registry = registry;
|
||||
@@ -33,6 +41,10 @@ export class ToolExecutor {
|
||||
this.sandboxManager = manager;
|
||||
}
|
||||
|
||||
setExecutionObserver(observer?: (event: ToolExecutionObserverEvent) => void): void {
|
||||
this.executionObserver = observer;
|
||||
}
|
||||
|
||||
private isElevationActive(context?: ToolPolicyContext): boolean {
|
||||
const untilMs = context?.elevatedHostUntilMs;
|
||||
return typeof untilMs === 'number' && Number.isFinite(untilMs) && untilMs > Date.now();
|
||||
@@ -271,6 +283,13 @@ export class ToolExecutor {
|
||||
session_id: context?.sessionId,
|
||||
});
|
||||
|
||||
this.notifyExecutionObserver({
|
||||
toolName,
|
||||
sessionId: context?.sessionId,
|
||||
success: result.success,
|
||||
timestampSeconds: Math.floor(Date.now() / 1000),
|
||||
});
|
||||
|
||||
return result;
|
||||
} catch (error) {
|
||||
const duration = Date.now() - startTime;
|
||||
@@ -289,6 +308,13 @@ export class ToolExecutor {
|
||||
redactions_applied: argsRedaction.redactions + errorRedaction.redactions,
|
||||
});
|
||||
|
||||
this.notifyExecutionObserver({
|
||||
toolName,
|
||||
sessionId: context?.sessionId,
|
||||
success: false,
|
||||
timestampSeconds: Math.floor(Date.now() / 1000),
|
||||
});
|
||||
|
||||
return {
|
||||
success: false,
|
||||
output: '',
|
||||
@@ -301,6 +327,20 @@ export class ToolExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
private notifyExecutionObserver(event: ToolExecutionObserverEvent): void {
|
||||
if (!this.executionObserver) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.executionObserver(event);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'ToolExecutor: execution observer failed:',
|
||||
error instanceof Error ? error.message : String(error),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private resolveAllowedSecretScopes(context?: ToolPolicyContext): string[] {
|
||||
if (context?.allowedSecretScopes) {
|
||||
return context.allowedSecretScopes;
|
||||
|
||||
Reference in New Issue
Block a user