feat(hooks): add Hermes telemetry handler

2026-05-20 17:35:56 -07:00
parent 78376bdd83
commit 27d40ce28f
9 changed files with 2452 additions and 0 deletions
@@ -0,0 +1,386 @@
+#!/usr/bin/env node
+import { randomUUID } from 'node:crypto';
+import { mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
+import { homedir, hostname } from 'node:os';
+import { join } from 'node:path';
+import {
+  type Dict,
+  isRecord,
+  pickString,
+  pickNumber,
+  truncate,
+  buildEnvelope,
+  createTransport,
+  readStdin,
+} from '../shared/lib';
+
+const INGEST_URL = process.env.AGENTMON_INGEST_URL || 'http://localhost:8080';
+const FRAMEWORK = process.env.AGENTMON_FRAMEWORK || 'hermes';
+const HOST = process.env.AGENTMON_HOST || hostname();
+
+const { enqueue, flush } = createTransport(INGEST_URL);
+
+interface SessionState {
+  sessionStarted?: boolean;
+  runId?: string;
+  spans: { [key: string]: string };
+}
+
+const STATE_DIR = join(homedir(), '.agentmon-state', 'hermes');
+
+function ensureStateDir() {
+  try {
+    mkdirSync(STATE_DIR, { recursive: true });
+  } catch {
+    // Hooks should never fail the agent because telemetry state is unavailable.
+  }
+}
+
+function loadState(sessionKey: string): SessionState {
+  try {
+    const raw = readFileSync(join(STATE_DIR, sessionKey + '.json'), 'utf8');
+    const state = JSON.parse(raw) as SessionState;
+    return { spans: {}, ...state };
+  } catch {
+    return { spans: {} };
+  }
+}
+
+function saveState(sessionKey: string | undefined, state: SessionState) {
+  if (!sessionKey) {
+    return;
+  }
+  ensureStateDir();
+  try {
+    writeFileSync(join(STATE_DIR, sessionKey + '.json'), JSON.stringify(state), 'utf8');
+  } catch {
+    // Best effort only.
+  }
+}
+
+function clearState(sessionKey: string | undefined) {
+  if (!sessionKey) {
+    return;
+  }
+  try {
+    unlinkSync(join(STATE_DIR, sessionKey + '.json'));
+  } catch {
+    // Best effort only.
+  }
+}
+
+function getExtra(input: Dict): Dict {
+  return isRecord(input.extra) ? input.extra : {};
+}
+
+function getSessionKey(input: Dict): string | undefined {
+  const extra = getExtra(input);
+  return pickString(
+    input.session_id,
+    input.sessionId,
+    input.sessionID,
+    input.session,
+    extra.session_id,
+    extra.sessionId,
+    extra.sessionID,
+    extra.parent_session_id,
+    extra.task_id,
+  );
+}
+
+function getToolCallId(input: Dict): string | undefined {
+  const extra = getExtra(input);
+  return pickString(input.tool_call_id, extra.tool_call_id, extra.call_id, extra.id);
+}
+
+function getToolName(input: Dict): string {
+  const extra = getExtra(input);
+  return pickString(input.tool_name, input.tool, input.name, extra.tool_name, extra.tool, extra.name) || 'unknown';
+}
+
+function getUsage(input: Dict): Dict | undefined {
+  const extra = getExtra(input);
+  const usage = isRecord(input.usage) ? input.usage :
+                isRecord(extra.usage) ? extra.usage :
+                undefined;
+  if (!usage) return undefined;
+
+  const result: Dict = {};
+  if (usage.input_tokens !== undefined) result.input_tokens = usage.input_tokens;
+  if (usage.prompt_tokens !== undefined) result.input_tokens = usage.prompt_tokens;
+  if (usage.output_tokens !== undefined) result.output_tokens = usage.output_tokens;
+  if (usage.completion_tokens !== undefined) result.output_tokens = usage.completion_tokens;
+  if (usage.cache_read_tokens !== undefined) result.cache_read_tokens = usage.cache_read_tokens;
+  if (usage.cache_write_tokens !== undefined) result.cache_write_tokens = usage.cache_write_tokens;
+  if (usage.cache_creation_tokens !== undefined) result.cache_write_tokens = usage.cache_creation_tokens;
+  if (usage.reasoning_tokens !== undefined) result.reasoning_tokens = usage.reasoning_tokens;
+  if (usage.total_tokens !== undefined) result.total_tokens = usage.total_tokens;
+  if (usage.total_cost !== undefined) result.total_cost = usage.total_cost;
+  if (usage.cost !== undefined) result.total_cost = usage.cost;
+
+  return Object.keys(result).length > 0 ? result : undefined;
+}
+
+function getModel(input: Dict): string | undefined {
+  const extra = getExtra(input);
+  return pickString(input.model, extra.model, extra.response_model);
+}
+
+function getPlatform(input: Dict): string | undefined {
+  const extra = getExtra(input);
+  return pickString(input.platform, extra.platform);
+}
+
+function getDuration(input: Dict): number | undefined {
+  const extra = getExtra(input);
+  return pickNumber(input.duration_ms, input.elapsed_ms, extra.duration_ms, extra.api_duration, extra.elapsed_ms);
+}
+
+function ensureSessionStarted(sessionKey: string | undefined, input: Dict, state: SessionState) {
+  if (!sessionKey || state.sessionStarted) {
+    return;
+  }
+
+  enqueue(buildEnvelope(FRAMEWORK, HOST, 'session.start', sessionKey, {
+    attributes: {
+      synthetic: true,
+      recovered_from: 'hermes-hook',
+      cwd: pickString(input.cwd),
+      platform: getPlatform(input),
+      model: getModel(input),
+    },
+  }));
+  state.sessionStarted = true;
+  saveState(sessionKey, state);
+}
+
+async function handleSessionStart(input: Dict) {
+  const sessionKey = getSessionKey(input) || randomUUID();
+  const state = loadState(sessionKey);
+  if (!state.sessionStarted) {
+    enqueue(buildEnvelope(FRAMEWORK, HOST, 'session.start', sessionKey, {
+      attributes: {
+        cwd: pickString(input.cwd),
+        platform: getPlatform(input),
+        model: getModel(input),
+      },
+    }));
+    state.sessionStarted = true;
+    saveState(sessionKey, state);
+  }
+  await flush();
+}
+
+async function handleRunStart(input: Dict) {
+  const sessionKey = getSessionKey(input) || randomUUID();
+  const state = loadState(sessionKey);
+  ensureSessionStarted(sessionKey, input, state);
+
+  if (state.runId) {
+    enqueue(buildEnvelope(FRAMEWORK, HOST, 'run.end', sessionKey, {
+      runId: state.runId,
+      payload: { status: 'success' },
+    }));
+  }
+
+  const runId = randomUUID();
+  state.runId = runId;
+  saveState(sessionKey, state);
+
+  const extra = getExtra(input);
+  enqueue(buildEnvelope(FRAMEWORK, HOST, 'run.start', sessionKey, {
+    runId,
+    attributes: {
+      is_first_turn: extra.is_first_turn,
+      platform: getPlatform(input),
+      model: getModel(input),
+    },
+    payload: {
+      prompt_preview: truncate(extra.user_message ?? input.user_message, 200),
+    },
+  }));
+  await flush();
+}
+
+async function handleRunEnd(input: Dict) {
+  const sessionKey = getSessionKey(input);
+  if (!sessionKey) {
+    await flush();
+    return;
+  }
+
+  const state = loadState(sessionKey);
+  ensureSessionStarted(sessionKey, input, state);
+
+  if (state.runId) {
+    enqueue(buildEnvelope(FRAMEWORK, HOST, 'run.end', sessionKey, {
+      runId: state.runId,
+      payload: {
+        status: 'success',
+        duration_ms: getDuration(input),
+        model: getModel(input),
+        response_preview: truncate(getExtra(input).assistant_response, 500),
+      },
+    }));
+    state.runId = undefined;
+    saveState(sessionKey, state);
+  }
+  await flush();
+}
+
+async function handleToolStart(input: Dict) {
+  const sessionKey = getSessionKey(input);
+  const state = sessionKey ? loadState(sessionKey) : { spans: {} };
+  ensureSessionStarted(sessionKey, input, state);
+
+  const toolName = getToolName(input);
+  const toolCallId = getToolCallId(input);
+  const spanKey = toolCallId || toolName;
+  const spanId = randomUUID();
+  state.spans[spanKey] = spanId;
+  saveState(sessionKey, state);
+
+  enqueue(buildEnvelope(FRAMEWORK, HOST, 'span.start', sessionKey, {
+    runId: state.runId,
+    spanId,
+    attributes: {
+      span_kind: 'tool',
+      name: toolName,
+      tool_call_id: toolCallId,
+    },
+    payload: {
+      input: truncate(input.tool_input ?? getExtra(input).args, 500),
+    },
+  }));
+  await flush();
+}
+
+async function handleToolEnd(input: Dict) {
+  const sessionKey = getSessionKey(input);
+  const state = sessionKey ? loadState(sessionKey) : { spans: {} };
+  const toolName = getToolName(input);
+  const toolCallId = getToolCallId(input);
+  const spanKey = toolCallId || toolName;
+  const spanId = state.spans[spanKey];
+  const extra = getExtra(input);
+  const result = extra.result ?? input.result;
+  const resultRecord = isRecord(result) ? result : {};
+  const success = extra.error === undefined && resultRecord.error === undefined;
+
+  enqueue(buildEnvelope(FRAMEWORK, HOST, 'span.end', sessionKey, {
+    runId: state.runId,
+    spanId,
+    attributes: {
+      span_kind: 'tool',
+      name: toolName,
+      tool_call_id: toolCallId,
+    },
+    payload: {
+      status: success ? 'success' : 'error',
+      result_preview: truncate(resultRecord.output ?? resultRecord.error ?? extra.error ?? result, 500),
+      duration_ms: getDuration(input),
+    },
+  }));
+
+  delete state.spans[spanKey];
+  saveState(sessionKey, state);
+  await flush();
+}
+
+async function handleAPIResult(input: Dict) {
+  const sessionKey = getSessionKey(input);
+  const state = sessionKey ? loadState(sessionKey) : { spans: {} };
+  const usage = getUsage(input);
+  if (!usage) {
+    await flush();
+    return;
+  }
+
+  enqueue(buildEnvelope(FRAMEWORK, HOST, 'metric.snapshot', sessionKey, {
+    runId: state.runId,
+    payload: {
+      metrics: {
+        usage,
+        model: getModel(input),
+        provider: pickString(getExtra(input).provider),
+        duration_ms: getDuration(input),
+      },
+    },
+  }));
+  await flush();
+}
+
+async function handleSessionEnd(input: Dict) {
+  const sessionKey = getSessionKey(input);
+  const state = sessionKey ? loadState(sessionKey) : { spans: {} };
+  if (state.runId) {
+    enqueue(buildEnvelope(FRAMEWORK, HOST, 'run.end', sessionKey, {
+      runId: state.runId,
+      payload: {
+        status: input.interrupted || getExtra(input).interrupted ? 'interrupted' : 'success',
+        model: getModel(input),
+      },
+    }));
+  }
+
+  enqueue(buildEnvelope(FRAMEWORK, HOST, 'session.end', sessionKey, {
+    payload: {
+      model: getModel(input),
+      completed: getExtra(input).completed,
+      interrupted: getExtra(input).interrupted,
+    },
+  }));
+  clearState(sessionKey);
+  await flush();
+}
+
+const handler = async () => {
+  const hookType = process.argv[2] || 'unknown';
+
+  let input: Dict = {};
+  try {
+    const stdin = await readStdin();
+    if (stdin) {
+      input = JSON.parse(stdin);
+    }
+  } catch {
+    input = {};
+  }
+
+  try {
+    switch (hookType) {
+      case 'session-start':
+      case 'start':
+        await handleSessionStart(input);
+        break;
+      case 'run-start':
+      case 'pre-llm':
+        await handleRunStart(input);
+        break;
+      case 'run-end':
+      case 'post-llm':
+        await handleRunEnd(input);
+        break;
+      case 'tool-start':
+        await handleToolStart(input);
+        break;
+      case 'tool-end':
+        await handleToolEnd(input);
+        break;
+      case 'api-result':
+      case 'post-api':
+        await handleAPIResult(input);
+        break;
+      case 'session-end':
+      case 'stop':
+        await handleSessionEnd(input);
+        break;
+      default:
+        console.debug(`[agentmon] unknown hermes hook type: ${hookType}`);
+    }
+  } catch (err) {
+    console.debug('[agentmon] hermes handler error:', err);
+  }
+};
+
+handler();