feat(audit): Add core audit logging infrastructure
- Add AuditLogger class with rotation support - Add audit configuration to config schema - Instrument tool execution with full audit logging - Instrument session lifecycle (create, message, delete, transfer, compact) - Add audit logger initialization in daemon - Add cron scheduler audit logging Audit events captured: - tool.start/success/error/denied - session.create/message/delete/transfer/compact - cron.trigger/add/remove All logs go to ~/.local/share/flynn/audit.log (JSON lines) with rotation (10MB files, 30-day retention)
This commit is contained in:
William Valentin
@@ -0,0 +1,75 @@
|
||||
import { createReadStream, promises as fs } from 'fs';
|
||||
import { dirname, basename } from 'path';
|
||||
import type { AuditEvent, AuditQuery } from './types.js';
|
||||
|
||||
export async function queryAuditLogs(logPath: string, query: AuditQuery): Promise<AuditEvent[]> {
|
||||
const logs: AuditEvent[] = [];
|
||||
|
||||
try {
|
||||
const content = await fs.readFile(logPath, 'utf-8');
|
||||
const lines = content.split('\n').filter(line => line.trim() !== '');
|
||||
|
||||
for (const line of lines) {
|
||||
try {
|
||||
const event: AuditEvent = JSON.parse(line);
|
||||
if (matchesQuery(event, query)) {
|
||||
logs.push(event);
|
||||
}
|
||||
} catch {
|
||||
// Skip malformed lines
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
return logs.sort((a, b) => a.timestamp - b.timestamp);
|
||||
}
|
||||
|
||||
function matchesQuery(event: AuditEvent, query: AuditQuery): boolean {
|
||||
if (query.start_time && event.timestamp < query.start_time) {
|
||||
return false;
|
||||
}
|
||||
if (query.end_time && event.timestamp > query.end_time) {
|
||||
return false;
|
||||
}
|
||||
if (query.event_types && !query.event_types.includes(event.event_type)) {
|
||||
return false;
|
||||
}
|
||||
if (query.level && event.level !== query.level) {
|
||||
return false;
|
||||
}
|
||||
if (query.session_id) {
|
||||
const eventSessionId = (event.event as Record<string, unknown>).session_id as string | undefined;
|
||||
if (eventSessionId !== query.session_id) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (query.tool_name) {
|
||||
const eventToolName = (event.event as Record<string, unknown>).tool_name as string | undefined;
|
||||
if (eventToolName !== query.tool_name) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function exportAuditLogs(logPath: string, query: AuditQuery, format: 'json' | 'csv'): Promise<string> {
|
||||
const logs = await queryAuditLogs(logPath, query);
|
||||
|
||||
if (format === 'json') {
|
||||
return JSON.stringify(logs, null, 2);
|
||||
}
|
||||
|
||||
// CSV format
|
||||
const headers = ['timestamp', 'level', 'event_type', 'event'];
|
||||
const rows = logs.map(log => {
|
||||
const eventStr = JSON.stringify(log.event).replace(/"/g, '""');
|
||||
return `${log.timestamp},${log.level},${log.event_type},"${eventStr}"`;
|
||||
});
|
||||
|
||||
return [headers.join(','), ...rows].join('\n');
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import type { AuditLogger } from './logger.js';
|
||||
export * from './types.js';
|
||||
export { AuditLogger } from './logger.js';
|
||||
|
||||
export let auditLogger: AuditLogger | null = null;
|
||||
|
||||
export function initAuditLogger(logger: AuditLogger): void {
|
||||
auditLogger = logger;
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
import { createWriteStream, existsSync, mkdirSync, promises as fs } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
import type {
|
||||
AuditEvent,
|
||||
AuditConfig,
|
||||
ToolStartEvent,
|
||||
ToolSuccessEvent,
|
||||
ToolErrorEvent,
|
||||
ToolDeniedEvent,
|
||||
SessionCreateEvent,
|
||||
SessionMessageEvent,
|
||||
SessionDeleteEvent,
|
||||
SessionCompactEvent,
|
||||
CronTriggerEvent,
|
||||
WebhookReceiveEvent,
|
||||
HeartbeatCycleEvent,
|
||||
HeartbeatCheckEvent,
|
||||
HeartbeatFailEvent,
|
||||
HeartbeatRecoverEvent,
|
||||
GmailPollEvent,
|
||||
GmailNewEmailEvent,
|
||||
} from './types.js';
|
||||
import { AuditRotator } from './rotation.js';
|
||||
|
||||
export class AuditLogger {
|
||||
private writeStream: import('fs').WriteStream | null = null;
|
||||
private config: AuditConfig;
|
||||
private rotator: AuditRotator;
|
||||
|
||||
constructor(config: AuditConfig) {
|
||||
this.config = config;
|
||||
this.rotator = new AuditRotator(config);
|
||||
|
||||
if (!this.config.enabled) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.ensureLogDirectory();
|
||||
this.rotator.checkRotation();
|
||||
this.writeStream = createWriteStream(config.path, { flags: 'a' });
|
||||
}
|
||||
|
||||
private ensureLogDirectory(): void {
|
||||
const logDir = dirname(this.config.path);
|
||||
if (!existsSync(logDir)) {
|
||||
mkdirSync(logDir, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
private write(event: Omit<AuditEvent, 'timestamp'>): void {
|
||||
if (!this.config.enabled || !this.writeStream) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.rotator.checkRotation();
|
||||
|
||||
const fullEvent: AuditEvent = { ...event, timestamp: Date.now() };
|
||||
this.writeStream!.write(JSON.stringify(fullEvent) + '\n');
|
||||
}
|
||||
|
||||
private shouldLog(category: 'tools' | 'sessions' | 'automation', level: string): boolean {
|
||||
const levelOrder = { debug: 0, info: 1, warn: 2, error: 3 };
|
||||
const configLevel = this.config.levels[category];
|
||||
return levelOrder[level as keyof typeof levelOrder] >= levelOrder[configLevel];
|
||||
}
|
||||
|
||||
// ── Tool Events ───────────────────────────────────────────────
|
||||
|
||||
toolStart(event: ToolStartEvent): void {
|
||||
if (!this.shouldLog('tools', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'tool.start', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
toolSuccess(event: ToolSuccessEvent): void {
|
||||
if (!this.shouldLog('tools', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'tool.success', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
toolError(event: ToolErrorEvent): void {
|
||||
if (!this.shouldLog('tools', 'error')) return;
|
||||
this.write({ level: 'error', event_type: 'tool.error', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
toolDenied(event: ToolDeniedEvent): void {
|
||||
if (!this.shouldLog('tools', 'warn')) return;
|
||||
this.write({ level: 'warn', event_type: 'tool.denied', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
// ── Session Events ───────────────────────────────────────────
|
||||
|
||||
sessionCreate(event: SessionCreateEvent): void {
|
||||
if (!this.shouldLog('sessions', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'session.create', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
sessionMessage(event: SessionMessageEvent): void {
|
||||
if (!this.shouldLog('sessions', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'session.message', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
sessionDelete(event: SessionDeleteEvent): void {
|
||||
if (!this.shouldLog('sessions', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'session.delete', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
sessionCompact(event: SessionCompactEvent): void {
|
||||
if (!this.shouldLog('sessions', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'session.compact', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
sessionTransfer(from: string, to: string, messageCount: number): void {
|
||||
if (!this.shouldLog('sessions', 'debug')) return;
|
||||
this.write({
|
||||
level: 'debug',
|
||||
event_type: 'session.transfer',
|
||||
event: { from_session: from, to_session: to, message_count: messageCount },
|
||||
});
|
||||
}
|
||||
|
||||
// ── Automation Events ───────────────────────────────────────
|
||||
|
||||
// Cron
|
||||
cronTrigger(event: CronTriggerEvent): void {
|
||||
if (!this.shouldLog('automation', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'cron.trigger', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
cronAdd(jobName: string, schedule: string): void {
|
||||
if (!this.shouldLog('automation', 'info')) return;
|
||||
this.write({
|
||||
level: 'info',
|
||||
event_type: 'cron.add',
|
||||
event: { job_name: jobName, schedule },
|
||||
});
|
||||
}
|
||||
|
||||
cronRemove(jobName: string): void {
|
||||
if (!this.shouldLog('automation', 'info')) return;
|
||||
this.write({
|
||||
level: 'info',
|
||||
event_type: 'cron.remove',
|
||||
event: { job_name: jobName },
|
||||
});
|
||||
}
|
||||
|
||||
// Webhook
|
||||
webhookReceive(event: WebhookReceiveEvent): void {
|
||||
if (!this.shouldLog('automation', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'webhook.receive', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
webhookNotFound(webhookName: string): void {
|
||||
if (!this.shouldLog('automation', 'warn')) return;
|
||||
this.write({
|
||||
level: 'warn',
|
||||
event_type: 'webhook.not_found',
|
||||
event: { webhook_name: webhookName },
|
||||
});
|
||||
}
|
||||
|
||||
webhookDenied(webhookName: string, reason: string): void {
|
||||
if (!this.shouldLog('automation', 'warn')) return;
|
||||
this.write({
|
||||
level: 'warn',
|
||||
event_type: 'webhook.denied',
|
||||
event: { webhook_name: webhookName, reason },
|
||||
});
|
||||
}
|
||||
|
||||
// Heartbeat
|
||||
heartbeatCycle(event: HeartbeatCycleEvent): void {
|
||||
if (!this.shouldLog('automation', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'heartbeat.cycle', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
heartbeatCheck(event: HeartbeatCheckEvent): void {
|
||||
if (!this.shouldLog('automation', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'heartbeat.check', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
heartbeatFail(event: HeartbeatFailEvent): void {
|
||||
if (!this.shouldLog('automation', 'warn')) return;
|
||||
this.write({ level: 'warn', event_type: 'heartbeat.fail', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
heartbeatRecover(event: HeartbeatRecoverEvent): void {
|
||||
if (!this.shouldLog('automation', 'info')) return;
|
||||
this.write({ level: 'info', event_type: 'heartbeat.recover', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
// Gmail
|
||||
gmailPoll(event: GmailPollEvent): void {
|
||||
if (!this.shouldLog('automation', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'gmail.poll', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
gmailNewEmail(event: GmailNewEmailEvent): void {
|
||||
if (!this.shouldLog('automation', 'debug')) return;
|
||||
this.write({ level: 'debug', event_type: 'gmail.new_email', event: event as unknown as Record<string, unknown> });
|
||||
}
|
||||
|
||||
gmailError(error: string, context?: string): void {
|
||||
if (!this.shouldLog('automation', 'error')) return;
|
||||
this.write({
|
||||
level: 'error',
|
||||
event_type: 'gmail.error',
|
||||
event: { error, context },
|
||||
});
|
||||
}
|
||||
|
||||
// ── System Events ────────────────────────────────────────────
|
||||
|
||||
systemStart(component: string, config?: Record<string, unknown>): void {
|
||||
if (!this.config.enabled) return;
|
||||
this.write({
|
||||
level: 'info',
|
||||
event_type: 'system.start',
|
||||
event: { component, config },
|
||||
});
|
||||
}
|
||||
|
||||
systemStop(component: string, reason?: string): void {
|
||||
if (!this.config.enabled) return;
|
||||
this.write({
|
||||
level: 'info',
|
||||
event_type: 'system.stop',
|
||||
event: { component, reason },
|
||||
});
|
||||
}
|
||||
|
||||
systemConfig(component: string, action: string, config: Record<string, unknown>): void {
|
||||
if (!this.config.enabled) return;
|
||||
this.write({
|
||||
level: 'info',
|
||||
event_type: 'system.config',
|
||||
event: { component, action, config },
|
||||
});
|
||||
}
|
||||
|
||||
// ── Lifecycle ───────────────────────────────────────────────
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.writeStream) {
|
||||
await new Promise<void>((resolve) => {
|
||||
this.writeStream!.end(() => resolve());
|
||||
});
|
||||
this.writeStream = null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
import { createReadStream, createWriteStream, existsSync, promises as fs, statSync } from 'fs';
|
||||
import { basename, dirname } from 'path';
|
||||
import { createGzip } from 'zlib';
|
||||
import { pipeline } from 'stream/promises';
|
||||
import type { AuditConfig } from './types.js';
|
||||
|
||||
export class AuditRotator {
|
||||
private config: AuditConfig;
|
||||
private lastRotationCheck = 0;
|
||||
private readonly CHECK_INTERVAL = 60_000; // Check every minute
|
||||
|
||||
constructor(config: AuditConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
checkRotation(): boolean {
|
||||
const now = Date.now();
|
||||
if (now - this.lastRotationCheck < this.CHECK_INTERVAL) {
|
||||
return false;
|
||||
}
|
||||
this.lastRotationCheck = now;
|
||||
|
||||
return this.rotateIfNeeded();
|
||||
}
|
||||
|
||||
private rotateIfNeeded(): boolean {
|
||||
if (!existsSync(this.config.path)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const stats = statSync(this.config.path);
|
||||
const sizeMb = stats.size / (1024 * 1024);
|
||||
|
||||
if (sizeMb < this.config.max_size_mb) {
|
||||
return false;
|
||||
}
|
||||
|
||||
this.rotate();
|
||||
return true;
|
||||
}
|
||||
|
||||
private rotate(): void {
|
||||
const basePath = this.config.path;
|
||||
const baseName = basename(basePath);
|
||||
const dir = dirname(basePath);
|
||||
const timestamp = new Date().toISOString().split('T')[0];
|
||||
const rotatedPath = `${dir}/${baseName}.${timestamp}`;
|
||||
const compressedPath = `${rotatedPath}.gz`;
|
||||
|
||||
try {
|
||||
// Move current log to rotated name
|
||||
if (existsSync(rotatedPath)) {
|
||||
fs.unlink(rotatedPath);
|
||||
}
|
||||
if (existsSync(compressedPath)) {
|
||||
fs.unlink(compressedPath);
|
||||
}
|
||||
|
||||
fs.rename(basePath, rotatedPath);
|
||||
|
||||
// Compress the rotated file
|
||||
const gzip = createGzip();
|
||||
const input = createReadStream(rotatedPath);
|
||||
const output = createWriteStream(compressedPath);
|
||||
|
||||
pipeline(input, gzip, output).then(() => {
|
||||
fs.unlink(rotatedPath);
|
||||
}).catch((err) => {
|
||||
console.error('Audit log compression failed:', err);
|
||||
});
|
||||
|
||||
// Clean up old logs
|
||||
this.cleanupOldLogs();
|
||||
} catch (err) {
|
||||
console.error('Audit log rotation failed:', err);
|
||||
}
|
||||
}
|
||||
|
||||
private async cleanupOldLogs(): Promise<void> {
|
||||
const dir = dirname(this.config.path);
|
||||
const baseName = basename(this.config.path);
|
||||
const cutoffDate = new Date();
|
||||
cutoffDate.setDate(cutoffDate.getDate() - this.config.keep_days);
|
||||
|
||||
try {
|
||||
const files = await fs.readdir(dir);
|
||||
|
||||
for (const file of files) {
|
||||
if (!file.startsWith(baseName)) continue;
|
||||
|
||||
const filePath = `${dir}/${file}`;
|
||||
const stats = await fs.stat(filePath);
|
||||
const fileDate = new Date(stats.mtime);
|
||||
|
||||
if (fileDate < cutoffDate) {
|
||||
await fs.unlink(filePath);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Audit log cleanup failed:', err);
|
||||
}
|
||||
}
|
||||
|
||||
async forceRotation(): Promise<void> {
|
||||
this.rotate();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
export type AuditLevel = 'debug' | 'info' | 'warn' | 'error';
|
||||
|
||||
export type AuditEventType =
|
||||
// Tool execution
|
||||
| 'tool.start' | 'tool.success' | 'tool.error' | 'tool.denied'
|
||||
// Session lifecycle
|
||||
| 'session.create' | 'session.message' | 'session.delete' | 'session.transfer' | 'session.compact'
|
||||
// Automation - Cron
|
||||
| 'cron.trigger' | 'cron.sent' | 'cron.add' | 'cron.remove'
|
||||
// Automation - Webhook
|
||||
| 'webhook.receive' | 'webhook.sent' | 'webhook.not_found' | 'webhook.denied'
|
||||
// Automation - Heartbeat
|
||||
| 'heartbeat.cycle' | 'heartbeat.check' | 'heartbeat.fail' | 'heartbeat.recover'
|
||||
// Automation - Gmail
|
||||
| 'gmail.poll' | 'gmail.new_email' | 'gmail.error'
|
||||
// System events
|
||||
| 'system.start' | 'system.stop' | 'system.config';
|
||||
|
||||
export interface AuditEvent {
|
||||
timestamp: number;
|
||||
level: AuditLevel;
|
||||
event_type: AuditEventType;
|
||||
event: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface AuditConfig {
|
||||
enabled: boolean;
|
||||
path: string;
|
||||
max_size_mb: number;
|
||||
keep_days: number;
|
||||
levels: {
|
||||
tools: AuditLevel;
|
||||
sessions: AuditLevel;
|
||||
automation: AuditLevel;
|
||||
};
|
||||
}
|
||||
|
||||
export interface AuditQuery {
|
||||
start_time?: number;
|
||||
end_time?: number;
|
||||
event_types?: AuditEventType[];
|
||||
session_id?: string;
|
||||
tool_name?: string;
|
||||
level?: AuditLevel;
|
||||
}
|
||||
|
||||
export interface ToolStartEvent {
|
||||
tool_name: string;
|
||||
tool_args: unknown;
|
||||
session_id?: string;
|
||||
channel?: string;
|
||||
sender?: string;
|
||||
agent_tier?: string;
|
||||
}
|
||||
|
||||
export interface ToolSuccessEvent {
|
||||
tool_name: string;
|
||||
result: { success: boolean; output: string; error?: string };
|
||||
duration_ms: number;
|
||||
session_id?: string;
|
||||
}
|
||||
|
||||
export interface ToolErrorEvent {
|
||||
tool_name: string;
|
||||
error: string;
|
||||
duration_ms: number;
|
||||
reason?: string;
|
||||
session_id?: string;
|
||||
}
|
||||
|
||||
export interface ToolDeniedEvent {
|
||||
tool_name: string;
|
||||
reason: string;
|
||||
session_id?: string;
|
||||
denial_type: 'policy' | 'hook' | 'not_found';
|
||||
}
|
||||
|
||||
export interface SessionCreateEvent {
|
||||
session_id: string;
|
||||
frontend: string;
|
||||
user_id: string;
|
||||
}
|
||||
|
||||
export interface SessionMessageEvent {
|
||||
session_id: string;
|
||||
role: 'user' | 'assistant';
|
||||
content_length: number;
|
||||
}
|
||||
|
||||
export interface SessionDeleteEvent {
|
||||
session_id: string;
|
||||
message_count: number;
|
||||
}
|
||||
|
||||
export interface SessionCompactEvent {
|
||||
session_id: string;
|
||||
messages_before: number;
|
||||
messages_after: number;
|
||||
tokens_before: number;
|
||||
tokens_after: number;
|
||||
}
|
||||
|
||||
export interface CronTriggerEvent {
|
||||
job_name: string;
|
||||
schedule: string;
|
||||
message: string;
|
||||
output_channel: string;
|
||||
output_peer: string;
|
||||
}
|
||||
|
||||
export interface WebhookReceiveEvent {
|
||||
webhook_name: string;
|
||||
body: string;
|
||||
signature_verified: boolean;
|
||||
output_channel: string;
|
||||
output_peer: string;
|
||||
}
|
||||
|
||||
export interface HeartbeatCycleEvent {
|
||||
interval_ms: number;
|
||||
checks: string[];
|
||||
healthy: boolean;
|
||||
consecutive_failures: number;
|
||||
}
|
||||
|
||||
export interface HeartbeatCheckEvent {
|
||||
check_name: string;
|
||||
healthy: boolean;
|
||||
message: string;
|
||||
duration_ms: number;
|
||||
}
|
||||
|
||||
export interface HeartbeatFailEvent {
|
||||
checks_failed: string[];
|
||||
consecutive_failures: number;
|
||||
threshold: number;
|
||||
}
|
||||
|
||||
export interface HeartbeatRecoverEvent {
|
||||
consecutive_failures_before: number;
|
||||
}
|
||||
|
||||
export interface GmailPollEvent {
|
||||
mode: 'watch' | 'poll';
|
||||
interval_ms?: number;
|
||||
emails_processed: number;
|
||||
new_emails: number;
|
||||
}
|
||||
|
||||
export interface GmailNewEmailEvent {
|
||||
email_id: string;
|
||||
from: string;
|
||||
subject: string;
|
||||
labels: string[];
|
||||
}
|
||||
Reference in New Issue
Block a user