diff --git a/README.md b/README.md index 0d4bbba..a196cfd 100644 --- a/README.md +++ b/README.md @@ -1042,6 +1042,13 @@ Memory persistence is hybrid: memory: enabled: true auto_extract: true + proactive_extract: + enabled: false # Per-turn extraction beyond compaction-only writes + min_tool_calls: 1 # Trigger only when this many tool calls occur in a turn + namespace: global # Target namespace for extracted durable facts + daily_log: + enabled: false # Append each turn to dated namespaces (daily/YYYY-MM-DD) + namespace_prefix: daily max_context_tokens: 2000 embedding: enabled: true @@ -1079,6 +1086,8 @@ Search backend selection: When the selected backend is unavailable (for example embedding provider errors), search falls back gracefully to keyword matching. `memory.auto_extract` controls whether compaction appends extracted durable facts to `global` memory. +`memory.proactive_extract` controls optional per-turn extraction after responses (useful for tool-heavy workflows). +`memory.daily_log` controls optional append-only daily turn logs in dated namespaces. ### Proactive Context Management diff --git a/docs/plans/state.json b/docs/plans/state.json index 728a032..aecfb60 100644 --- a/docs/plans/state.json +++ b/docs/plans/state.json @@ -5179,10 +5179,27 @@ "docs/plans/state.json" ], "test_status": "pnpm test:run src/gateway/session-bridge.test.ts src/gateway/handlers/agent.test.ts + pnpm typecheck passing" + }, + "memory-daily-log-and-proactive-extraction": { + "status": "completed", + "date": "2026-02-18", + "updated": "2026-02-18", + "summary": "Implemented Tier A2 memory cadence improvements: optional per-turn proactive extraction (tool-call threshold + target namespace) and optional append-only daily memory logs. Wired config through routing/session-bridge to orchestrator, added schema/docs updates, and expanded regression coverage.", + "files_modified": [ + "src/backends/native/orchestrator.ts", + "src/backends/native/orchestrator.test.ts", + "src/config/schema.ts", + "src/config/schema.test.ts", + "src/daemon/routing.ts", + "src/gateway/session-bridge.ts", + "README.md", + "docs/plans/state.json" + ], + "test_status": "pnpm test:run src/backends/native/orchestrator.test.ts src/config/schema.test.ts src/gateway/session-bridge.test.ts src/gateway/handlers/agent.test.ts + pnpm typecheck passing" } }, "overall_progress": { - "total_test_count": 1903, + "total_test_count": 1908, "all_tests_passing": true, "p0_completion": "3/3 (100%)", "p1_completion": "4/4 (100%)", @@ -5202,7 +5219,7 @@ "gmail_auth_cli": "flynn gmail-auth command implemented with OAuth2 flow, doctor check, config routed to Telegram", "native_audio_support": "completed — smart routing for native audio (Gemini/OpenAI/GitHub) vs Whisper transcription fallback", "remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening", - "next_up": "Implement Tier A2 from the OpenClaw roadmap: daily memory-log cadence + proactive extraction beyond compaction-only paths" + "next_up": "Implement Tier A3 from the OpenClaw roadmap: proactive announce delivery mode for automation jobs" }, "soul_md_and_cron_create": { "date": "2026-02-11", diff --git a/src/backends/native/orchestrator.test.ts b/src/backends/native/orchestrator.test.ts index 4378e2a..3d60ef1 100644 --- a/src/backends/native/orchestrator.test.ts +++ b/src/backends/native/orchestrator.test.ts @@ -452,6 +452,68 @@ describe('AgentOrchestrator', () => { getPromptSectionsSpy.mockRestore(); rmSync(tempDir, { recursive: true, force: true }); }); + + it('appends daily memory log entries when enabled', async () => { + const tempDir = mkdtempSync(join(tmpdir(), 'flynn-orchestrator-daily-log-')); + const memoryStore = new MemoryStore({ dir: tempDir, maxContextTokens: 2000 }); + + const orchestrator = new AgentOrchestrator({ + modelRouter: mockRouter, + systemPrompt: 'You are a helpful agent.', + primaryTier: 'default', + delegation: { + compaction: 'fast', + memory_extraction: 'default', + classification: 'complex', + tool_summarisation: 'default', + complex_reasoning: 'complex', + }, + maxDelegationDepth: 10, + memoryStore, + memoryDailyLogEnabled: true, + memoryDailyLogNamespacePrefix: 'daily', + }); + + await orchestrator.process('Log this turn'); + const date = new Date().toISOString().slice(0, 10); + const dailyLog = memoryStore.read(`daily/${date}`); + + expect(dailyLog).toContain('Log this turn'); + expect(dailyLog).toContain('default response'); + expect(dailyLog).toContain('tool_calls: 0'); + + rmSync(tempDir, { recursive: true, force: true }); + }); + + it('runs proactive per-turn extraction when enabled and threshold is met', async () => { + const tempDir = mkdtempSync(join(tmpdir(), 'flynn-orchestrator-proactive-extract-')); + const memoryStore = new MemoryStore({ dir: tempDir, maxContextTokens: 2000 }); + + const orchestrator = new AgentOrchestrator({ + modelRouter: mockRouter, + systemPrompt: 'You are a helpful agent.', + primaryTier: 'default', + delegation: { + compaction: 'fast', + memory_extraction: 'default', + classification: 'complex', + tool_summarisation: 'default', + complex_reasoning: 'complex', + }, + maxDelegationDepth: 10, + memoryStore, + memoryProactiveExtractEnabled: true, + memoryProactiveExtractMinToolCalls: 0, + memoryProactiveExtractNamespace: 'global', + }); + + await orchestrator.process('Capture durable context'); + const extracted = memoryStore.read('global'); + + expect(extracted).toContain('default response'); + + rmSync(tempDir, { recursive: true, force: true }); + }); }); describe('compact()', () => { diff --git a/src/backends/native/orchestrator.ts b/src/backends/native/orchestrator.ts index a1e29a6..e7b1d54 100644 --- a/src/backends/native/orchestrator.ts +++ b/src/backends/native/orchestrator.ts @@ -15,7 +15,7 @@ import { estimateCost } from '../../models/costs.js'; import { auditLogger } from '../../audit/index.js'; import { buildAdaptiveMemoryContext, buildRecentMemoryContext } from '../../memory/adaptive.js'; import { buildUserMessage } from '../../models/media.js'; -import { CONTEXT_CHECKPOINT_PROMPT } from './prompts.js'; +import { CONTEXT_CHECKPOINT_PROMPT, MEMORY_EXTRACTION_PROMPT } from './prompts.js'; // ── Public types ────────────────────────────────────────────────────── @@ -123,6 +123,16 @@ export interface OrchestratorConfig { memoryInjectionStrategy?: 'all' | 'recent' | 'adaptive'; /** Maximum tokens allowed for injected memory context. */ memoryMaxInjectionTokens?: number; + /** Enable per-turn proactive memory extraction (beyond compaction-time extraction). */ + memoryProactiveExtractEnabled?: boolean; + /** Minimum tool calls in a turn to trigger proactive memory extraction. */ + memoryProactiveExtractMinToolCalls?: number; + /** Namespace to append proactive extracted facts into. */ + memoryProactiveExtractNamespace?: string; + /** Enable daily append-only memory logs of user/assistant turns. */ + memoryDailyLogEnabled?: boolean; + /** Namespace prefix for daily logs (full namespace: /YYYY-MM-DD). */ + memoryDailyLogNamespacePrefix?: string; /** Automatically retry failed primary runs on a higher tier. */ autoEscalate?: boolean; /** Tier to try for auto-escalation retries. Defaults to complex. */ @@ -157,9 +167,16 @@ export class AgentOrchestrator { private _memoryAutoExtract: boolean; private _memoryInjectionStrategy: 'all' | 'recent' | 'adaptive'; private _memoryMaxInjectionTokens: number; + private _memoryProactiveExtractEnabled: boolean; + private _memoryProactiveExtractMinToolCalls: number; + private _memoryProactiveExtractNamespace: string; + private _memoryDailyLogEnabled: boolean; + private _memoryDailyLogNamespacePrefix: string; private _autoEscalate: boolean; private _autoEscalateTier: ModelTier; private _systemPromptBase: string; + private _externalOnToolUse?: (event: ToolUseEvent) => void; + private _activeRunToolStarts = 0; private _usageByTier: Map = new Map(); private _lastContextAlertLevel: ContextAlertLevel | null = null; private _pendingContextAlert?: ContextAlert; @@ -178,9 +195,15 @@ export class AgentOrchestrator { this._memoryAutoExtract = config.memoryAutoExtract ?? true; this._memoryInjectionStrategy = config.memoryInjectionStrategy ?? 'all'; this._memoryMaxInjectionTokens = config.memoryMaxInjectionTokens ?? 2000; + this._memoryProactiveExtractEnabled = config.memoryProactiveExtractEnabled ?? false; + this._memoryProactiveExtractMinToolCalls = Math.max(0, config.memoryProactiveExtractMinToolCalls ?? 1); + this._memoryProactiveExtractNamespace = config.memoryProactiveExtractNamespace ?? 'global'; + this._memoryDailyLogEnabled = config.memoryDailyLogEnabled ?? false; + this._memoryDailyLogNamespacePrefix = config.memoryDailyLogNamespacePrefix ?? 'daily'; this._autoEscalate = config.autoEscalate ?? false; this._autoEscalateTier = config.autoEscalateTier ?? 'complex'; this._systemPromptBase = config.systemPrompt; + this._externalOnToolUse = config.onToolUse; // Create the primary NativeAgent for user-facing conversation this._agent = new NativeAgent({ @@ -190,7 +213,7 @@ export class AgentOrchestrator { toolRegistry: config.toolRegistry, toolExecutor: config.toolExecutor, maxIterations: config.maxIterations, - onToolUse: config.onToolUse, + onToolUse: (event) => this._handleToolUse(event), toolPolicyContext: config.toolPolicyContext, attachmentCollector: config.attachmentCollector, }); @@ -265,6 +288,7 @@ export class AgentOrchestrator { * exceeds the context window threshold and compacts it before processing. */ async process(userMessage: string, attachments?: Attachment[]): Promise { + this._activeRunToolStarts = 0; this._injectMemoryContext(userMessage); await this._runProactiveContextMaintenance(); await this.compactIfNeeded(); @@ -281,6 +305,7 @@ export class AgentOrchestrator { this._restoreHistory(before); const escalated = await this._retryWithEscalation(userMessage, attachments, before, originalTier); if (escalated) { + await this._runPostTurnMemoryMaintenance(userMessage, escalated, this._activeRunToolStarts); return escalated; } const friendly = @@ -316,6 +341,7 @@ export class AgentOrchestrator { const escalated = await this._retryWithEscalation(userMessage, attachments, before, originalTier); if (escalated) { + await this._runPostTurnMemoryMaintenance(userMessage, escalated, this._activeRunToolStarts); return escalated; } @@ -334,6 +360,7 @@ export class AgentOrchestrator { return friendly; } + await this._runPostTurnMemoryMaintenance(userMessage, result, this._activeRunToolStarts); return result; } @@ -444,7 +471,7 @@ export class AgentOrchestrator { /** Set the tool-use callback on the primary agent. */ setOnToolUse(callback: ((event: ToolUseEvent) => void) | undefined): void { - this._agent.setOnToolUse(callback); + this._externalOnToolUse = callback; } /** Request cancellation for the current primary-agent operation. */ @@ -595,6 +622,89 @@ export class AgentOrchestrator { return context.slice(0, maxChars); } + private _handleToolUse(event: ToolUseEvent): void { + if (event.type === 'start') { + this._activeRunToolStarts += 1; + } + this._externalOnToolUse?.(event); + } + + private async _runPostTurnMemoryMaintenance(userMessage: string, assistantText: string, toolCallsInRun: number): Promise { + if (!this._memoryStore) { + return; + } + + if (this._memoryDailyLogEnabled) { + this._appendDailyLog(userMessage, assistantText, toolCallsInRun); + } + + if (!this._memoryProactiveExtractEnabled || toolCallsInRun < this._memoryProactiveExtractMinToolCalls) { + return; + } + + try { + const extractionTier = this.getDelegationTier('memory_extraction'); + const extraction = await this.delegate({ + tier: extractionTier, + systemPrompt: MEMORY_EXTRACTION_PROMPT, + message: this._buildExtractionInput(userMessage, assistantText, toolCallsInRun), + maxTokens: 512, + }); + const extractedContent = extraction.content.trim(); + if (!extractedContent || extractedContent.toLowerCase().includes('no facts')) { + return; + } + this._memoryStore.write(this._memoryProactiveExtractNamespace, extractedContent, 'append'); + } catch (error) { + console.warn('[Flynn:memory] Proactive per-turn extraction failed:', error); + } + } + + private _appendDailyLog(userMessage: string, assistantText: string, toolCallsInRun: number): void { + if (!this._memoryStore) { + return; + } + const date = new Date().toISOString().slice(0, 10); + const timestamp = new Date().toISOString(); + const namespace = `${this._memoryDailyLogNamespacePrefix}/${date}`; + const user = this._truncateForMemory(userMessage, 2000); + const assistant = this._truncateForMemory(assistantText, 4000); + const block = [ + `## ${timestamp}`, + '', + `- tool_calls: ${toolCallsInRun}`, + '', + '### User', + user, + '', + '### Assistant', + assistant, + '', + ].join('\n'); + this._memoryStore.write(namespace, block, 'append'); + } + + private _buildExtractionInput(userMessage: string, assistantText: string, toolCallsInRun: number): string { + const user = this._truncateForMemory(userMessage, 4000); + const assistant = this._truncateForMemory(assistantText, 6000); + return [ + `Tool calls in this turn: ${toolCallsInRun}`, + '', + 'User message:', + user, + '', + 'Assistant response:', + assistant, + ].join('\n'); + } + + private _truncateForMemory(text: string, maxChars: number): string { + if (text.length <= maxChars) { + return text; + } + return `${text.slice(0, maxChars)}...[truncated]`; + } + /** * Check whether automatic compaction should run, and if so, compact. * Called before each `process()` call when compaction is configured. diff --git a/src/config/schema.test.ts b/src/config/schema.test.ts index ca53a93..014526d 100644 --- a/src/config/schema.test.ts +++ b/src/config/schema.test.ts @@ -1277,6 +1277,11 @@ describe('configSchema — memory injection strategy', () => { const result = configSchema.parse(minimalConfig); expect(result.memory.injection_strategy).toBe('all'); expect(result.memory.max_injection_tokens).toBe(2000); + expect(result.memory.proactive_extract.enabled).toBe(false); + expect(result.memory.proactive_extract.min_tool_calls).toBe(1); + expect(result.memory.proactive_extract.namespace).toBe('global'); + expect(result.memory.daily_log.enabled).toBe(false); + expect(result.memory.daily_log.namespace_prefix).toBe('daily'); expect(result.memory.qmd.enabled).toBe(false); expect(result.memory.qmd.top_k).toBe(8); expect(result.memory.qmd.min_score).toBe(0.15); @@ -1309,6 +1314,29 @@ describe('configSchema — memory injection strategy', () => { expect(result.memory.qmd.top_k).toBe(12); expect(result.memory.qmd.min_score).toBe(0.2); }); + + it('accepts proactive extraction and daily log settings', () => { + const result = configSchema.parse({ + ...minimalConfig, + memory: { + proactive_extract: { + enabled: true, + min_tool_calls: 3, + namespace: 'global/facts', + }, + daily_log: { + enabled: true, + namespace_prefix: 'memory', + }, + }, + }); + + expect(result.memory.proactive_extract.enabled).toBe(true); + expect(result.memory.proactive_extract.min_tool_calls).toBe(3); + expect(result.memory.proactive_extract.namespace).toBe('global/facts'); + expect(result.memory.daily_log.enabled).toBe(true); + expect(result.memory.daily_log.namespace_prefix).toBe('memory'); + }); }); describe('configSchema — compaction importance threshold', () => { diff --git a/src/config/schema.ts b/src/config/schema.ts index dbcbceb..c495eb0 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -516,6 +516,15 @@ const memorySchema = z.object({ enabled: z.boolean().default(true), dir: z.string().optional(), // Default: ~/.local/share/flynn/memory auto_extract: z.boolean().default(true), + proactive_extract: z.object({ + enabled: z.boolean().default(false), + min_tool_calls: z.number().min(0).max(50).default(1), + namespace: z.string().default('global'), + }).default({}), + daily_log: z.object({ + enabled: z.boolean().default(false), + namespace_prefix: z.string().default('daily'), + }).default({}), injection_strategy: z.enum(['all', 'recent', 'adaptive']).default('all'), max_injection_tokens: z.number().min(100).max(10000).default(2000), max_context_tokens: z.number().min(100).max(10000).default(2000), diff --git a/src/daemon/routing.ts b/src/daemon/routing.ts index 35d1ee3..d01c0f4 100644 --- a/src/daemon/routing.ts +++ b/src/daemon/routing.ts @@ -315,6 +315,11 @@ export function createMessageRouter(deps: { memoryAutoExtract: deps.config.memory?.auto_extract, memoryInjectionStrategy: deps.config.memory?.injection_strategy, memoryMaxInjectionTokens: deps.config.memory?.max_injection_tokens, + memoryProactiveExtractEnabled: deps.config.memory?.proactive_extract?.enabled, + memoryProactiveExtractMinToolCalls: deps.config.memory?.proactive_extract?.min_tool_calls, + memoryProactiveExtractNamespace: deps.config.memory?.proactive_extract?.namespace, + memoryDailyLogEnabled: deps.config.memory?.daily_log?.enabled, + memoryDailyLogNamespacePrefix: deps.config.memory?.daily_log?.namespace_prefix, autoEscalate: deps.config.agents.auto_escalate, autoEscalateTier: 'complex', toolPolicyContext, diff --git a/src/gateway/session-bridge.ts b/src/gateway/session-bridge.ts index d042a31..d49c85b 100644 --- a/src/gateway/session-bridge.ts +++ b/src/gateway/session-bridge.ts @@ -313,6 +313,11 @@ export class SessionBridge { memoryAutoExtract: config?.memory?.auto_extract, memoryInjectionStrategy: config?.memory?.injection_strategy, memoryMaxInjectionTokens: config?.memory?.max_injection_tokens, + memoryProactiveExtractEnabled: config?.memory?.proactive_extract?.enabled, + memoryProactiveExtractMinToolCalls: config?.memory?.proactive_extract?.min_tool_calls, + memoryProactiveExtractNamespace: config?.memory?.proactive_extract?.namespace, + memoryDailyLogEnabled: config?.memory?.daily_log?.enabled, + memoryDailyLogNamespacePrefix: config?.memory?.daily_log?.namespace_prefix, toolPolicyContext: { agent: primaryTier, provider: config?.models.default.provider,