feat: add proactive memory extraction and daily logs

This commit is contained in:
William Valentin
2026-02-18 10:10:47 -08:00
parent 9cbd66cdcc
commit f38fc063d2
8 changed files with 250 additions and 5 deletions
+9
View File
@@ -1042,6 +1042,13 @@ Memory persistence is hybrid:
memory: memory:
enabled: true enabled: true
auto_extract: true auto_extract: true
proactive_extract:
enabled: false # Per-turn extraction beyond compaction-only writes
min_tool_calls: 1 # Trigger only when this many tool calls occur in a turn
namespace: global # Target namespace for extracted durable facts
daily_log:
enabled: false # Append each turn to dated namespaces (daily/YYYY-MM-DD)
namespace_prefix: daily
max_context_tokens: 2000 max_context_tokens: 2000
embedding: embedding:
enabled: true enabled: true
@@ -1079,6 +1086,8 @@ Search backend selection:
When the selected backend is unavailable (for example embedding provider errors), search falls back gracefully to keyword matching. When the selected backend is unavailable (for example embedding provider errors), search falls back gracefully to keyword matching.
`memory.auto_extract` controls whether compaction appends extracted durable facts to `global` memory. `memory.auto_extract` controls whether compaction appends extracted durable facts to `global` memory.
`memory.proactive_extract` controls optional per-turn extraction after responses (useful for tool-heavy workflows).
`memory.daily_log` controls optional append-only daily turn logs in dated namespaces.
### Proactive Context Management ### Proactive Context Management
+19 -2
View File
@@ -5179,10 +5179,27 @@
"docs/plans/state.json" "docs/plans/state.json"
], ],
"test_status": "pnpm test:run src/gateway/session-bridge.test.ts src/gateway/handlers/agent.test.ts + pnpm typecheck passing" "test_status": "pnpm test:run src/gateway/session-bridge.test.ts src/gateway/handlers/agent.test.ts + pnpm typecheck passing"
},
"memory-daily-log-and-proactive-extraction": {
"status": "completed",
"date": "2026-02-18",
"updated": "2026-02-18",
"summary": "Implemented Tier A2 memory cadence improvements: optional per-turn proactive extraction (tool-call threshold + target namespace) and optional append-only daily memory logs. Wired config through routing/session-bridge to orchestrator, added schema/docs updates, and expanded regression coverage.",
"files_modified": [
"src/backends/native/orchestrator.ts",
"src/backends/native/orchestrator.test.ts",
"src/config/schema.ts",
"src/config/schema.test.ts",
"src/daemon/routing.ts",
"src/gateway/session-bridge.ts",
"README.md",
"docs/plans/state.json"
],
"test_status": "pnpm test:run src/backends/native/orchestrator.test.ts src/config/schema.test.ts src/gateway/session-bridge.test.ts src/gateway/handlers/agent.test.ts + pnpm typecheck passing"
} }
}, },
"overall_progress": { "overall_progress": {
"total_test_count": 1903, "total_test_count": 1908,
"all_tests_passing": true, "all_tests_passing": true,
"p0_completion": "3/3 (100%)", "p0_completion": "3/3 (100%)",
"p1_completion": "4/4 (100%)", "p1_completion": "4/4 (100%)",
@@ -5202,7 +5219,7 @@
"gmail_auth_cli": "flynn gmail-auth command implemented with OAuth2 flow, doctor check, config routed to Telegram", "gmail_auth_cli": "flynn gmail-auth command implemented with OAuth2 flow, doctor check, config routed to Telegram",
"native_audio_support": "completed — smart routing for native audio (Gemini/OpenAI/GitHub) vs Whisper transcription fallback", "native_audio_support": "completed — smart routing for native audio (Gemini/OpenAI/GitHub) vs Whisper transcription fallback",
"remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening", "remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening",
"next_up": "Implement Tier A2 from the OpenClaw roadmap: daily memory-log cadence + proactive extraction beyond compaction-only paths" "next_up": "Implement Tier A3 from the OpenClaw roadmap: proactive announce delivery mode for automation jobs"
}, },
"soul_md_and_cron_create": { "soul_md_and_cron_create": {
"date": "2026-02-11", "date": "2026-02-11",
+62
View File
@@ -452,6 +452,68 @@ describe('AgentOrchestrator', () => {
getPromptSectionsSpy.mockRestore(); getPromptSectionsSpy.mockRestore();
rmSync(tempDir, { recursive: true, force: true }); rmSync(tempDir, { recursive: true, force: true });
}); });
it('appends daily memory log entries when enabled', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'flynn-orchestrator-daily-log-'));
const memoryStore = new MemoryStore({ dir: tempDir, maxContextTokens: 2000 });
const orchestrator = new AgentOrchestrator({
modelRouter: mockRouter,
systemPrompt: 'You are a helpful agent.',
primaryTier: 'default',
delegation: {
compaction: 'fast',
memory_extraction: 'default',
classification: 'complex',
tool_summarisation: 'default',
complex_reasoning: 'complex',
},
maxDelegationDepth: 10,
memoryStore,
memoryDailyLogEnabled: true,
memoryDailyLogNamespacePrefix: 'daily',
});
await orchestrator.process('Log this turn');
const date = new Date().toISOString().slice(0, 10);
const dailyLog = memoryStore.read(`daily/${date}`);
expect(dailyLog).toContain('Log this turn');
expect(dailyLog).toContain('default response');
expect(dailyLog).toContain('tool_calls: 0');
rmSync(tempDir, { recursive: true, force: true });
});
it('runs proactive per-turn extraction when enabled and threshold is met', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'flynn-orchestrator-proactive-extract-'));
const memoryStore = new MemoryStore({ dir: tempDir, maxContextTokens: 2000 });
const orchestrator = new AgentOrchestrator({
modelRouter: mockRouter,
systemPrompt: 'You are a helpful agent.',
primaryTier: 'default',
delegation: {
compaction: 'fast',
memory_extraction: 'default',
classification: 'complex',
tool_summarisation: 'default',
complex_reasoning: 'complex',
},
maxDelegationDepth: 10,
memoryStore,
memoryProactiveExtractEnabled: true,
memoryProactiveExtractMinToolCalls: 0,
memoryProactiveExtractNamespace: 'global',
});
await orchestrator.process('Capture durable context');
const extracted = memoryStore.read('global');
expect(extracted).toContain('default response');
rmSync(tempDir, { recursive: true, force: true });
});
}); });
describe('compact()', () => { describe('compact()', () => {
+113 -3
View File
@@ -15,7 +15,7 @@ import { estimateCost } from '../../models/costs.js';
import { auditLogger } from '../../audit/index.js'; import { auditLogger } from '../../audit/index.js';
import { buildAdaptiveMemoryContext, buildRecentMemoryContext } from '../../memory/adaptive.js'; import { buildAdaptiveMemoryContext, buildRecentMemoryContext } from '../../memory/adaptive.js';
import { buildUserMessage } from '../../models/media.js'; import { buildUserMessage } from '../../models/media.js';
import { CONTEXT_CHECKPOINT_PROMPT } from './prompts.js'; import { CONTEXT_CHECKPOINT_PROMPT, MEMORY_EXTRACTION_PROMPT } from './prompts.js';
// ── Public types ────────────────────────────────────────────────────── // ── Public types ──────────────────────────────────────────────────────
@@ -123,6 +123,16 @@ export interface OrchestratorConfig {
memoryInjectionStrategy?: 'all' | 'recent' | 'adaptive'; memoryInjectionStrategy?: 'all' | 'recent' | 'adaptive';
/** Maximum tokens allowed for injected memory context. */ /** Maximum tokens allowed for injected memory context. */
memoryMaxInjectionTokens?: number; memoryMaxInjectionTokens?: number;
/** Enable per-turn proactive memory extraction (beyond compaction-time extraction). */
memoryProactiveExtractEnabled?: boolean;
/** Minimum tool calls in a turn to trigger proactive memory extraction. */
memoryProactiveExtractMinToolCalls?: number;
/** Namespace to append proactive extracted facts into. */
memoryProactiveExtractNamespace?: string;
/** Enable daily append-only memory logs of user/assistant turns. */
memoryDailyLogEnabled?: boolean;
/** Namespace prefix for daily logs (full namespace: <prefix>/YYYY-MM-DD). */
memoryDailyLogNamespacePrefix?: string;
/** Automatically retry failed primary runs on a higher tier. */ /** Automatically retry failed primary runs on a higher tier. */
autoEscalate?: boolean; autoEscalate?: boolean;
/** Tier to try for auto-escalation retries. Defaults to complex. */ /** Tier to try for auto-escalation retries. Defaults to complex. */
@@ -157,9 +167,16 @@ export class AgentOrchestrator {
private _memoryAutoExtract: boolean; private _memoryAutoExtract: boolean;
private _memoryInjectionStrategy: 'all' | 'recent' | 'adaptive'; private _memoryInjectionStrategy: 'all' | 'recent' | 'adaptive';
private _memoryMaxInjectionTokens: number; private _memoryMaxInjectionTokens: number;
private _memoryProactiveExtractEnabled: boolean;
private _memoryProactiveExtractMinToolCalls: number;
private _memoryProactiveExtractNamespace: string;
private _memoryDailyLogEnabled: boolean;
private _memoryDailyLogNamespacePrefix: string;
private _autoEscalate: boolean; private _autoEscalate: boolean;
private _autoEscalateTier: ModelTier; private _autoEscalateTier: ModelTier;
private _systemPromptBase: string; private _systemPromptBase: string;
private _externalOnToolUse?: (event: ToolUseEvent) => void;
private _activeRunToolStarts = 0;
private _usageByTier: Map<string, TierUsageStats> = new Map(); private _usageByTier: Map<string, TierUsageStats> = new Map();
private _lastContextAlertLevel: ContextAlertLevel | null = null; private _lastContextAlertLevel: ContextAlertLevel | null = null;
private _pendingContextAlert?: ContextAlert; private _pendingContextAlert?: ContextAlert;
@@ -178,9 +195,15 @@ export class AgentOrchestrator {
this._memoryAutoExtract = config.memoryAutoExtract ?? true; this._memoryAutoExtract = config.memoryAutoExtract ?? true;
this._memoryInjectionStrategy = config.memoryInjectionStrategy ?? 'all'; this._memoryInjectionStrategy = config.memoryInjectionStrategy ?? 'all';
this._memoryMaxInjectionTokens = config.memoryMaxInjectionTokens ?? 2000; this._memoryMaxInjectionTokens = config.memoryMaxInjectionTokens ?? 2000;
this._memoryProactiveExtractEnabled = config.memoryProactiveExtractEnabled ?? false;
this._memoryProactiveExtractMinToolCalls = Math.max(0, config.memoryProactiveExtractMinToolCalls ?? 1);
this._memoryProactiveExtractNamespace = config.memoryProactiveExtractNamespace ?? 'global';
this._memoryDailyLogEnabled = config.memoryDailyLogEnabled ?? false;
this._memoryDailyLogNamespacePrefix = config.memoryDailyLogNamespacePrefix ?? 'daily';
this._autoEscalate = config.autoEscalate ?? false; this._autoEscalate = config.autoEscalate ?? false;
this._autoEscalateTier = config.autoEscalateTier ?? 'complex'; this._autoEscalateTier = config.autoEscalateTier ?? 'complex';
this._systemPromptBase = config.systemPrompt; this._systemPromptBase = config.systemPrompt;
this._externalOnToolUse = config.onToolUse;
// Create the primary NativeAgent for user-facing conversation // Create the primary NativeAgent for user-facing conversation
this._agent = new NativeAgent({ this._agent = new NativeAgent({
@@ -190,7 +213,7 @@ export class AgentOrchestrator {
toolRegistry: config.toolRegistry, toolRegistry: config.toolRegistry,
toolExecutor: config.toolExecutor, toolExecutor: config.toolExecutor,
maxIterations: config.maxIterations, maxIterations: config.maxIterations,
onToolUse: config.onToolUse, onToolUse: (event) => this._handleToolUse(event),
toolPolicyContext: config.toolPolicyContext, toolPolicyContext: config.toolPolicyContext,
attachmentCollector: config.attachmentCollector, attachmentCollector: config.attachmentCollector,
}); });
@@ -265,6 +288,7 @@ export class AgentOrchestrator {
* exceeds the context window threshold and compacts it before processing. * exceeds the context window threshold and compacts it before processing.
*/ */
async process(userMessage: string, attachments?: Attachment[]): Promise<string> { async process(userMessage: string, attachments?: Attachment[]): Promise<string> {
this._activeRunToolStarts = 0;
this._injectMemoryContext(userMessage); this._injectMemoryContext(userMessage);
await this._runProactiveContextMaintenance(); await this._runProactiveContextMaintenance();
await this.compactIfNeeded(); await this.compactIfNeeded();
@@ -281,6 +305,7 @@ export class AgentOrchestrator {
this._restoreHistory(before); this._restoreHistory(before);
const escalated = await this._retryWithEscalation(userMessage, attachments, before, originalTier); const escalated = await this._retryWithEscalation(userMessage, attachments, before, originalTier);
if (escalated) { if (escalated) {
await this._runPostTurnMemoryMaintenance(userMessage, escalated, this._activeRunToolStarts);
return escalated; return escalated;
} }
const friendly = const friendly =
@@ -316,6 +341,7 @@ export class AgentOrchestrator {
const escalated = await this._retryWithEscalation(userMessage, attachments, before, originalTier); const escalated = await this._retryWithEscalation(userMessage, attachments, before, originalTier);
if (escalated) { if (escalated) {
await this._runPostTurnMemoryMaintenance(userMessage, escalated, this._activeRunToolStarts);
return escalated; return escalated;
} }
@@ -334,6 +360,7 @@ export class AgentOrchestrator {
return friendly; return friendly;
} }
await this._runPostTurnMemoryMaintenance(userMessage, result, this._activeRunToolStarts);
return result; return result;
} }
@@ -444,7 +471,7 @@ export class AgentOrchestrator {
/** Set the tool-use callback on the primary agent. */ /** Set the tool-use callback on the primary agent. */
setOnToolUse(callback: ((event: ToolUseEvent) => void) | undefined): void { setOnToolUse(callback: ((event: ToolUseEvent) => void) | undefined): void {
this._agent.setOnToolUse(callback); this._externalOnToolUse = callback;
} }
/** Request cancellation for the current primary-agent operation. */ /** Request cancellation for the current primary-agent operation. */
@@ -595,6 +622,89 @@ export class AgentOrchestrator {
return context.slice(0, maxChars); return context.slice(0, maxChars);
} }
private _handleToolUse(event: ToolUseEvent): void {
if (event.type === 'start') {
this._activeRunToolStarts += 1;
}
this._externalOnToolUse?.(event);
}
private async _runPostTurnMemoryMaintenance(userMessage: string, assistantText: string, toolCallsInRun: number): Promise<void> {
if (!this._memoryStore) {
return;
}
if (this._memoryDailyLogEnabled) {
this._appendDailyLog(userMessage, assistantText, toolCallsInRun);
}
if (!this._memoryProactiveExtractEnabled || toolCallsInRun < this._memoryProactiveExtractMinToolCalls) {
return;
}
try {
const extractionTier = this.getDelegationTier('memory_extraction');
const extraction = await this.delegate({
tier: extractionTier,
systemPrompt: MEMORY_EXTRACTION_PROMPT,
message: this._buildExtractionInput(userMessage, assistantText, toolCallsInRun),
maxTokens: 512,
});
const extractedContent = extraction.content.trim();
if (!extractedContent || extractedContent.toLowerCase().includes('no facts')) {
return;
}
this._memoryStore.write(this._memoryProactiveExtractNamespace, extractedContent, 'append');
} catch (error) {
console.warn('[Flynn:memory] Proactive per-turn extraction failed:', error);
}
}
private _appendDailyLog(userMessage: string, assistantText: string, toolCallsInRun: number): void {
if (!this._memoryStore) {
return;
}
const date = new Date().toISOString().slice(0, 10);
const timestamp = new Date().toISOString();
const namespace = `${this._memoryDailyLogNamespacePrefix}/${date}`;
const user = this._truncateForMemory(userMessage, 2000);
const assistant = this._truncateForMemory(assistantText, 4000);
const block = [
`## ${timestamp}`,
'',
`- tool_calls: ${toolCallsInRun}`,
'',
'### User',
user,
'',
'### Assistant',
assistant,
'',
].join('\n');
this._memoryStore.write(namespace, block, 'append');
}
private _buildExtractionInput(userMessage: string, assistantText: string, toolCallsInRun: number): string {
const user = this._truncateForMemory(userMessage, 4000);
const assistant = this._truncateForMemory(assistantText, 6000);
return [
`Tool calls in this turn: ${toolCallsInRun}`,
'',
'User message:',
user,
'',
'Assistant response:',
assistant,
].join('\n');
}
private _truncateForMemory(text: string, maxChars: number): string {
if (text.length <= maxChars) {
return text;
}
return `${text.slice(0, maxChars)}...[truncated]`;
}
/** /**
* Check whether automatic compaction should run, and if so, compact. * Check whether automatic compaction should run, and if so, compact.
* Called before each `process()` call when compaction is configured. * Called before each `process()` call when compaction is configured.
+28
View File
@@ -1277,6 +1277,11 @@ describe('configSchema — memory injection strategy', () => {
const result = configSchema.parse(minimalConfig); const result = configSchema.parse(minimalConfig);
expect(result.memory.injection_strategy).toBe('all'); expect(result.memory.injection_strategy).toBe('all');
expect(result.memory.max_injection_tokens).toBe(2000); expect(result.memory.max_injection_tokens).toBe(2000);
expect(result.memory.proactive_extract.enabled).toBe(false);
expect(result.memory.proactive_extract.min_tool_calls).toBe(1);
expect(result.memory.proactive_extract.namespace).toBe('global');
expect(result.memory.daily_log.enabled).toBe(false);
expect(result.memory.daily_log.namespace_prefix).toBe('daily');
expect(result.memory.qmd.enabled).toBe(false); expect(result.memory.qmd.enabled).toBe(false);
expect(result.memory.qmd.top_k).toBe(8); expect(result.memory.qmd.top_k).toBe(8);
expect(result.memory.qmd.min_score).toBe(0.15); expect(result.memory.qmd.min_score).toBe(0.15);
@@ -1309,6 +1314,29 @@ describe('configSchema — memory injection strategy', () => {
expect(result.memory.qmd.top_k).toBe(12); expect(result.memory.qmd.top_k).toBe(12);
expect(result.memory.qmd.min_score).toBe(0.2); expect(result.memory.qmd.min_score).toBe(0.2);
}); });
it('accepts proactive extraction and daily log settings', () => {
const result = configSchema.parse({
...minimalConfig,
memory: {
proactive_extract: {
enabled: true,
min_tool_calls: 3,
namespace: 'global/facts',
},
daily_log: {
enabled: true,
namespace_prefix: 'memory',
},
},
});
expect(result.memory.proactive_extract.enabled).toBe(true);
expect(result.memory.proactive_extract.min_tool_calls).toBe(3);
expect(result.memory.proactive_extract.namespace).toBe('global/facts');
expect(result.memory.daily_log.enabled).toBe(true);
expect(result.memory.daily_log.namespace_prefix).toBe('memory');
});
}); });
describe('configSchema — compaction importance threshold', () => { describe('configSchema — compaction importance threshold', () => {
+9
View File
@@ -516,6 +516,15 @@ const memorySchema = z.object({
enabled: z.boolean().default(true), enabled: z.boolean().default(true),
dir: z.string().optional(), // Default: ~/.local/share/flynn/memory dir: z.string().optional(), // Default: ~/.local/share/flynn/memory
auto_extract: z.boolean().default(true), auto_extract: z.boolean().default(true),
proactive_extract: z.object({
enabled: z.boolean().default(false),
min_tool_calls: z.number().min(0).max(50).default(1),
namespace: z.string().default('global'),
}).default({}),
daily_log: z.object({
enabled: z.boolean().default(false),
namespace_prefix: z.string().default('daily'),
}).default({}),
injection_strategy: z.enum(['all', 'recent', 'adaptive']).default('all'), injection_strategy: z.enum(['all', 'recent', 'adaptive']).default('all'),
max_injection_tokens: z.number().min(100).max(10000).default(2000), max_injection_tokens: z.number().min(100).max(10000).default(2000),
max_context_tokens: z.number().min(100).max(10000).default(2000), max_context_tokens: z.number().min(100).max(10000).default(2000),
+5
View File
@@ -315,6 +315,11 @@ export function createMessageRouter(deps: {
memoryAutoExtract: deps.config.memory?.auto_extract, memoryAutoExtract: deps.config.memory?.auto_extract,
memoryInjectionStrategy: deps.config.memory?.injection_strategy, memoryInjectionStrategy: deps.config.memory?.injection_strategy,
memoryMaxInjectionTokens: deps.config.memory?.max_injection_tokens, memoryMaxInjectionTokens: deps.config.memory?.max_injection_tokens,
memoryProactiveExtractEnabled: deps.config.memory?.proactive_extract?.enabled,
memoryProactiveExtractMinToolCalls: deps.config.memory?.proactive_extract?.min_tool_calls,
memoryProactiveExtractNamespace: deps.config.memory?.proactive_extract?.namespace,
memoryDailyLogEnabled: deps.config.memory?.daily_log?.enabled,
memoryDailyLogNamespacePrefix: deps.config.memory?.daily_log?.namespace_prefix,
autoEscalate: deps.config.agents.auto_escalate, autoEscalate: deps.config.agents.auto_escalate,
autoEscalateTier: 'complex', autoEscalateTier: 'complex',
toolPolicyContext, toolPolicyContext,
+5
View File
@@ -313,6 +313,11 @@ export class SessionBridge {
memoryAutoExtract: config?.memory?.auto_extract, memoryAutoExtract: config?.memory?.auto_extract,
memoryInjectionStrategy: config?.memory?.injection_strategy, memoryInjectionStrategy: config?.memory?.injection_strategy,
memoryMaxInjectionTokens: config?.memory?.max_injection_tokens, memoryMaxInjectionTokens: config?.memory?.max_injection_tokens,
memoryProactiveExtractEnabled: config?.memory?.proactive_extract?.enabled,
memoryProactiveExtractMinToolCalls: config?.memory?.proactive_extract?.min_tool_calls,
memoryProactiveExtractNamespace: config?.memory?.proactive_extract?.namespace,
memoryDailyLogEnabled: config?.memory?.daily_log?.enabled,
memoryDailyLogNamespacePrefix: config?.memory?.daily_log?.namespace_prefix,
toolPolicyContext: { toolPolicyContext: {
agent: primaryTier, agent: primaryTier,
provider: config?.models.default.provider, provider: config?.models.default.provider,