feat: add proactive memory extraction and daily logs

This commit is contained in:
William Valentin
2026-02-18 10:10:47 -08:00
parent 9cbd66cdcc
commit f38fc063d2
8 changed files with 250 additions and 5 deletions
+9
View File
@@ -1042,6 +1042,13 @@ Memory persistence is hybrid:
memory:
enabled: true
auto_extract: true
proactive_extract:
enabled: false # Per-turn extraction beyond compaction-only writes
min_tool_calls: 1 # Trigger only when this many tool calls occur in a turn
namespace: global # Target namespace for extracted durable facts
daily_log:
enabled: false # Append each turn to dated namespaces (daily/YYYY-MM-DD)
namespace_prefix: daily
max_context_tokens: 2000
embedding:
enabled: true
@@ -1079,6 +1086,8 @@ Search backend selection:
When the selected backend is unavailable (for example embedding provider errors), search falls back gracefully to keyword matching.
`memory.auto_extract` controls whether compaction appends extracted durable facts to `global` memory.
`memory.proactive_extract` controls optional per-turn extraction after responses (useful for tool-heavy workflows).
`memory.daily_log` controls optional append-only daily turn logs in dated namespaces.
### Proactive Context Management
+19 -2
View File
@@ -5179,10 +5179,27 @@
"docs/plans/state.json"
],
"test_status": "pnpm test:run src/gateway/session-bridge.test.ts src/gateway/handlers/agent.test.ts + pnpm typecheck passing"
},
"memory-daily-log-and-proactive-extraction": {
"status": "completed",
"date": "2026-02-18",
"updated": "2026-02-18",
"summary": "Implemented Tier A2 memory cadence improvements: optional per-turn proactive extraction (tool-call threshold + target namespace) and optional append-only daily memory logs. Wired config through routing/session-bridge to orchestrator, added schema/docs updates, and expanded regression coverage.",
"files_modified": [
"src/backends/native/orchestrator.ts",
"src/backends/native/orchestrator.test.ts",
"src/config/schema.ts",
"src/config/schema.test.ts",
"src/daemon/routing.ts",
"src/gateway/session-bridge.ts",
"README.md",
"docs/plans/state.json"
],
"test_status": "pnpm test:run src/backends/native/orchestrator.test.ts src/config/schema.test.ts src/gateway/session-bridge.test.ts src/gateway/handlers/agent.test.ts + pnpm typecheck passing"
}
},
"overall_progress": {
"total_test_count": 1903,
"total_test_count": 1908,
"all_tests_passing": true,
"p0_completion": "3/3 (100%)",
"p1_completion": "4/4 (100%)",
@@ -5202,7 +5219,7 @@
"gmail_auth_cli": "flynn gmail-auth command implemented with OAuth2 flow, doctor check, config routed to Telegram",
"native_audio_support": "completed — smart routing for native audio (Gemini/OpenAI/GitHub) vs Whisper transcription fallback",
"remaining_phases_completion": "Phase 1: 3/3 (100%) — context levels, command registry, memory structure. Phase 2: 3/3 (100%) — component registry, confidence routing, history index. Phase 3: 2/2 (100%) — adaptive memory/compaction, truthfulness/autonomy hardening",
"next_up": "Implement Tier A2 from the OpenClaw roadmap: daily memory-log cadence + proactive extraction beyond compaction-only paths"
"next_up": "Implement Tier A3 from the OpenClaw roadmap: proactive announce delivery mode for automation jobs"
},
"soul_md_and_cron_create": {
"date": "2026-02-11",
+62
View File
@@ -452,6 +452,68 @@ describe('AgentOrchestrator', () => {
getPromptSectionsSpy.mockRestore();
rmSync(tempDir, { recursive: true, force: true });
});
it('appends daily memory log entries when enabled', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'flynn-orchestrator-daily-log-'));
const memoryStore = new MemoryStore({ dir: tempDir, maxContextTokens: 2000 });
const orchestrator = new AgentOrchestrator({
modelRouter: mockRouter,
systemPrompt: 'You are a helpful agent.',
primaryTier: 'default',
delegation: {
compaction: 'fast',
memory_extraction: 'default',
classification: 'complex',
tool_summarisation: 'default',
complex_reasoning: 'complex',
},
maxDelegationDepth: 10,
memoryStore,
memoryDailyLogEnabled: true,
memoryDailyLogNamespacePrefix: 'daily',
});
await orchestrator.process('Log this turn');
const date = new Date().toISOString().slice(0, 10);
const dailyLog = memoryStore.read(`daily/${date}`);
expect(dailyLog).toContain('Log this turn');
expect(dailyLog).toContain('default response');
expect(dailyLog).toContain('tool_calls: 0');
rmSync(tempDir, { recursive: true, force: true });
});
it('runs proactive per-turn extraction when enabled and threshold is met', async () => {
const tempDir = mkdtempSync(join(tmpdir(), 'flynn-orchestrator-proactive-extract-'));
const memoryStore = new MemoryStore({ dir: tempDir, maxContextTokens: 2000 });
const orchestrator = new AgentOrchestrator({
modelRouter: mockRouter,
systemPrompt: 'You are a helpful agent.',
primaryTier: 'default',
delegation: {
compaction: 'fast',
memory_extraction: 'default',
classification: 'complex',
tool_summarisation: 'default',
complex_reasoning: 'complex',
},
maxDelegationDepth: 10,
memoryStore,
memoryProactiveExtractEnabled: true,
memoryProactiveExtractMinToolCalls: 0,
memoryProactiveExtractNamespace: 'global',
});
await orchestrator.process('Capture durable context');
const extracted = memoryStore.read('global');
expect(extracted).toContain('default response');
rmSync(tempDir, { recursive: true, force: true });
});
});
describe('compact()', () => {
+113 -3
View File
@@ -15,7 +15,7 @@ import { estimateCost } from '../../models/costs.js';
import { auditLogger } from '../../audit/index.js';
import { buildAdaptiveMemoryContext, buildRecentMemoryContext } from '../../memory/adaptive.js';
import { buildUserMessage } from '../../models/media.js';
import { CONTEXT_CHECKPOINT_PROMPT } from './prompts.js';
import { CONTEXT_CHECKPOINT_PROMPT, MEMORY_EXTRACTION_PROMPT } from './prompts.js';
// ── Public types ──────────────────────────────────────────────────────
@@ -123,6 +123,16 @@ export interface OrchestratorConfig {
memoryInjectionStrategy?: 'all' | 'recent' | 'adaptive';
/** Maximum tokens allowed for injected memory context. */
memoryMaxInjectionTokens?: number;
/** Enable per-turn proactive memory extraction (beyond compaction-time extraction). */
memoryProactiveExtractEnabled?: boolean;
/** Minimum tool calls in a turn to trigger proactive memory extraction. */
memoryProactiveExtractMinToolCalls?: number;
/** Namespace to append proactive extracted facts into. */
memoryProactiveExtractNamespace?: string;
/** Enable daily append-only memory logs of user/assistant turns. */
memoryDailyLogEnabled?: boolean;
/** Namespace prefix for daily logs (full namespace: <prefix>/YYYY-MM-DD). */
memoryDailyLogNamespacePrefix?: string;
/** Automatically retry failed primary runs on a higher tier. */
autoEscalate?: boolean;
/** Tier to try for auto-escalation retries. Defaults to complex. */
@@ -157,9 +167,16 @@ export class AgentOrchestrator {
private _memoryAutoExtract: boolean;
private _memoryInjectionStrategy: 'all' | 'recent' | 'adaptive';
private _memoryMaxInjectionTokens: number;
private _memoryProactiveExtractEnabled: boolean;
private _memoryProactiveExtractMinToolCalls: number;
private _memoryProactiveExtractNamespace: string;
private _memoryDailyLogEnabled: boolean;
private _memoryDailyLogNamespacePrefix: string;
private _autoEscalate: boolean;
private _autoEscalateTier: ModelTier;
private _systemPromptBase: string;
private _externalOnToolUse?: (event: ToolUseEvent) => void;
private _activeRunToolStarts = 0;
private _usageByTier: Map<string, TierUsageStats> = new Map();
private _lastContextAlertLevel: ContextAlertLevel | null = null;
private _pendingContextAlert?: ContextAlert;
@@ -178,9 +195,15 @@ export class AgentOrchestrator {
this._memoryAutoExtract = config.memoryAutoExtract ?? true;
this._memoryInjectionStrategy = config.memoryInjectionStrategy ?? 'all';
this._memoryMaxInjectionTokens = config.memoryMaxInjectionTokens ?? 2000;
this._memoryProactiveExtractEnabled = config.memoryProactiveExtractEnabled ?? false;
this._memoryProactiveExtractMinToolCalls = Math.max(0, config.memoryProactiveExtractMinToolCalls ?? 1);
this._memoryProactiveExtractNamespace = config.memoryProactiveExtractNamespace ?? 'global';
this._memoryDailyLogEnabled = config.memoryDailyLogEnabled ?? false;
this._memoryDailyLogNamespacePrefix = config.memoryDailyLogNamespacePrefix ?? 'daily';
this._autoEscalate = config.autoEscalate ?? false;
this._autoEscalateTier = config.autoEscalateTier ?? 'complex';
this._systemPromptBase = config.systemPrompt;
this._externalOnToolUse = config.onToolUse;
// Create the primary NativeAgent for user-facing conversation
this._agent = new NativeAgent({
@@ -190,7 +213,7 @@ export class AgentOrchestrator {
toolRegistry: config.toolRegistry,
toolExecutor: config.toolExecutor,
maxIterations: config.maxIterations,
onToolUse: config.onToolUse,
onToolUse: (event) => this._handleToolUse(event),
toolPolicyContext: config.toolPolicyContext,
attachmentCollector: config.attachmentCollector,
});
@@ -265,6 +288,7 @@ export class AgentOrchestrator {
* exceeds the context window threshold and compacts it before processing.
*/
async process(userMessage: string, attachments?: Attachment[]): Promise<string> {
this._activeRunToolStarts = 0;
this._injectMemoryContext(userMessage);
await this._runProactiveContextMaintenance();
await this.compactIfNeeded();
@@ -281,6 +305,7 @@ export class AgentOrchestrator {
this._restoreHistory(before);
const escalated = await this._retryWithEscalation(userMessage, attachments, before, originalTier);
if (escalated) {
await this._runPostTurnMemoryMaintenance(userMessage, escalated, this._activeRunToolStarts);
return escalated;
}
const friendly =
@@ -316,6 +341,7 @@ export class AgentOrchestrator {
const escalated = await this._retryWithEscalation(userMessage, attachments, before, originalTier);
if (escalated) {
await this._runPostTurnMemoryMaintenance(userMessage, escalated, this._activeRunToolStarts);
return escalated;
}
@@ -334,6 +360,7 @@ export class AgentOrchestrator {
return friendly;
}
await this._runPostTurnMemoryMaintenance(userMessage, result, this._activeRunToolStarts);
return result;
}
@@ -444,7 +471,7 @@ export class AgentOrchestrator {
/** Set the tool-use callback on the primary agent. */
setOnToolUse(callback: ((event: ToolUseEvent) => void) | undefined): void {
this._agent.setOnToolUse(callback);
this._externalOnToolUse = callback;
}
/** Request cancellation for the current primary-agent operation. */
@@ -595,6 +622,89 @@ export class AgentOrchestrator {
return context.slice(0, maxChars);
}
private _handleToolUse(event: ToolUseEvent): void {
if (event.type === 'start') {
this._activeRunToolStarts += 1;
}
this._externalOnToolUse?.(event);
}
private async _runPostTurnMemoryMaintenance(userMessage: string, assistantText: string, toolCallsInRun: number): Promise<void> {
if (!this._memoryStore) {
return;
}
if (this._memoryDailyLogEnabled) {
this._appendDailyLog(userMessage, assistantText, toolCallsInRun);
}
if (!this._memoryProactiveExtractEnabled || toolCallsInRun < this._memoryProactiveExtractMinToolCalls) {
return;
}
try {
const extractionTier = this.getDelegationTier('memory_extraction');
const extraction = await this.delegate({
tier: extractionTier,
systemPrompt: MEMORY_EXTRACTION_PROMPT,
message: this._buildExtractionInput(userMessage, assistantText, toolCallsInRun),
maxTokens: 512,
});
const extractedContent = extraction.content.trim();
if (!extractedContent || extractedContent.toLowerCase().includes('no facts')) {
return;
}
this._memoryStore.write(this._memoryProactiveExtractNamespace, extractedContent, 'append');
} catch (error) {
console.warn('[Flynn:memory] Proactive per-turn extraction failed:', error);
}
}
private _appendDailyLog(userMessage: string, assistantText: string, toolCallsInRun: number): void {
if (!this._memoryStore) {
return;
}
const date = new Date().toISOString().slice(0, 10);
const timestamp = new Date().toISOString();
const namespace = `${this._memoryDailyLogNamespacePrefix}/${date}`;
const user = this._truncateForMemory(userMessage, 2000);
const assistant = this._truncateForMemory(assistantText, 4000);
const block = [
`## ${timestamp}`,
'',
`- tool_calls: ${toolCallsInRun}`,
'',
'### User',
user,
'',
'### Assistant',
assistant,
'',
].join('\n');
this._memoryStore.write(namespace, block, 'append');
}
private _buildExtractionInput(userMessage: string, assistantText: string, toolCallsInRun: number): string {
const user = this._truncateForMemory(userMessage, 4000);
const assistant = this._truncateForMemory(assistantText, 6000);
return [
`Tool calls in this turn: ${toolCallsInRun}`,
'',
'User message:',
user,
'',
'Assistant response:',
assistant,
].join('\n');
}
private _truncateForMemory(text: string, maxChars: number): string {
if (text.length <= maxChars) {
return text;
}
return `${text.slice(0, maxChars)}...[truncated]`;
}
/**
* Check whether automatic compaction should run, and if so, compact.
* Called before each `process()` call when compaction is configured.
+28
View File
@@ -1277,6 +1277,11 @@ describe('configSchema — memory injection strategy', () => {
const result = configSchema.parse(minimalConfig);
expect(result.memory.injection_strategy).toBe('all');
expect(result.memory.max_injection_tokens).toBe(2000);
expect(result.memory.proactive_extract.enabled).toBe(false);
expect(result.memory.proactive_extract.min_tool_calls).toBe(1);
expect(result.memory.proactive_extract.namespace).toBe('global');
expect(result.memory.daily_log.enabled).toBe(false);
expect(result.memory.daily_log.namespace_prefix).toBe('daily');
expect(result.memory.qmd.enabled).toBe(false);
expect(result.memory.qmd.top_k).toBe(8);
expect(result.memory.qmd.min_score).toBe(0.15);
@@ -1309,6 +1314,29 @@ describe('configSchema — memory injection strategy', () => {
expect(result.memory.qmd.top_k).toBe(12);
expect(result.memory.qmd.min_score).toBe(0.2);
});
it('accepts proactive extraction and daily log settings', () => {
const result = configSchema.parse({
...minimalConfig,
memory: {
proactive_extract: {
enabled: true,
min_tool_calls: 3,
namespace: 'global/facts',
},
daily_log: {
enabled: true,
namespace_prefix: 'memory',
},
},
});
expect(result.memory.proactive_extract.enabled).toBe(true);
expect(result.memory.proactive_extract.min_tool_calls).toBe(3);
expect(result.memory.proactive_extract.namespace).toBe('global/facts');
expect(result.memory.daily_log.enabled).toBe(true);
expect(result.memory.daily_log.namespace_prefix).toBe('memory');
});
});
describe('configSchema — compaction importance threshold', () => {
+9
View File
@@ -516,6 +516,15 @@ const memorySchema = z.object({
enabled: z.boolean().default(true),
dir: z.string().optional(), // Default: ~/.local/share/flynn/memory
auto_extract: z.boolean().default(true),
proactive_extract: z.object({
enabled: z.boolean().default(false),
min_tool_calls: z.number().min(0).max(50).default(1),
namespace: z.string().default('global'),
}).default({}),
daily_log: z.object({
enabled: z.boolean().default(false),
namespace_prefix: z.string().default('daily'),
}).default({}),
injection_strategy: z.enum(['all', 'recent', 'adaptive']).default('all'),
max_injection_tokens: z.number().min(100).max(10000).default(2000),
max_context_tokens: z.number().min(100).max(10000).default(2000),
+5
View File
@@ -315,6 +315,11 @@ export function createMessageRouter(deps: {
memoryAutoExtract: deps.config.memory?.auto_extract,
memoryInjectionStrategy: deps.config.memory?.injection_strategy,
memoryMaxInjectionTokens: deps.config.memory?.max_injection_tokens,
memoryProactiveExtractEnabled: deps.config.memory?.proactive_extract?.enabled,
memoryProactiveExtractMinToolCalls: deps.config.memory?.proactive_extract?.min_tool_calls,
memoryProactiveExtractNamespace: deps.config.memory?.proactive_extract?.namespace,
memoryDailyLogEnabled: deps.config.memory?.daily_log?.enabled,
memoryDailyLogNamespacePrefix: deps.config.memory?.daily_log?.namespace_prefix,
autoEscalate: deps.config.agents.auto_escalate,
autoEscalateTier: 'complex',
toolPolicyContext,
+5
View File
@@ -313,6 +313,11 @@ export class SessionBridge {
memoryAutoExtract: config?.memory?.auto_extract,
memoryInjectionStrategy: config?.memory?.injection_strategy,
memoryMaxInjectionTokens: config?.memory?.max_injection_tokens,
memoryProactiveExtractEnabled: config?.memory?.proactive_extract?.enabled,
memoryProactiveExtractMinToolCalls: config?.memory?.proactive_extract?.min_tool_calls,
memoryProactiveExtractNamespace: config?.memory?.proactive_extract?.namespace,
memoryDailyLogEnabled: config?.memory?.daily_log?.enabled,
memoryDailyLogNamespacePrefix: config?.memory?.daily_log?.namespace_prefix,
toolPolicyContext: {
agent: primaryTier,
provider: config?.models.default.provider,