feat: add persistent memory system (Phase 2)
Implement file-based persistent memory with read/write/search tools: - MemoryStore with namespace-scoped JSON storage - memory-read, memory-write, memory-search builtin tools - Auto-extraction of facts during context compaction - Configurable via memory.enabled, memory.dir, memory.max_context_tokens
This commit is contained in:
William Valentin
@@ -0,0 +1,2 @@
|
||||
export { MemoryStore } from './store.js';
|
||||
export type { MemoryStoreConfig, SearchResult } from './store.js';
|
||||
@@ -0,0 +1,195 @@
|
||||
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
||||
import { MemoryStore } from './store.js';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync, readFileSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { tmpdir } from 'os';
|
||||
|
||||
describe('MemoryStore', () => {
|
||||
let dir: string;
|
||||
let store: MemoryStore;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = mkdtempSync(join(tmpdir(), 'flynn-memory-test-'));
|
||||
store = new MemoryStore({ dir, maxContextTokens: 4096 });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('read', () => {
|
||||
it('returns empty string for non-existent namespace', () => {
|
||||
const result = store.read('nonexistent');
|
||||
expect(result).toBe('');
|
||||
});
|
||||
|
||||
it('returns file contents for existing namespace', () => {
|
||||
// Pre-populate a memory file directly on disk
|
||||
const filePath = join(dir, 'notes.md');
|
||||
writeFileSync(filePath, 'Remember to check logs daily');
|
||||
|
||||
const result = store.read('notes');
|
||||
expect(result).toBe('Remember to check logs daily');
|
||||
});
|
||||
|
||||
it('handles nested namespaces (sessions/abc123)', () => {
|
||||
// Pre-populate a nested memory file
|
||||
const nestedDir = join(dir, 'sessions');
|
||||
mkdirSync(nestedDir, { recursive: true });
|
||||
writeFileSync(join(nestedDir, 'abc123.md'), 'Session context data');
|
||||
|
||||
const result = store.read('sessions/abc123');
|
||||
expect(result).toBe('Session context data');
|
||||
});
|
||||
});
|
||||
|
||||
describe('write', () => {
|
||||
it('creates file and directories on first write', () => {
|
||||
store.write('projects/alpha', 'Initial content', 'replace');
|
||||
|
||||
const filePath = join(dir, 'projects', 'alpha.md');
|
||||
const content = readFileSync(filePath, 'utf-8');
|
||||
expect(content).toContain('Initial content');
|
||||
});
|
||||
|
||||
it('appends content with newline separator in append mode', () => {
|
||||
store.write('journal', 'First entry', 'replace');
|
||||
store.write('journal', 'Second entry', 'append');
|
||||
|
||||
const result = store.read('journal');
|
||||
expect(result).toContain('First entry');
|
||||
expect(result).toContain('Second entry');
|
||||
|
||||
// Verify entries are separated by newline
|
||||
const lines = result.split('\n').filter((l: string) => l.trim().length > 0);
|
||||
expect(lines.length).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('replaces content in replace mode', () => {
|
||||
store.write('config', 'Old content', 'replace');
|
||||
store.write('config', 'New content', 'replace');
|
||||
|
||||
const result = store.read('config');
|
||||
expect(result).toContain('New content');
|
||||
expect(result).not.toContain('Old content');
|
||||
});
|
||||
|
||||
it('handles nested namespace paths', () => {
|
||||
store.write('deep/nested/path', 'Deep content', 'replace');
|
||||
|
||||
const result = store.read('deep/nested/path');
|
||||
expect(result).toContain('Deep content');
|
||||
});
|
||||
});
|
||||
|
||||
describe('search', () => {
|
||||
beforeEach(() => {
|
||||
store.write('notes', 'The quick brown fox jumps over the lazy dog\nAnother line of text\nFox sightings are common here', 'replace');
|
||||
store.write('journal', 'Today I saw a fox in the garden\nIt was a beautiful day\nThe weather was sunny', 'replace');
|
||||
store.write('config', 'No relevant animals mentioned\nJust some settings here', 'replace');
|
||||
});
|
||||
|
||||
it('finds matching lines case-insensitively', () => {
|
||||
const results = store.search('fox');
|
||||
|
||||
// Should match lines containing "fox" regardless of case
|
||||
expect(results.length).toBeGreaterThanOrEqual(1);
|
||||
|
||||
const contents = results.map((r) => r.content.toLowerCase());
|
||||
for (const content of contents) {
|
||||
expect(content).toContain('fox');
|
||||
}
|
||||
});
|
||||
|
||||
it('returns context lines around matches', () => {
|
||||
const results = store.search('fox');
|
||||
|
||||
// Each result should have a context field with surrounding text
|
||||
for (const result of results) {
|
||||
expect(result.context).toBeDefined();
|
||||
expect(result.context.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it('searches across multiple namespaces', () => {
|
||||
const results = store.search('fox');
|
||||
|
||||
// Should find matches in both 'notes' and 'journal' namespaces
|
||||
const namespaces = new Set(results.map((r) => r.namespace));
|
||||
expect(namespaces.size).toBeGreaterThanOrEqual(2);
|
||||
});
|
||||
|
||||
it('returns empty array when no matches', () => {
|
||||
const results = store.search('xyznonexistent');
|
||||
expect(results).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('listNamespaces', () => {
|
||||
it('returns empty array when no files exist', () => {
|
||||
const namespaces = store.listNamespaces();
|
||||
expect(namespaces).toEqual([]);
|
||||
});
|
||||
|
||||
it('lists all namespaces from existing files', () => {
|
||||
store.write('notes', 'Some notes', 'replace');
|
||||
store.write('journal', 'A journal entry', 'replace');
|
||||
store.write('config', 'Configuration data', 'replace');
|
||||
|
||||
const namespaces = store.listNamespaces();
|
||||
|
||||
expect(namespaces).toContain('notes');
|
||||
expect(namespaces).toContain('journal');
|
||||
expect(namespaces).toContain('config');
|
||||
expect(namespaces).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('includes nested namespaces', () => {
|
||||
store.write('sessions/abc123', 'Session data', 'replace');
|
||||
store.write('sessions/def456', 'Another session', 'replace');
|
||||
store.write('global', 'Global notes', 'replace');
|
||||
|
||||
const namespaces = store.listNamespaces();
|
||||
|
||||
expect(namespaces).toContain('sessions/abc123');
|
||||
expect(namespaces).toContain('sessions/def456');
|
||||
expect(namespaces).toContain('global');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getContextForPrompt', () => {
|
||||
it('returns empty string when no memory files exist', () => {
|
||||
const context = store.getContextForPrompt();
|
||||
expect(context).toBe('');
|
||||
});
|
||||
|
||||
it('includes user and global memory under headings', () => {
|
||||
store.write('user', 'User prefers concise answers', 'replace');
|
||||
store.write('global', 'System-wide knowledge base', 'replace');
|
||||
|
||||
const context = store.getContextForPrompt();
|
||||
|
||||
// Should contain both memory sections with some form of heading
|
||||
expect(context).toContain('User Memory');
|
||||
expect(context).toContain('User prefers concise answers');
|
||||
expect(context).toContain('Global Memory');
|
||||
expect(context).toContain('System-wide knowledge base');
|
||||
});
|
||||
|
||||
it('truncates content to stay within maxContextTokens', () => {
|
||||
// Create a store with a very small token limit (50 tokens ≈ 200 chars)
|
||||
const smallStore = new MemoryStore({ dir, maxContextTokens: 50 });
|
||||
|
||||
// Write a large amount of content to user and global (which getContextForPrompt reads)
|
||||
const longContent = 'A'.repeat(5000);
|
||||
smallStore.write('user', longContent, 'replace');
|
||||
smallStore.write('global', longContent, 'replace');
|
||||
|
||||
const context = smallStore.getContextForPrompt();
|
||||
|
||||
// 50 tokens * 4 chars/token = 200 chars max
|
||||
expect(context.length).toBeLessThanOrEqual(200);
|
||||
expect(context.length).toBeLessThan(longContent.length);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,216 @@
|
||||
import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync } from 'fs';
|
||||
import { join, relative, dirname } from 'path';
|
||||
|
||||
/**
|
||||
* Configuration for the MemoryStore.
|
||||
*/
|
||||
export interface MemoryStoreConfig {
|
||||
/** Base directory for memory files. */
|
||||
dir: string;
|
||||
/** Maximum tokens to inject into system prompt per turn. */
|
||||
maxContextTokens: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* A single search result from scanning memory files.
|
||||
*/
|
||||
export interface SearchResult {
|
||||
/** Namespace the match was found in (e.g. 'user', 'sessions/abc123'). */
|
||||
namespace: string;
|
||||
/** 1-based line number of the match. */
|
||||
line: number;
|
||||
/** The matched line content. */
|
||||
content: string;
|
||||
/** Lines of context around the match (1 line above + match + 1 line below). */
|
||||
context: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Manages persistent markdown memory files on disk.
|
||||
*
|
||||
* Directory layout:
|
||||
* {dir}/
|
||||
* ├── global.md # Cross-session knowledge
|
||||
* ├── user.md # User preferences and facts
|
||||
* └── sessions/
|
||||
* └── {session_id}.md # Per-session notes
|
||||
*
|
||||
* Namespaces map directly to filenames: 'user' -> user.md,
|
||||
* 'sessions/abc123' -> sessions/abc123.md.
|
||||
*/
|
||||
export class MemoryStore {
|
||||
private _config: MemoryStoreConfig;
|
||||
|
||||
constructor(config: MemoryStoreConfig) {
|
||||
this._config = config;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Read the contents of a memory file by namespace.
|
||||
* Returns an empty string if the file does not exist.
|
||||
*/
|
||||
read(namespace: string): string {
|
||||
const filePath = this._namespacePath(namespace);
|
||||
if (!existsSync(filePath)) {
|
||||
return '';
|
||||
}
|
||||
return readFileSync(filePath, 'utf-8');
|
||||
}
|
||||
|
||||
/**
|
||||
* Write content to a memory file.
|
||||
*
|
||||
* @param namespace - Target namespace (e.g. 'user', 'sessions/abc123').
|
||||
* @param content - Markdown content to write.
|
||||
* @param mode - 'append' adds content after a newline separator;
|
||||
* 'replace' overwrites the file entirely.
|
||||
*/
|
||||
write(namespace: string, content: string, mode: 'append' | 'replace'): void {
|
||||
const filePath = this._namespacePath(namespace);
|
||||
|
||||
// Ensure parent directories exist on first write
|
||||
const dir = dirname(filePath);
|
||||
if (!existsSync(dir)) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
|
||||
if (mode === 'replace') {
|
||||
writeFileSync(filePath, content, 'utf-8');
|
||||
} else {
|
||||
// Append: read existing content, add a newline separator, then the new content
|
||||
const existing = existsSync(filePath)
|
||||
? readFileSync(filePath, 'utf-8')
|
||||
: '';
|
||||
const separator = existing.length > 0 ? '\n' : '';
|
||||
writeFileSync(filePath, existing + separator + content, 'utf-8');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Search across all memory files for a keyword or phrase.
|
||||
* Performs case-insensitive line-by-line matching.
|
||||
* Returns matching lines with 1 line of context above and below.
|
||||
*/
|
||||
search(query: string): SearchResult[] {
|
||||
const results: SearchResult[] = [];
|
||||
const namespaces = this.listNamespaces();
|
||||
const lowerQuery = query.toLowerCase();
|
||||
|
||||
for (const namespace of namespaces) {
|
||||
const content = this.read(namespace);
|
||||
if (content.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const lines = content.split('\n');
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (lines[i].toLowerCase().includes(lowerQuery)) {
|
||||
// Gather 1 line of context above and below
|
||||
const contextLines: string[] = [];
|
||||
if (i > 0) {
|
||||
contextLines.push(lines[i - 1]);
|
||||
}
|
||||
contextLines.push(lines[i]);
|
||||
if (i < lines.length - 1) {
|
||||
contextLines.push(lines[i + 1]);
|
||||
}
|
||||
|
||||
results.push({
|
||||
namespace,
|
||||
line: i + 1, // 1-based
|
||||
content: lines[i],
|
||||
context: contextLines.join('\n'),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* List all available namespaces by scanning the memory directory
|
||||
* recursively for `.md` files.
|
||||
*
|
||||
* Returns namespace strings (file path relative to the base dir, without
|
||||
* the `.md` extension). E.g. ['global', 'user', 'sessions/abc123'].
|
||||
*/
|
||||
listNamespaces(): string[] {
|
||||
if (!existsSync(this._config.dir)) {
|
||||
return [];
|
||||
}
|
||||
return this._scanDir(this._config.dir);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build memory context suitable for injection into a system prompt.
|
||||
*
|
||||
* Reads `user.md` and `global.md`, formats them under markdown headings,
|
||||
* and truncates to stay within {@link MemoryStoreConfig.maxContextTokens}
|
||||
* (estimated at 4 characters per token).
|
||||
*/
|
||||
getContextForPrompt(): string {
|
||||
const userMemory = this.read('user');
|
||||
const globalMemory = this.read('global');
|
||||
|
||||
// Nothing to inject
|
||||
if (userMemory.length === 0 && globalMemory.length === 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
const sections: string[] = [];
|
||||
|
||||
if (userMemory.length > 0) {
|
||||
sections.push(`## User Memory\n\n${userMemory}`);
|
||||
}
|
||||
if (globalMemory.length > 0) {
|
||||
sections.push(`## Global Memory\n\n${globalMemory}`);
|
||||
}
|
||||
|
||||
const full = sections.join('\n\n');
|
||||
|
||||
// Truncate to fit within the token budget (estimate: 4 chars ≈ 1 token)
|
||||
const maxChars = this._config.maxContextTokens * 4;
|
||||
if (full.length <= maxChars) {
|
||||
return full;
|
||||
}
|
||||
return full.slice(0, maxChars);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Private helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Resolve a namespace string to an absolute file path. */
|
||||
private _namespacePath(namespace: string): string {
|
||||
return join(this._config.dir, `${namespace}.md`);
|
||||
}
|
||||
|
||||
/** Recursively scan a directory for .md files, returning namespace strings. */
|
||||
private _scanDir(dir: string): string[] {
|
||||
const namespaces: string[] = [];
|
||||
const entries = readdirSync(dir);
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(dir, entry);
|
||||
const stat = statSync(fullPath);
|
||||
|
||||
if (stat.isDirectory()) {
|
||||
// Recurse into subdirectories
|
||||
namespaces.push(...this._scanDir(fullPath));
|
||||
} else if (stat.isFile() && entry.endsWith('.md')) {
|
||||
// Convert absolute path back to namespace: strip base dir and .md extension
|
||||
const rel = relative(this._config.dir, fullPath);
|
||||
const namespace = rel.slice(0, -3); // remove '.md'
|
||||
namespaces.push(namespace);
|
||||
}
|
||||
}
|
||||
|
||||
return namespaces;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user