From 00f8f74aac1ad92829fbb4d676ac346a9fc208d5 Mon Sep 17 00:00:00 2001 From: William Valentin Date: Mon, 9 Feb 2026 20:10:49 -0800 Subject: [PATCH] refactor(01-01): extract memory initialization into src/daemon/memory.ts - Create initMemory() factory encapsulating MemoryStore, VectorStore, HybridSearch, background indexer, and memory tools registration - Replace ~65 lines of inline memory init in startDaemon() with single initMemory() call - Clean up memory-specific imports from daemon/index.ts (MemoryStore, VectorStore, HybridSearch, createEmbeddingProvider, chunkText, contentHash, createMemoryTools) --- src/daemon/index.ts | 77 ++-------------------------------- src/daemon/memory.ts | 99 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 74 deletions(-) create mode 100644 src/daemon/memory.ts diff --git a/src/daemon/index.ts b/src/daemon/index.ts index 1deabf6..fb1bb5e 100644 --- a/src/daemon/index.ts +++ b/src/daemon/index.ts @@ -1,5 +1,6 @@ import { Lifecycle } from './lifecycle.js'; import { createModelRouter } from './models.js'; +import { initMemory } from './memory.js'; import { createMessageRouter } from './routing.js'; import type { Config } from '../config/index.js'; import type { AudioTranscriptionConfig } from '../models/media.js'; @@ -9,10 +10,6 @@ import { OutboundAttachmentCollector } from '../backends/native/attachments.js'; import { SessionStore, SessionManager, parseDuration } from '../session/index.js'; import { HookEngine } from '../hooks/index.js'; import { ToolRegistry, ToolExecutor, ToolPolicy, allBuiltinTools, createWebSearchTools, createProcessTools, ProcessManager, BrowserManager, createBrowserTools, createSessionTools, createAgentsListTool, createMessageSendTool, createCronTools } from '../tools/index.js'; -import { MemoryStore } from '../memory/index.js'; -import { VectorStore, HybridSearch, createEmbeddingProvider, chunkText, contentHash } from '../memory/index.js'; -import type { EmbeddingProvider as EmbeddingProviderInterface } from '../memory/index.js'; -import { createMemoryTools } from '../tools/builtin/index.js'; import { GatewayServer } from '../gateway/index.js'; import { ChannelRegistry, TelegramAdapter, WebChatAdapter, DiscordAdapter, SlackAdapter, WhatsAppAdapter, PairingManager } from '../channels/index.js'; import { CronScheduler, WebhookHandler, HeartbeatMonitor, GmailWatcher } from '../automation/index.js'; @@ -71,13 +68,6 @@ export async function startDaemon(config: Config): Promise { const dataDir = process.env.FLYNN_DATA_DIR ?? resolve(homedir(), '.local/share/flynn'); mkdirSync(dataDir, { recursive: true }); - // Initialize memory store - const memoryDir = config.memory.dir ?? resolve(dataDir, 'memory'); - mkdirSync(memoryDir, { recursive: true }); - const memoryStore = config.memory.enabled - ? new MemoryStore({ dir: memoryDir, maxContextTokens: config.memory.max_context_tokens }) - : undefined; - // Initialize session store and manager const sessionStore = new SessionStore(resolve(dataDir, 'sessions.db')); const sessionManager = new SessionManager(sessionStore); @@ -113,69 +103,8 @@ export async function startDaemon(config: Config): Promise { toolRegistry.register(tool); } - // Register memory tools if memory is enabled - let hybridSearch: HybridSearch | undefined; - - if (memoryStore && config.memory.embedding.enabled) { - try { - const embeddingProvider: EmbeddingProviderInterface = createEmbeddingProvider(config.memory.embedding); - const vectorStore = new VectorStore(resolve(dataDir, 'vectors.db')); - hybridSearch = new HybridSearch( - memoryStore, - vectorStore, - embeddingProvider, - config.memory.embedding.hybrid_weight, - ); - - // Background indexer: re-embed dirty namespaces every 30 seconds - const indexerInterval = setInterval(async () => { - const dirtyNamespaces = memoryStore.getDirtyNamespaces(); - for (const ns of dirtyNamespaces) { - try { - const content = memoryStore.read(ns); - if (content.length === 0) { - vectorStore.deleteNamespace(ns); - continue; - } - - const hash = contentHash(content); - if (vectorStore.hasContentHash(ns, hash)) continue; - - const chunks = chunkText(content, ns, { - chunkSize: config.memory.embedding.chunk_size, - chunkOverlap: config.memory.embedding.chunk_overlap, - }); - - if (chunks.length > 0) { - const embeddings = await embeddingProvider.embed(chunks.map((c) => c.text)); - vectorStore.upsertChunks(chunks, embeddings, hash); - } - } catch (err) { - console.error(`Failed to index namespace "${ns}":`, err); - } - } - }, 30_000); - - // Initial full index — mark all existing namespaces as dirty - memoryStore.markAllDirty(); - - lifecycle.onShutdown(async () => { - clearInterval(indexerInterval); - vectorStore.close(); - console.log('Vector store closed'); - }); - - console.log(`Vector memory search enabled (provider=${config.memory.embedding.provider}, model=${config.memory.embedding.model})`); - } catch (err) { - console.error('Failed to initialize vector search:', err); - } - } - - if (memoryStore) { - for (const tool of createMemoryTools(memoryStore, hybridSearch)) { - toolRegistry.register(tool); - } - } + // Initialize memory store, vector search, and memory tools + const { memoryStore, hybridSearch, memoryDir } = await initMemory({ config, dataDir, lifecycle, toolRegistry }); // Register web search tool if configured with credentials if (config.web_search.api_key || config.web_search.endpoint) { diff --git a/src/daemon/memory.ts b/src/daemon/memory.ts new file mode 100644 index 0000000..2cb1f58 --- /dev/null +++ b/src/daemon/memory.ts @@ -0,0 +1,99 @@ +import type { Config } from '../config/index.js'; +import type { Lifecycle } from './lifecycle.js'; +import { MemoryStore } from '../memory/index.js'; +import { VectorStore, HybridSearch, createEmbeddingProvider, chunkText, contentHash } from '../memory/index.js'; +import type { EmbeddingProvider as EmbeddingProviderInterface } from '../memory/index.js'; +import { createMemoryTools } from '../tools/builtin/index.js'; +import type { ToolRegistry } from '../tools/index.js'; +import { resolve } from 'path'; +import { mkdirSync } from 'fs'; + +export interface MemoryDeps { + config: Config; + dataDir: string; + lifecycle: Lifecycle; + toolRegistry: ToolRegistry; +} + +export interface MemoryResult { + memoryStore?: MemoryStore; + hybridSearch?: HybridSearch; + memoryDir: string; +} + +export async function initMemory(deps: MemoryDeps): Promise { + const { config, dataDir, lifecycle, toolRegistry } = deps; + + // Initialize memory store + const memoryDir = config.memory.dir ?? resolve(dataDir, 'memory'); + mkdirSync(memoryDir, { recursive: true }); + const memoryStore = config.memory.enabled + ? new MemoryStore({ dir: memoryDir, maxContextTokens: config.memory.max_context_tokens }) + : undefined; + + // Register memory tools if memory is enabled + let hybridSearch: HybridSearch | undefined; + + if (memoryStore && config.memory.embedding.enabled) { + try { + const embeddingProvider: EmbeddingProviderInterface = createEmbeddingProvider(config.memory.embedding); + const vectorStore = new VectorStore(resolve(dataDir, 'vectors.db')); + hybridSearch = new HybridSearch( + memoryStore, + vectorStore, + embeddingProvider, + config.memory.embedding.hybrid_weight, + ); + + // Background indexer: re-embed dirty namespaces every 30 seconds + const indexerInterval = setInterval(async () => { + const dirtyNamespaces = memoryStore.getDirtyNamespaces(); + for (const ns of dirtyNamespaces) { + try { + const content = memoryStore.read(ns); + if (content.length === 0) { + vectorStore.deleteNamespace(ns); + continue; + } + + const hash = contentHash(content); + if (vectorStore.hasContentHash(ns, hash)) continue; + + const chunks = chunkText(content, ns, { + chunkSize: config.memory.embedding.chunk_size, + chunkOverlap: config.memory.embedding.chunk_overlap, + }); + + if (chunks.length > 0) { + const embeddings = await embeddingProvider.embed(chunks.map((c) => c.text)); + vectorStore.upsertChunks(chunks, embeddings, hash); + } + } catch (err) { + console.error(`Failed to index namespace "${ns}":`, err); + } + } + }, 30_000); + + // Initial full index — mark all existing namespaces as dirty + memoryStore.markAllDirty(); + + lifecycle.onShutdown(async () => { + clearInterval(indexerInterval); + vectorStore.close(); + console.log('Vector store closed'); + }); + + console.log(`Vector memory search enabled (provider=${config.memory.embedding.provider}, model=${config.memory.embedding.model})`); + } catch (err) { + console.error('Failed to initialize vector search:', err); + } + } + + if (memoryStore) { + for (const tool of createMemoryTools(memoryStore, hybridSearch)) { + toolRegistry.register(tool); + } + } + + return { memoryStore, hybridSearch, memoryDir }; +}