refactor(01-01): extract memory initialization into src/daemon/memory.ts
- Create initMemory() factory encapsulating MemoryStore, VectorStore, HybridSearch, background indexer, and memory tools registration - Replace ~65 lines of inline memory init in startDaemon() with single initMemory() call - Clean up memory-specific imports from daemon/index.ts (MemoryStore, VectorStore, HybridSearch, createEmbeddingProvider, chunkText, contentHash, createMemoryTools)
This commit is contained in:
+3
-74
@@ -1,5 +1,6 @@
|
|||||||
import { Lifecycle } from './lifecycle.js';
|
import { Lifecycle } from './lifecycle.js';
|
||||||
import { createModelRouter } from './models.js';
|
import { createModelRouter } from './models.js';
|
||||||
|
import { initMemory } from './memory.js';
|
||||||
import { createMessageRouter } from './routing.js';
|
import { createMessageRouter } from './routing.js';
|
||||||
import type { Config } from '../config/index.js';
|
import type { Config } from '../config/index.js';
|
||||||
import type { AudioTranscriptionConfig } from '../models/media.js';
|
import type { AudioTranscriptionConfig } from '../models/media.js';
|
||||||
@@ -9,10 +10,6 @@ import { OutboundAttachmentCollector } from '../backends/native/attachments.js';
|
|||||||
import { SessionStore, SessionManager, parseDuration } from '../session/index.js';
|
import { SessionStore, SessionManager, parseDuration } from '../session/index.js';
|
||||||
import { HookEngine } from '../hooks/index.js';
|
import { HookEngine } from '../hooks/index.js';
|
||||||
import { ToolRegistry, ToolExecutor, ToolPolicy, allBuiltinTools, createWebSearchTools, createProcessTools, ProcessManager, BrowserManager, createBrowserTools, createSessionTools, createAgentsListTool, createMessageSendTool, createCronTools } from '../tools/index.js';
|
import { ToolRegistry, ToolExecutor, ToolPolicy, allBuiltinTools, createWebSearchTools, createProcessTools, ProcessManager, BrowserManager, createBrowserTools, createSessionTools, createAgentsListTool, createMessageSendTool, createCronTools } from '../tools/index.js';
|
||||||
import { MemoryStore } from '../memory/index.js';
|
|
||||||
import { VectorStore, HybridSearch, createEmbeddingProvider, chunkText, contentHash } from '../memory/index.js';
|
|
||||||
import type { EmbeddingProvider as EmbeddingProviderInterface } from '../memory/index.js';
|
|
||||||
import { createMemoryTools } from '../tools/builtin/index.js';
|
|
||||||
import { GatewayServer } from '../gateway/index.js';
|
import { GatewayServer } from '../gateway/index.js';
|
||||||
import { ChannelRegistry, TelegramAdapter, WebChatAdapter, DiscordAdapter, SlackAdapter, WhatsAppAdapter, PairingManager } from '../channels/index.js';
|
import { ChannelRegistry, TelegramAdapter, WebChatAdapter, DiscordAdapter, SlackAdapter, WhatsAppAdapter, PairingManager } from '../channels/index.js';
|
||||||
import { CronScheduler, WebhookHandler, HeartbeatMonitor, GmailWatcher } from '../automation/index.js';
|
import { CronScheduler, WebhookHandler, HeartbeatMonitor, GmailWatcher } from '../automation/index.js';
|
||||||
@@ -71,13 +68,6 @@ export async function startDaemon(config: Config): Promise<DaemonContext> {
|
|||||||
const dataDir = process.env.FLYNN_DATA_DIR ?? resolve(homedir(), '.local/share/flynn');
|
const dataDir = process.env.FLYNN_DATA_DIR ?? resolve(homedir(), '.local/share/flynn');
|
||||||
mkdirSync(dataDir, { recursive: true });
|
mkdirSync(dataDir, { recursive: true });
|
||||||
|
|
||||||
// Initialize memory store
|
|
||||||
const memoryDir = config.memory.dir ?? resolve(dataDir, 'memory');
|
|
||||||
mkdirSync(memoryDir, { recursive: true });
|
|
||||||
const memoryStore = config.memory.enabled
|
|
||||||
? new MemoryStore({ dir: memoryDir, maxContextTokens: config.memory.max_context_tokens })
|
|
||||||
: undefined;
|
|
||||||
|
|
||||||
// Initialize session store and manager
|
// Initialize session store and manager
|
||||||
const sessionStore = new SessionStore(resolve(dataDir, 'sessions.db'));
|
const sessionStore = new SessionStore(resolve(dataDir, 'sessions.db'));
|
||||||
const sessionManager = new SessionManager(sessionStore);
|
const sessionManager = new SessionManager(sessionStore);
|
||||||
@@ -113,69 +103,8 @@ export async function startDaemon(config: Config): Promise<DaemonContext> {
|
|||||||
toolRegistry.register(tool);
|
toolRegistry.register(tool);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Register memory tools if memory is enabled
|
// Initialize memory store, vector search, and memory tools
|
||||||
let hybridSearch: HybridSearch | undefined;
|
const { memoryStore, hybridSearch, memoryDir } = await initMemory({ config, dataDir, lifecycle, toolRegistry });
|
||||||
|
|
||||||
if (memoryStore && config.memory.embedding.enabled) {
|
|
||||||
try {
|
|
||||||
const embeddingProvider: EmbeddingProviderInterface = createEmbeddingProvider(config.memory.embedding);
|
|
||||||
const vectorStore = new VectorStore(resolve(dataDir, 'vectors.db'));
|
|
||||||
hybridSearch = new HybridSearch(
|
|
||||||
memoryStore,
|
|
||||||
vectorStore,
|
|
||||||
embeddingProvider,
|
|
||||||
config.memory.embedding.hybrid_weight,
|
|
||||||
);
|
|
||||||
|
|
||||||
// Background indexer: re-embed dirty namespaces every 30 seconds
|
|
||||||
const indexerInterval = setInterval(async () => {
|
|
||||||
const dirtyNamespaces = memoryStore.getDirtyNamespaces();
|
|
||||||
for (const ns of dirtyNamespaces) {
|
|
||||||
try {
|
|
||||||
const content = memoryStore.read(ns);
|
|
||||||
if (content.length === 0) {
|
|
||||||
vectorStore.deleteNamespace(ns);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
const hash = contentHash(content);
|
|
||||||
if (vectorStore.hasContentHash(ns, hash)) continue;
|
|
||||||
|
|
||||||
const chunks = chunkText(content, ns, {
|
|
||||||
chunkSize: config.memory.embedding.chunk_size,
|
|
||||||
chunkOverlap: config.memory.embedding.chunk_overlap,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (chunks.length > 0) {
|
|
||||||
const embeddings = await embeddingProvider.embed(chunks.map((c) => c.text));
|
|
||||||
vectorStore.upsertChunks(chunks, embeddings, hash);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.error(`Failed to index namespace "${ns}":`, err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, 30_000);
|
|
||||||
|
|
||||||
// Initial full index — mark all existing namespaces as dirty
|
|
||||||
memoryStore.markAllDirty();
|
|
||||||
|
|
||||||
lifecycle.onShutdown(async () => {
|
|
||||||
clearInterval(indexerInterval);
|
|
||||||
vectorStore.close();
|
|
||||||
console.log('Vector store closed');
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log(`Vector memory search enabled (provider=${config.memory.embedding.provider}, model=${config.memory.embedding.model})`);
|
|
||||||
} catch (err) {
|
|
||||||
console.error('Failed to initialize vector search:', err);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (memoryStore) {
|
|
||||||
for (const tool of createMemoryTools(memoryStore, hybridSearch)) {
|
|
||||||
toolRegistry.register(tool);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Register web search tool if configured with credentials
|
// Register web search tool if configured with credentials
|
||||||
if (config.web_search.api_key || config.web_search.endpoint) {
|
if (config.web_search.api_key || config.web_search.endpoint) {
|
||||||
|
|||||||
@@ -0,0 +1,99 @@
|
|||||||
|
import type { Config } from '../config/index.js';
|
||||||
|
import type { Lifecycle } from './lifecycle.js';
|
||||||
|
import { MemoryStore } from '../memory/index.js';
|
||||||
|
import { VectorStore, HybridSearch, createEmbeddingProvider, chunkText, contentHash } from '../memory/index.js';
|
||||||
|
import type { EmbeddingProvider as EmbeddingProviderInterface } from '../memory/index.js';
|
||||||
|
import { createMemoryTools } from '../tools/builtin/index.js';
|
||||||
|
import type { ToolRegistry } from '../tools/index.js';
|
||||||
|
import { resolve } from 'path';
|
||||||
|
import { mkdirSync } from 'fs';
|
||||||
|
|
||||||
|
export interface MemoryDeps {
|
||||||
|
config: Config;
|
||||||
|
dataDir: string;
|
||||||
|
lifecycle: Lifecycle;
|
||||||
|
toolRegistry: ToolRegistry;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MemoryResult {
|
||||||
|
memoryStore?: MemoryStore;
|
||||||
|
hybridSearch?: HybridSearch;
|
||||||
|
memoryDir: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function initMemory(deps: MemoryDeps): Promise<MemoryResult> {
|
||||||
|
const { config, dataDir, lifecycle, toolRegistry } = deps;
|
||||||
|
|
||||||
|
// Initialize memory store
|
||||||
|
const memoryDir = config.memory.dir ?? resolve(dataDir, 'memory');
|
||||||
|
mkdirSync(memoryDir, { recursive: true });
|
||||||
|
const memoryStore = config.memory.enabled
|
||||||
|
? new MemoryStore({ dir: memoryDir, maxContextTokens: config.memory.max_context_tokens })
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
// Register memory tools if memory is enabled
|
||||||
|
let hybridSearch: HybridSearch | undefined;
|
||||||
|
|
||||||
|
if (memoryStore && config.memory.embedding.enabled) {
|
||||||
|
try {
|
||||||
|
const embeddingProvider: EmbeddingProviderInterface = createEmbeddingProvider(config.memory.embedding);
|
||||||
|
const vectorStore = new VectorStore(resolve(dataDir, 'vectors.db'));
|
||||||
|
hybridSearch = new HybridSearch(
|
||||||
|
memoryStore,
|
||||||
|
vectorStore,
|
||||||
|
embeddingProvider,
|
||||||
|
config.memory.embedding.hybrid_weight,
|
||||||
|
);
|
||||||
|
|
||||||
|
// Background indexer: re-embed dirty namespaces every 30 seconds
|
||||||
|
const indexerInterval = setInterval(async () => {
|
||||||
|
const dirtyNamespaces = memoryStore.getDirtyNamespaces();
|
||||||
|
for (const ns of dirtyNamespaces) {
|
||||||
|
try {
|
||||||
|
const content = memoryStore.read(ns);
|
||||||
|
if (content.length === 0) {
|
||||||
|
vectorStore.deleteNamespace(ns);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const hash = contentHash(content);
|
||||||
|
if (vectorStore.hasContentHash(ns, hash)) continue;
|
||||||
|
|
||||||
|
const chunks = chunkText(content, ns, {
|
||||||
|
chunkSize: config.memory.embedding.chunk_size,
|
||||||
|
chunkOverlap: config.memory.embedding.chunk_overlap,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (chunks.length > 0) {
|
||||||
|
const embeddings = await embeddingProvider.embed(chunks.map((c) => c.text));
|
||||||
|
vectorStore.upsertChunks(chunks, embeddings, hash);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`Failed to index namespace "${ns}":`, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, 30_000);
|
||||||
|
|
||||||
|
// Initial full index — mark all existing namespaces as dirty
|
||||||
|
memoryStore.markAllDirty();
|
||||||
|
|
||||||
|
lifecycle.onShutdown(async () => {
|
||||||
|
clearInterval(indexerInterval);
|
||||||
|
vectorStore.close();
|
||||||
|
console.log('Vector store closed');
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(`Vector memory search enabled (provider=${config.memory.embedding.provider}, model=${config.memory.embedding.model})`);
|
||||||
|
} catch (err) {
|
||||||
|
console.error('Failed to initialize vector search:', err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (memoryStore) {
|
||||||
|
for (const tool of createMemoryTools(memoryStore, hybridSearch)) {
|
||||||
|
toolRegistry.register(tool);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { memoryStore, hybridSearch, memoryDir };
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user