diff --git a/docs/swarm-infrastructure.md b/docs/swarm-infrastructure.md index a44423e..b780285 100644 --- a/docs/swarm-infrastructure.md +++ b/docs/swarm-infrastructure.md @@ -125,7 +125,7 @@ Host/user services: - `ollama.service` — `:18807`, legacy/CPU embeddings API fallback - `openvino-embeddings.service` — `:18817`, OpenVINO NPU embeddings API (`/v1/embeddings`, `/api/embed`, `/api/embeddings`) - `docker-health-endpoint.service` — `:18809`, read-only container health for n8n -- `obsidian-reindex-endpoint.service` — `:18810`, Obsidian/RAG reindex trigger +- `obsidian-reindex-endpoint.service` — `:18810`, Obsidian/RAG reindex trigger; default collection `obsidian_bge_npu` using OpenVINO NPU embeddings - `url-content-extractor.service` — `:18812`, YouTube/PDF/web extraction - `voice-memo-processor.service` — `:18813`, voice memo processing - `rag-embedding-health.service` — `:18814`, RAG/embedding health wrapper @@ -144,9 +144,9 @@ Local REST API: RAG/vector store: - ChromaDB path: `~/.hermes/data/rag-search/chroma/` -- Reindex state/progress: `~/.hermes/data/rag-search/obsidian_index_state.json` and `obsidian_reindex_progress.json` -- RAG query/reindex embedding backend: still Ollama on `:18807` with `nomic-embed-text` until a deliberate full Chroma rebuild/migration is run. -- RAG/embedding health probe backend: OpenVINO NPU embeddings service on `:18817`, currently `bge-base-en-v1.5-int8-ov`. +- Reindex state/progress: active BGE/NPU state in `~/.hermes/data/rag-search/obsidian_bge_npu_index_state.json` and `obsidian_bge_npu_reindex_progress.json`; legacy Ollama state in `obsidian_index_state.json` remains for comparison/fallback. +- Active RAG query/reindex embedding backend: OpenVINO NPU embeddings service on `:18817`, currently `bge-base-en-v1.5-int8-ov`, collection `obsidian_bge_npu`. +- Legacy comparison/fallback collection: `obsidian`, built with Ollama on `:18807` using `nomic-embed-text`. - Reindex endpoint: `POST :18810/reindex` for incremental updates, `POST :18810/reindex?full=true` for full semantic rebuilds, `GET :18810/semantic-health` to verify vectors plus a search smoke test. ## Monitoring model diff --git a/scripts/obsidian-reindex-server.py b/scripts/obsidian-reindex-server.py index 57e8e5d..6133b8c 100644 --- a/scripts/obsidian-reindex-server.py +++ b/scripts/obsidian-reindex-server.py @@ -26,14 +26,20 @@ from urllib.parse import parse_qs, urlparse PORT = int(os.environ.get("PORT", 18810)) REINDEX_TIMEOUT = int(os.environ.get("REINDEX_TIMEOUT", "1800")) +RAG_COLLECTION = os.environ.get("RAG_COLLECTION", "obsidian").strip() or "obsidian" +RAG_EMBED_MODEL = os.environ.get("RAG_EMBED_MODEL", "nomic-embed-text").strip() or "nomic-embed-text" +OLLAMA_BASE_URL = (os.environ.get("OLLAMA_BASE_URL") or "http://127.0.0.1:18807").rstrip("/") REINDEX_SCRIPT = str( Path.home() / ".hermes/skills/note-taking/rag-search/scripts/reindex_obsidian.sh" ) -STATE_FILE = ( - Path.home() / ".hermes/data/rag-search/obsidian_index_state.json" -) +STATE_FILE = Path( + os.environ.get("RAG_STATE_FILE") + or Path.home() / ".hermes/data/rag-search" / ( + "obsidian_index_state.json" if RAG_COLLECTION == "obsidian" else f"{RAG_COLLECTION}_index_state.json" + ) +).expanduser() SEARCH_SCRIPT = str(Path.home() / ".hermes/skills/note-taking/rag-search/scripts/search.py") VENV_PYTHON = str(Path.home() / ".hermes/skills/note-taking/rag-search/venv/bin/python") @@ -50,11 +56,16 @@ def run_reindex(full: bool = False) -> dict: cmd = [REINDEX_SCRIPT] if full: cmd.append("--full") + env = os.environ.copy() + env.setdefault("RAG_COLLECTION", RAG_COLLECTION) + env.setdefault("RAG_EMBED_MODEL", RAG_EMBED_MODEL) + env.setdefault("OLLAMA_BASE_URL", OLLAMA_BASE_URL) result = subprocess.run( cmd, capture_output=True, text=True, timeout=REINDEX_TIMEOUT, + env=env, ) if result.returncode != 0: return { @@ -97,12 +108,16 @@ def run_semantic_search(query: str, top_k: int = 5) -> dict: if not query: return {"ok": False, "error": "query is required", "results": []} top_k = max(1, min(int(top_k or 5), 20)) + env = os.environ.copy() + env.setdefault("RAG_COLLECTION", RAG_COLLECTION) + env.setdefault("RAG_EMBED_MODEL", RAG_EMBED_MODEL) + env.setdefault("OLLAMA_BASE_URL", OLLAMA_BASE_URL) result = subprocess.run( [ VENV_PYTHON if Path(VENV_PYTHON).exists() else sys.executable, SEARCH_SCRIPT, "--index", - "obsidian", + RAG_COLLECTION, "--top-k", str(top_k), "--raw", @@ -111,6 +126,7 @@ def run_semantic_search(query: str, top_k: int = 5) -> dict: capture_output=True, text=True, timeout=90, + env=env, ) if result.returncode != 0: return { @@ -125,7 +141,7 @@ def run_semantic_search(query: str, top_k: int = 5) -> dict: return { "ok": True, "query": query, - "index": payload.get("index", "obsidian"), + "index": payload.get("index", RAG_COLLECTION), "top_k": top_k, "result_count": len(results), "results": results, @@ -144,7 +160,8 @@ def semantic_health() -> dict: "note_count", "vector_count", "collection", - "chroma_path", + "embedding_backend", + "embedding_model", "last_full_index", "last_incremental_index", ) diff --git a/swarm-common/obsidian-reindex-endpoint.service b/swarm-common/obsidian-reindex-endpoint.service new file mode 100644 index 0000000..4928f23 --- /dev/null +++ b/swarm-common/obsidian-reindex-endpoint.service @@ -0,0 +1,16 @@ +[Unit] +Description=Obsidian Vault Reindex Endpoint +After=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/python3 /home/will/lab/swarm/scripts/obsidian-reindex-server.py +Restart=on-failure +RestartSec=5 +Environment=PORT=18810 +Environment=RAG_COLLECTION=obsidian_bge_npu +Environment=RAG_EMBED_MODEL=bge-base-en-v1.5-int8-ov +Environment=OLLAMA_BASE_URL=http://127.0.0.1:18817 + +[Install] +WantedBy=default.target