feat(rag): switch Obsidian endpoint to NPU embeddings

docs/swarm-infrastructure.md

@@ -125,7 +125,7 @@ Host/user services:
 - `ollama.service` — `:18807`, legacy/CPU embeddings API fallback
 - `openvino-embeddings.service` — `:18817`, OpenVINO NPU embeddings API (`/v1/embeddings`, `/api/embed`, `/api/embeddings`)
 - `docker-health-endpoint.service` — `:18809`, read-only container health for n8n
-- `obsidian-reindex-endpoint.service` — `:18810`, Obsidian/RAG reindex trigger
+- `obsidian-reindex-endpoint.service` — `:18810`, Obsidian/RAG reindex trigger; default collection `obsidian_bge_npu` using OpenVINO NPU embeddings
 - `url-content-extractor.service` — `:18812`, YouTube/PDF/web extraction
 - `voice-memo-processor.service` — `:18813`, voice memo processing
 - `rag-embedding-health.service` — `:18814`, RAG/embedding health wrapper
@@ -144,9 +144,9 @@ Local REST API:
 RAG/vector store:
 
 - ChromaDB path: `~/.hermes/data/rag-search/chroma/`
-- Reindex state/progress: `~/.hermes/data/rag-search/obsidian_index_state.json` and `obsidian_reindex_progress.json`
+- Reindex state/progress: active BGE/NPU state in `~/.hermes/data/rag-search/obsidian_bge_npu_index_state.json` and `obsidian_bge_npu_reindex_progress.json`; legacy Ollama state in `obsidian_index_state.json` remains for comparison/fallback.
-- RAG query/reindex embedding backend: still Ollama on `:18807` with `nomic-embed-text` until a deliberate full Chroma rebuild/migration is run.
+- Active RAG query/reindex embedding backend: OpenVINO NPU embeddings service on `:18817`, currently `bge-base-en-v1.5-int8-ov`, collection `obsidian_bge_npu`.
-- RAG/embedding health probe backend: OpenVINO NPU embeddings service on `:18817`, currently `bge-base-en-v1.5-int8-ov`.
+- Legacy comparison/fallback collection: `obsidian`, built with Ollama on `:18807` using `nomic-embed-text`.
 - Reindex endpoint: `POST :18810/reindex` for incremental updates, `POST :18810/reindex?full=true` for full semantic rebuilds, `GET :18810/semantic-health` to verify vectors plus a search smoke test.
 
 ## Monitoring model

scripts/obsidian-reindex-server.py

@@ -26,14 +26,20 @@ from urllib.parse import parse_qs, urlparse
 
 PORT = int(os.environ.get("PORT", 18810))
 REINDEX_TIMEOUT = int(os.environ.get("REINDEX_TIMEOUT", "1800"))
+RAG_COLLECTION = os.environ.get("RAG_COLLECTION", "obsidian").strip() or "obsidian"
+RAG_EMBED_MODEL = os.environ.get("RAG_EMBED_MODEL", "nomic-embed-text").strip() or "nomic-embed-text"
+OLLAMA_BASE_URL = (os.environ.get("OLLAMA_BASE_URL") or "http://127.0.0.1:18807").rstrip("/")
 
 REINDEX_SCRIPT = str(
     Path.home()
     / ".hermes/skills/note-taking/rag-search/scripts/reindex_obsidian.sh"
 )
-STATE_FILE = (
+STATE_FILE = Path(
-    Path.home() / ".hermes/data/rag-search/obsidian_index_state.json"
+    os.environ.get("RAG_STATE_FILE")
+    or Path.home() / ".hermes/data/rag-search" / (
+        "obsidian_index_state.json" if RAG_COLLECTION == "obsidian" else f"{RAG_COLLECTION}_index_state.json"
     )
+).expanduser()
 SEARCH_SCRIPT = str(Path.home() / ".hermes/skills/note-taking/rag-search/scripts/search.py")
 VENV_PYTHON = str(Path.home() / ".hermes/skills/note-taking/rag-search/venv/bin/python")
 
@@ -50,11 +56,16 @@ def run_reindex(full: bool = False) -> dict:
         cmd = [REINDEX_SCRIPT]
         if full:
             cmd.append("--full")
+        env = os.environ.copy()
+        env.setdefault("RAG_COLLECTION", RAG_COLLECTION)
+        env.setdefault("RAG_EMBED_MODEL", RAG_EMBED_MODEL)
+        env.setdefault("OLLAMA_BASE_URL", OLLAMA_BASE_URL)
         result = subprocess.run(
             cmd,
             capture_output=True,
             text=True,
             timeout=REINDEX_TIMEOUT,
+            env=env,
         )
         if result.returncode != 0:
             return {
@@ -97,12 +108,16 @@ def run_semantic_search(query: str, top_k: int = 5) -> dict:
     if not query:
         return {"ok": False, "error": "query is required", "results": []}
     top_k = max(1, min(int(top_k or 5), 20))
+    env = os.environ.copy()
+    env.setdefault("RAG_COLLECTION", RAG_COLLECTION)
+    env.setdefault("RAG_EMBED_MODEL", RAG_EMBED_MODEL)
+    env.setdefault("OLLAMA_BASE_URL", OLLAMA_BASE_URL)
     result = subprocess.run(
         [
             VENV_PYTHON if Path(VENV_PYTHON).exists() else sys.executable,
             SEARCH_SCRIPT,
             "--index",
-            "obsidian",
+            RAG_COLLECTION,
             "--top-k",
             str(top_k),
             "--raw",
@@ -111,6 +126,7 @@ def run_semantic_search(query: str, top_k: int = 5) -> dict:
         capture_output=True,
         text=True,
         timeout=90,
+        env=env,
     )
     if result.returncode != 0:
         return {
@@ -125,7 +141,7 @@ def run_semantic_search(query: str, top_k: int = 5) -> dict:
     return {
         "ok": True,
         "query": query,
-        "index": payload.get("index", "obsidian"),
+        "index": payload.get("index", RAG_COLLECTION),
         "top_k": top_k,
         "result_count": len(results),
         "results": results,
@@ -144,7 +160,8 @@ def semantic_health() -> dict:
                 "note_count",
                 "vector_count",
                 "collection",
-                "chroma_path",
+                "embedding_backend",
+                "embedding_model",
                 "last_full_index",
                 "last_incremental_index",
             )

swarm-common/obsidian-reindex-endpoint.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Obsidian Vault Reindex Endpoint
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/python3 /home/will/lab/swarm/scripts/obsidian-reindex-server.py
+Restart=on-failure
+RestartSec=5
+Environment=PORT=18810
+Environment=RAG_COLLECTION=obsidian_bge_npu
+Environment=RAG_EMBED_MODEL=bge-base-en-v1.5-int8-ov
+Environment=OLLAMA_BASE_URL=http://127.0.0.1:18817
+
+[Install]
+WantedBy=default.target