[verified] refresh OpenVINO router classifier prototype

openvino-classifier-npu/smoke_classifier.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python3
+"""Local-only smoke test for the dry-run OpenVINO router classifier.
+
+This script uses only synthetic fixture messages. It assumes router_classifier.py is
+already running on localhost and never installs/enables a persistent service.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+from typing import Any
+
+DEFAULT_BASE_URL = "http://127.0.0.1:18819"
+BUSY_FILE = Path("/sys/class/accel/accel0/device/npu_busy_time_us")
+FIXTURE = Path(__file__).resolve().parent / "fixtures" / "atlas_hermes_messages.jsonl"
+
+
+def npu_busy_time_us() -> int | None:
+    try:
+        return int(BUSY_FILE.read_text().strip())
+    except Exception:
+        return None
+
+
+def get_json(url: str, timeout_s: float) -> dict[str, Any]:
+    with urllib.request.urlopen(url, timeout=timeout_s) as response:  # noqa: S310 - localhost smoke URL
+        return json.loads(response.read().decode("utf-8"))
+
+
+def post_json(url: str, payload: dict[str, Any], timeout_s: float) -> dict[str, Any]:
+    request = urllib.request.Request(
+        url,
+        data=json.dumps(payload).encode("utf-8"),
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    with urllib.request.urlopen(request, timeout=timeout_s) as response:  # noqa: S310 - localhost smoke URL
+        return json.loads(response.read().decode("utf-8"))
+
+
+def load_fixture(limit: int) -> list[dict[str, Any]]:
+    rows = [json.loads(line) for line in FIXTURE.read_text().splitlines() if line.strip()]
+    return rows[:limit]
+
+
+def assert_expected(result: dict[str, Any], expected: dict[str, Any]) -> list[str]:
+    failures: list[str] = []
+    labels = result.get("labels", {})
+    for key, value in expected.items():
+        actual_label = labels.get(key, {})
+        actual_value = actual_label.get("value")
+        if actual_value != value:
+            failures.append(f"{result.get('id')}: {key} expected {value!r}, got {actual_value!r}")
+    return failures
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Smoke-test a running localhost router classifier")
+    parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
+    parser.add_argument("--timeout-s", type=float, default=30.0)
+    parser.add_argument("--limit", type=int, default=10)
+    args = parser.parse_args()
+
+    if not args.base_url.startswith("http://127.0.0.1:") and not args.base_url.startswith("http://localhost:"):
+        raise SystemExit("refusing non-local base URL; this smoke is localhost-only")
+
+    before = npu_busy_time_us()
+    started = time.perf_counter()
+    try:
+        health = get_json(f"{args.base_url.rstrip('/')}/healthz", args.timeout_s)
+        labels = get_json(f"{args.base_url.rstrip('/')}/v1/labels", args.timeout_s)
+        rows = load_fixture(args.limit)
+        results = []
+        failures: list[str] = []
+        for row in rows:
+            result = post_json(
+                f"{args.base_url.rstrip('/')}/v1/classify",
+                {"id": row["id"], "text": row["text"], "options": {"include_evidence": False, "dry_run": True}},
+                args.timeout_s,
+            )
+            results.append(result)
+            failures.extend(assert_expected(result, row.get("expected", {})))
+        after = npu_busy_time_us()
+    except urllib.error.URLError as exc:
+        raise SystemExit(f"smoke failed: {exc}") from exc
+
+    response_npu_delta = sum((r.get("npu_busy_delta_us") or 0) for r in results)
+    outer_sysfs_delta = None if before is None or after is None else after - before
+    npu_proven = response_npu_delta > 0 and (outer_sysfs_delta is None or outer_sysfs_delta > 0)
+    summary = {
+        "ok": not failures,
+        "service": health.get("service"),
+        "mode": health.get("mode"),
+        "model": health.get("model"),
+        "label_count": len(labels.get("prototype_ids", [])),
+        "fixture_count": len(results),
+        "duration_ms": round((time.perf_counter() - started) * 1000, 3),
+        "response_npu_busy_delta_us": response_npu_delta,
+        "outer_sysfs_npu_busy_delta_us": outer_sysfs_delta,
+        "npu_proven": npu_proven,
+        "failures": failures,
+    }
+    print(json.dumps(summary, indent=2, sort_keys=True))
+    return 0 if not failures and npu_proven else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())