William Valentin
114 lines
4.3 KiB
Python
114 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Local-only smoke test for the dry-run OpenVINO router classifier.
|
|
|
|
This script uses only synthetic fixture messages. It assumes router_classifier.py is
|
|
already running on localhost and never installs/enables a persistent service.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
DEFAULT_BASE_URL = "http://127.0.0.1:18819"
|
|
BUSY_FILE = Path("/sys/class/accel/accel0/device/npu_busy_time_us")
|
|
FIXTURE = Path(__file__).resolve().parent / "fixtures" / "atlas_hermes_messages.jsonl"
|
|
|
|
|
|
def npu_busy_time_us() -> int | None:
|
|
try:
|
|
return int(BUSY_FILE.read_text().strip())
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def get_json(url: str, timeout_s: float) -> dict[str, Any]:
|
|
with urllib.request.urlopen(url, timeout=timeout_s) as response: # noqa: S310 - localhost smoke URL
|
|
return json.loads(response.read().decode("utf-8"))
|
|
|
|
|
|
def post_json(url: str, payload: dict[str, Any], timeout_s: float) -> dict[str, Any]:
|
|
request = urllib.request.Request(
|
|
url,
|
|
data=json.dumps(payload).encode("utf-8"),
|
|
headers={"Content-Type": "application/json"},
|
|
method="POST",
|
|
)
|
|
with urllib.request.urlopen(request, timeout=timeout_s) as response: # noqa: S310 - localhost smoke URL
|
|
return json.loads(response.read().decode("utf-8"))
|
|
|
|
|
|
def load_fixture(limit: int) -> list[dict[str, Any]]:
|
|
rows = [json.loads(line) for line in FIXTURE.read_text().splitlines() if line.strip()]
|
|
return rows[:limit]
|
|
|
|
|
|
def assert_expected(result: dict[str, Any], expected: dict[str, Any]) -> list[str]:
|
|
failures: list[str] = []
|
|
labels = result.get("labels", {})
|
|
for key, value in expected.items():
|
|
actual_label = labels.get(key, {})
|
|
actual_value = actual_label.get("value")
|
|
if actual_value != value:
|
|
failures.append(f"{result.get('id')}: {key} expected {value!r}, got {actual_value!r}")
|
|
return failures
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Smoke-test a running localhost router classifier")
|
|
parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
|
|
parser.add_argument("--timeout-s", type=float, default=30.0)
|
|
parser.add_argument("--limit", type=int, default=10)
|
|
args = parser.parse_args()
|
|
|
|
if not args.base_url.startswith("http://127.0.0.1:") and not args.base_url.startswith("http://localhost:"):
|
|
raise SystemExit("refusing non-local base URL; this smoke is localhost-only")
|
|
|
|
before = npu_busy_time_us()
|
|
started = time.perf_counter()
|
|
try:
|
|
health = get_json(f"{args.base_url.rstrip('/')}/healthz", args.timeout_s)
|
|
labels = get_json(f"{args.base_url.rstrip('/')}/v1/labels", args.timeout_s)
|
|
rows = load_fixture(args.limit)
|
|
results = []
|
|
failures: list[str] = []
|
|
for row in rows:
|
|
result = post_json(
|
|
f"{args.base_url.rstrip('/')}/v1/classify",
|
|
{"id": row["id"], "text": row["text"], "options": {"include_evidence": False, "dry_run": True}},
|
|
args.timeout_s,
|
|
)
|
|
results.append(result)
|
|
failures.extend(assert_expected(result, row.get("expected", {})))
|
|
after = npu_busy_time_us()
|
|
except urllib.error.URLError as exc:
|
|
raise SystemExit(f"smoke failed: {exc}") from exc
|
|
|
|
response_npu_delta = sum((r.get("npu_busy_delta_us") or 0) for r in results)
|
|
outer_sysfs_delta = None if before is None or after is None else after - before
|
|
npu_proven = response_npu_delta > 0 and (outer_sysfs_delta is None or outer_sysfs_delta > 0)
|
|
summary = {
|
|
"ok": not failures,
|
|
"service": health.get("service"),
|
|
"mode": health.get("mode"),
|
|
"model": health.get("model"),
|
|
"label_count": len(labels.get("prototype_ids", [])),
|
|
"fixture_count": len(results),
|
|
"duration_ms": round((time.perf_counter() - started) * 1000, 3),
|
|
"response_npu_busy_delta_us": response_npu_delta,
|
|
"outer_sysfs_npu_busy_delta_us": outer_sysfs_delta,
|
|
"npu_proven": npu_proven,
|
|
"failures": failures,
|
|
}
|
|
print(json.dumps(summary, indent=2, sort_keys=True))
|
|
return 0 if not failures and npu_proven else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|