From 748b930a1f6340942d7e205d234ded63bd39b2c4 Mon Sep 17 00:00:00 2001 From: William Valentin Date: Sat, 31 Jan 2026 22:00:10 -0800 Subject: [PATCH] docs: add all-future-features implementation plan --- .gitignore | 3 + .tmp-render.yaml | 665 +----------------- .tmp-values.yaml | 27 +- docs/plans/2026-02-01-all-future-features.md | 681 +++++++++++++++++++ 4 files changed, 688 insertions(+), 688 deletions(-) create mode 100644 docs/plans/2026-02-01-all-future-features.md diff --git a/.gitignore b/.gitignore index 2db921e..4da2df4 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,9 @@ worktrees/ # TypeScript incremental build info *.tsbuildinfo +# Local scratch files +.tmp-* + # Test binary *.test diff --git a/.tmp-render.yaml b/.tmp-render.yaml index 8bc4a1f..d1a615b 100644 --- a/.tmp-render.yaml +++ b/.tmp-render.yaml @@ -1,663 +1,2 @@ ---- -# Source: tline/templates/secret.yaml.tpl -apiVersion: v1 -kind: Secret -metadata: - name: tline-tline-secrets - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" -type: Opaque -data: - POSTGRES_PASSWORD: Y2hhbmdlLW1l - MINIO_ACCESS_KEY_ID: bWluaW9hZG1pbg== - MINIO_SECRET_ACCESS_KEY: bWluaW9hZG1pbg==--- -apiVersion: v1 -kind: Secret -metadata: - name: tline-tline-registry - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" -type: kubernetes.io/dockerconfigjson -data: - .dockerconfigjson: eyJhdXRocyI6eyJyZWdpc3RyeS5sYW46NTAwMCI6eyJhdXRoIjoiZFRwdyIsImVtYWlsIjoiZUBleGFtcGxlLmNvbSIsInBhc3N3b3JkIjoicCIsInVzZXJuYW1lIjoidSJ9fX0= ---- -# Source: tline/templates/configmap.yaml.tpl -apiVersion: v1 -kind: ConfigMap -metadata: - name: tline-tline-config - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" -data: - APP_NAME: "flux" - NEXT_PUBLIC_APP_NAME: "flux" - QUEUE_NAME: "tline" - DATABASE_URL: "postgres://tline:change-me@tline-tline-postgres:5432/tline" - REDIS_URL: "redis://tline-tline-redis:6379" - MINIO_INTERNAL_ENDPOINT: "http://tline-tline-minio:9000" - MINIO_PUBLIC_ENDPOINT_TS: "https://minio.tailxyz.ts.net" - MINIO_REGION: "us-east-1" - MINIO_BUCKET: "media" - MINIO_PRESIGN_EXPIRES_SECONDS: "900" ---- -# Source: tline/templates/minio.yaml.tpl -apiVersion: v1 -kind: Service -metadata: - name: tline-tline-minio - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" - app.kubernetes.io/component: minio -spec: - type: ClusterIP - ports: - - name: s3 - port: 9000 - targetPort: s3 - - name: console - port: 9001 - targetPort: console - selector: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: minio ---- -# Source: tline/templates/postgres.yaml.tpl -apiVersion: v1 -kind: Service -metadata: - name: tline-tline-postgres - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" -spec: - type: ClusterIP - ports: - - name: postgres - port: 5432 - targetPort: postgres - selector: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: postgres ---- -# Source: tline/templates/redis.yaml.tpl -apiVersion: v1 -kind: Service -metadata: - name: tline-tline-redis - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" -spec: - type: ClusterIP - ports: - - name: redis - port: 6379 - targetPort: redis - selector: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: redis ---- -# Source: tline/templates/web.yaml.tpl -apiVersion: v1 -kind: Service -metadata: - name: tline-tline-web - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" - app.kubernetes.io/component: web -spec: - type: ClusterIP - ports: - - name: http - port: 3000 - targetPort: http - selector: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: web ---- -# Source: tline/templates/redis.yaml.tpl -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tline-tline-redis - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" - app.kubernetes.io/component: redis -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: redis - template: - metadata: - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: redis - spec: - imagePullSecrets: - - name: "tline-tline-registry" - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-class - operator: In - values: - - compute - containers: - - name: redis - image: "redis:7" - imagePullPolicy: IfNotPresent - ports: - - name: redis - containerPort: 6379 - resources: - limits: - cpu: 300m - memory: 512Mi - requests: - cpu: 50m - memory: 128Mi ---- -# Source: tline/templates/web.yaml.tpl -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tline-tline-web - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" - app.kubernetes.io/component: web -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: web - template: - metadata: - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: web - spec: - imagePullSecrets: - - name: "tline-tline-registry" - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-class - operator: In - values: - - compute - containers: - - name: web - image: "registry.lan:5000/tline-web:dev" - imagePullPolicy: IfNotPresent - ports: - - name: http - containerPort: 3000 - envFrom: - - configMapRef: - name: tline-tline-config - env: - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: tline-tline-secrets - key: POSTGRES_PASSWORD - - name: MINIO_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: tline-tline-secrets - key: MINIO_ACCESS_KEY_ID - - name: MINIO_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: tline-tline-secrets - key: MINIO_SECRET_ACCESS_KEY - readinessProbe: - httpGet: - path: /api/healthz - port: http - initialDelaySeconds: 5 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /api/healthz - port: http - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - limits: - cpu: 1000m - memory: 1Gi - requests: - cpu: 200m - memory: 256Mi ---- -# Source: tline/templates/worker.yaml.tpl -apiVersion: apps/v1 -kind: Deployment -metadata: - name: tline-tline-worker - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" - app.kubernetes.io/component: worker -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: worker - template: - metadata: - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: worker - spec: - imagePullSecrets: - - name: "tline-tline-registry" - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-class - operator: In - values: - - compute - containers: - - name: worker - image: "registry.lan:5000/tline-worker:dev" - imagePullPolicy: IfNotPresent - envFrom: - - configMapRef: - name: tline-tline-config - env: - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: tline-tline-secrets - key: POSTGRES_PASSWORD - - name: MINIO_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: tline-tline-secrets - key: MINIO_ACCESS_KEY_ID - - name: MINIO_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: tline-tline-secrets - key: MINIO_SECRET_ACCESS_KEY - resources: - limits: - cpu: 2000m - memory: 2Gi - requests: - cpu: 500m - memory: 1Gi ---- -# Source: tline/templates/minio.yaml.tpl -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: tline-tline-minio - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" - app.kubernetes.io/component: minio -spec: - serviceName: tline-tline-minio - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: minio - template: - metadata: - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: minio - spec: - imagePullSecrets: - - name: "tline-tline-registry" - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-class - operator: In - values: - - compute - containers: - - name: minio - image: "minio/minio:RELEASE.2024-01-16T16-07-38Z" - imagePullPolicy: IfNotPresent - args: - - server - - /data - - "--console-address=:9001" - ports: - - name: s3 - containerPort: 9000 - - name: console - containerPort: 9001 - env: - - name: MINIO_ROOT_USER - valueFrom: - secretKeyRef: - name: tline-tline-secrets - key: MINIO_ACCESS_KEY_ID - - name: MINIO_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: tline-tline-secrets - key: MINIO_SECRET_ACCESS_KEY - readinessProbe: - httpGet: - path: /minio/health/ready - port: s3 - initialDelaySeconds: 10 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /minio/health/live - port: s3 - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - limits: - cpu: 1500m - memory: 2Gi - requests: - cpu: 250m - memory: 512Mi - volumeMounts: - - name: data - mountPath: /data - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "200Gi" ---- -# Source: tline/templates/postgres.yaml.tpl -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: tline-tline-postgres - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" - app.kubernetes.io/component: postgres -spec: - serviceName: tline-tline-postgres - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: postgres - template: - metadata: - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: postgres - spec: - imagePullSecrets: - - name: "tline-tline-registry" - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-class - operator: In - values: - - compute - containers: - - name: postgres - image: "postgres:16" - imagePullPolicy: IfNotPresent - ports: - - name: postgres - containerPort: 5432 - env: - - name: POSTGRES_USER - value: "tline" - - name: POSTGRES_DB - value: "tline" - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: tline-tline-secrets - key: POSTGRES_PASSWORD - readinessProbe: - exec: - command: - - sh - - -c - - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" - initialDelaySeconds: 5 - periodSeconds: 5 - livenessProbe: - exec: - command: - - sh - - -c - - pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - limits: - cpu: 1500m - memory: 2Gi - requests: - cpu: 500m - memory: 1Gi - volumeMounts: - - name: data - mountPath: /var/lib/postgresql/data - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: "20Gi" ---- -# Source: tline/templates/ingress-tailscale.yaml.tpl -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: tline-tline-web - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" - app.kubernetes.io/component: web - annotations: -spec: - ingressClassName: tailscale - tls: - - hosts: - - "app" - rules: - - host: "app" - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: tline-tline-web - port: - number: 3000 ---- -# Source: tline/templates/ingress-tailscale.yaml.tpl -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: tline-tline-minio - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" - app.kubernetes.io/component: minio - annotations: -spec: - ingressClassName: tailscale - tls: - - hosts: - - "minio" - rules: - - host: "minio" - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: tline-tline-minio - port: - number: 9000 ---- -# Source: tline/templates/ingress-tailscale.yaml.tpl -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: tline-tline-minio-console - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" - app.kubernetes.io/component: minio - annotations: -spec: - ingressClassName: tailscale - tls: - - hosts: - - "minio-console" - rules: - - host: "minio-console" - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: tline-tline-minio - port: - number: 9001 ---- -# Source: tline/templates/job-migrate.yaml.tpl -apiVersion: batch/v1 -kind: Job -metadata: - name: tline-tline-migrate - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/managed-by: Helm - helm.sh/chart: "tline-0.1.0" - app.kubernetes.io/component: migrate - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-weight": "-10" - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -spec: - backoffLimit: 3 - template: - metadata: - labels: - app.kubernetes.io/name: tline - app.kubernetes.io/instance: tline - app.kubernetes.io/component: migrate - spec: - restartPolicy: Never - imagePullSecrets: - - name: "tline-tline-registry" - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-class - operator: In - values: - - compute - containers: - - name: migrate - image: "registry.lan:5000/tline-worker:dev" - imagePullPolicy: IfNotPresent - command: - - bun - - run - - packages/db/src/migrate.ts - envFrom: - - configMapRef: - name: tline-tline-config - env: - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: tline-tline-secrets - key: POSTGRES_PASSWORD +# Temporary file used during local helm rendering. +# This file is intentionally empty in-repo; real rendered output should not be committed. diff --git a/.tmp-values.yaml b/.tmp-values.yaml index 43e706a..4f4352d 100644 --- a/.tmp-values.yaml +++ b/.tmp-values.yaml @@ -1,25 +1,2 @@ -secrets: - postgres: - password: "change-me" - minio: - accessKeyId: "minioadmin" - secretAccessKey: "minioadmin" - -images: - web: - repository: registry.lan:5000/tline-web - tag: dev - worker: - repository: registry.lan:5000/tline-worker - tag: dev - -global: - tailscale: - tailnetFQDN: "tailxyz.ts.net" - -registrySecret: - create: true - server: "registry.lan:5000" - username: "u" - password: "p" - email: "e@example.com" +# Temporary file used during local helm rendering. +# This file is intentionally empty in-repo; real values should not be committed. diff --git a/docs/plans/2026-02-01-all-future-features.md b/docs/plans/2026-02-01-all-future-features.md new file mode 100644 index 0000000..5371bc6 --- /dev/null +++ b/docs/plans/2026-02-01-all-future-features.md @@ -0,0 +1,681 @@ +# All Future Features Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Implement all items in `PLAN.md` under "Future Features (Tracked)" for the `porthole` app. + +**Architecture:** Add features incrementally behind small, testable boundaries: shared-secret admin auth for writes, a normalized derived-variant model for media, a transcoding pipeline (MP4 first), tagging/albums and metadata overrides with audit logging, dedupe + moments, GPS extraction + map UI (no reverse geocoding), endpoint selection for presigned URLs, and CI-based multi-arch builds. + +**Tech Stack:** Bun workspaces, Next.js API route handlers (`apps/web/app/api/**/route.ts`), Node worker + BullMQ (`apps/worker/src/jobs.ts`), Postgres migrations (`packages/db/migrations/*.sql`), MinIO (S3) clients (`packages/minio/src/index.ts`), Helm (`helm/porthole/*`). + +## Preconditions / Ground Rules + +- Do not mutate or delete anything under `originals/`. +- Prefer additive schema changes first; deprecate old columns after compatibility is maintained. +- Use Bun’s test runner (`bun test`) for new TypeScript tests. +- Keep Pi constraints: CPU-heavy work stays in worker; keep transcoding concurrency low. + +## Phase 0: Test Harness + Repo Hygiene + +### Task 0.1: Add Bun test runner scripts + +**Files:** + +- Modify: `package.json` +- Create: `apps/web/src/__tests__/smoke.test.ts` + +**Step 1: Write failing test** + +Create `apps/web/src/__tests__/smoke.test.ts`: + +```ts +import { test, expect } from "bun:test"; + +test("bun test runs", () => { + expect(1 + 1).toBe(2); +}); +``` + +**Step 2: Run test to verify it fails** + +Run: `bun test` +Expected: FAIL (no tests configured / command missing) + +**Step 3: Write minimal implementation** + +Add script to `package.json`: + +```json +{ + "scripts": { + "test": "bun test" + } +} +``` + +**Step 4: Run test to verify it passes** + +Run: `bun test` +Expected: PASS + +**Step 5: Commit** + +```bash +git add package.json apps/web/src/__tests__/smoke.test.ts +git commit -m "test: add bun test runner" +``` + +## Phase 1: Shared-Secret Admin Auth (Write Protection) + +### Task 1.1: Add ADMIN_TOKEN env + helpers + +**Files:** + +- Modify: `packages/config/src/index.ts` +- Create: `packages/config/src/adminAuth.ts` +- Test: `packages/config/src/adminAuth.test.ts` + +**Step 1: Write failing test** + +Create `packages/config/src/adminAuth.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { isAdminRequest } from "./adminAuth"; + +test("isAdminRequest returns false when ADMIN_TOKEN unset", () => { + expect(isAdminRequest({ adminToken: undefined }, { headerToken: "x" })).toBe( + false, + ); +}); + +test("isAdminRequest returns true when header token matches", () => { + expect( + isAdminRequest({ adminToken: "secret" }, { headerToken: "secret" }), + ).toBe(true); +}); +``` + +**Step 2: Run test to verify it fails** + +Run: `bun test packages/config/src/adminAuth.test.ts` +Expected: FAIL (module missing) + +**Step 3: Write minimal implementation** + +Create `packages/config/src/adminAuth.ts`: + +```ts +export function isAdminRequest( + env: { adminToken: string | undefined }, + input: { headerToken: string | null | undefined }, +) { + if (!env.adminToken) return false; + return input.headerToken === env.adminToken; +} +``` + +Extend `packages/config/src/index.ts` to parse `ADMIN_TOKEN` (optional) and export `getAdminToken()`. + +**Step 4: Run test to verify it passes** + +Run: `bun test packages/config/src/adminAuth.test.ts` +Expected: PASS + +**Step 5: Commit** + +```bash +git add packages/config/src/index.ts packages/config/src/adminAuth.ts packages/config/src/adminAuth.test.ts +git commit -m "feat: add admin token config and auth helper" +``` + +### Task 1.2: Enforce admin on mutation API routes + +**Files:** + +- Modify: `apps/web/app/api/imports/route.ts` +- Modify: `apps/web/app/api/imports/[id]/upload/route.ts` +- Modify: `apps/web/app/api/imports/[id]/scan-minio/route.ts` +- Test: `apps/web/src/__tests__/admin-gates-imports.test.ts` + +**Step 1: Write failing test** + +Create `apps/web/src/__tests__/admin-gates-imports.test.ts`: + +```ts +import { test, expect } from "bun:test"; + +// This test intentionally asserts the handler behavior by calling the route function. +// It will require exporting a small pure helper from each route in the implementation. + +test("imports POST rejects when missing admin token", async () => { + const { handleCreateImport } = await import("../../app/api/imports/handlers"); + const res = await handleCreateImport({ adminOk: false, body: {} }); + expect(res.status).toBe(401); +}); +``` + +**Step 2: Run test to verify it fails** + +Run: `bun test apps/web/src/__tests__/admin-gates-imports.test.ts` +Expected: FAIL (handlers module missing) + +**Step 3: Write minimal implementation** + +- Create `apps/web/app/api/imports/handlers.ts` exporting pure functions that return `{ status, body }` for tests. +- Update `apps/web/app/api/imports/route.ts` to: + - read `X-Porthole-Admin-Token` + - compute adminOk via `@tline/config` helper + - reject with 401 `{ error: "admin_required" }` when not admin +- Repeat pattern for upload + scan routes. + +**Step 4: Run test to verify it passes** + +Run: `bun test apps/web/src/__tests__/admin-gates-imports.test.ts` +Expected: PASS + +**Step 5: Commit** + +```bash +git add apps/web/app/api/imports/route.ts apps/web/app/api/imports/handlers.ts \ + apps/web/app/api/imports/[id]/upload/route.ts apps/web/app/api/imports/[id]/scan-minio/route.ts \ + apps/web/src/__tests__/admin-gates-imports.test.ts +git commit -m "feat: require admin token for ingestion endpoints" +``` + +## Phase 2: Derived Variants Model (Thumbs/Posters/Video) + +### Task 2.1: Add derived variants table + minimal writer/reader + +**Files:** + +- Create: `packages/db/migrations/0003_asset_variants.sql` +- Modify: `apps/web/app/api/assets/[id]/url/route.ts` +- Modify: `apps/worker/src/jobs.ts` +- Test: `apps/web/src/__tests__/variant-url-404.test.ts` + +**Schema (migration):** + +```sql +CREATE TYPE IF NOT EXISTS asset_variant_kind AS ENUM ( + 'thumb', + 'poster', + 'video_mp4' +); + +CREATE TABLE IF NOT EXISTS asset_variants ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + asset_id uuid NOT NULL REFERENCES assets(id) ON DELETE CASCADE, + kind asset_variant_kind NOT NULL, + size int NOT NULL, + key text NOT NULL, + mime_type text NOT NULL, + width int, + height int, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE(asset_id, kind, size) +); + +CREATE INDEX IF NOT EXISTS asset_variants_asset_id_idx ON asset_variants(asset_id); +``` + +**Step 1: Write failing test** + +Create `apps/web/src/__tests__/variant-url-404.test.ts`: + +```ts +import { test, expect } from "bun:test"; + +test("/api/assets/:id/url returns 404 when requested variant missing", async () => { + const { pickVariantKey } = + await import("../../app/api/assets/[id]/url/variant"); + const key = pickVariantKey({ variants: [] }, { kind: "thumb", size: 256 }); + expect(key).toBeNull(); +}); +``` + +**Step 2: Run test to verify it fails** + +Run: `bun test apps/web/src/__tests__/variant-url-404.test.ts` +Expected: FAIL (module missing) + +**Step 3: Write minimal implementation** + +- Create `apps/web/app/api/assets/[id]/url/variant.ts`: + +```ts +export function pickVariantKey( + input: { variants: Array<{ kind: string; size: number; key: string }> }, + req: { kind: string; size: number }, +) { + const v = input.variants.find( + (x) => x.kind === req.kind && x.size === req.size, + ); + return v?.key ?? null; +} +``` + +- Update `apps/web/app/api/assets/[id]/url/route.ts` to support query: + - `kind=original|thumb|poster|video_mp4` + - `size=` (required for non-original) + - Keep backward-compatible `variant=thumb_small|thumb_med|poster|original` for now. + +- Update `apps/worker/src/jobs.ts` to insert rows into `asset_variants` when it uploads thumbs/posters. + +**Step 4: Run test to verify it passes** + +Run: `bun test apps/web/src/__tests__/variant-url-404.test.ts` +Expected: PASS + +**Step 5: Commit** + +```bash +git add packages/db/migrations/0003_asset_variants.sql \ + apps/web/app/api/assets/[id]/url/route.ts apps/web/app/api/assets/[id]/url/variant.ts \ + apps/web/src/__tests__/variant-url-404.test.ts apps/worker/src/jobs.ts +git commit -m "feat: add asset variants table and URL selection" +``` + +### Task 2.2: Multiple thumb + poster sizes + +**Files:** + +- Modify: `apps/worker/src/jobs.ts` +- Modify: `apps/web/app/api/assets/[id]/url/route.ts` +- Test: `apps/worker/src/__tests__/variants-sizes.test.ts` + +**Step 1: Write failing test** + +Create `apps/worker/src/__tests__/variants-sizes.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { computeImageVariantPlan } from "../variants"; + +test("computeImageVariantPlan includes 256 and 768 thumbs", () => { + expect(computeImageVariantPlan()).toEqual([ + { kind: "thumb", size: 256 }, + { kind: "thumb", size: 768 }, + ]); +}); +``` + +**Step 2: Run test to verify it fails** + +Run: `bun test apps/worker/src/__tests__/variants-sizes.test.ts` +Expected: FAIL (module missing) + +**Step 3: Write minimal implementation** + +- Create `apps/worker/src/variants.ts` with exported `computeImageVariantPlan()` and `computeVideoPosterPlan()`. +- Refactor `apps/worker/src/jobs.ts` to use these plans and generate additional poster size(s) (e.g. 256 + 768). +- Insert each uploaded object into `asset_variants` with (kind,size,key,mime_type,width,height). + +**Step 4: Run test to verify it passes** + +Run: `bun test apps/worker/src/__tests__/variants-sizes.test.ts` +Expected: PASS + +**Step 5: Commit** + +```bash +git add apps/worker/src/jobs.ts apps/worker/src/variants.ts apps/worker/src/__tests__/variants-sizes.test.ts +git commit -m "feat: generate multiple thumbs and posters" +``` + +## Phase 3: Video Transcoding + Prefer-Derived Playback + +### Task 3.1: Add MP4 transcode worker job + +**Files:** + +- Modify: `packages/queue/src/index.ts` +- Modify: `apps/worker/src/jobs.ts` +- Test: `apps/worker/src/__tests__/transcode-plan.test.ts` + +**Step 1: Write failing test** + +Create `apps/worker/src/__tests__/transcode-plan.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { shouldTranscodeToMp4 } from "../transcode"; + +test("transcode runs for non-mp4 videos", () => { + expect(shouldTranscodeToMp4({ mimeType: "video/x-matroska" })).toBe(true); +}); + +test("transcode skips for mp4", () => { + expect(shouldTranscodeToMp4({ mimeType: "video/mp4" })).toBe(false); +}); +``` + +**Step 2: Run test to verify it fails** + +Run: `bun test apps/worker/src/__tests__/transcode-plan.test.ts` +Expected: FAIL + +**Step 3: Write minimal implementation** + +- Create `apps/worker/src/transcode.ts` implementing `shouldTranscodeToMp4`. +- Add BullMQ job payload + enqueue helper (e.g. `enqueueTranscodeVideoMp4({ assetId })`). +- In `handleProcessAsset` for video, enqueue mp4 transcode when needed. +- Implement ffmpeg transcode to `derived/video/${assetId}/mp4_720p.mp4` (H.264 + AAC, fast preset). +- Insert into `asset_variants` as `kind='video_mp4', size=720, mime_type='video/mp4'`. +- Keep concurrency low (1) in worker for transcodes. + +**Step 4: Run test to verify it passes** + +Run: `bun test apps/worker/src/__tests__/transcode-plan.test.ts` +Expected: PASS + +**Step 5: Commit** + +```bash +git add packages/queue/src/index.ts apps/worker/src/jobs.ts apps/worker/src/transcode.ts \ + apps/worker/src/__tests__/transcode-plan.test.ts +git commit -m "feat: add mp4 transcode job and variant record" +``` + +### Task 3.2: Prefer derived in URL endpoint + viewer + +**Files:** + +- Modify: `apps/web/app/api/assets/[id]/url/route.ts` +- Modify: `apps/web/app/components/MediaPanel.tsx` +- Modify: `apps/web/app/components/Viewer.tsx` +- Test: `apps/web/src/__tests__/prefer-derived.test.ts` + +**Step 1: Write failing test** + +Create `apps/web/src/__tests__/prefer-derived.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { pickVideoPlaybackVariant } from "../../app/lib/playback"; + +test("prefer mp4 derived over original", () => { + const picked = pickVideoPlaybackVariant({ + originalMimeType: "video/x-matroska", + variants: [{ kind: "video_mp4", size: 720, key: "derived/video/a.mp4" }], + }); + expect(picked).toEqual({ kind: "video_mp4", size: 720 }); +}); +``` + +**Step 2: Run test to verify it fails** + +Run: `bun test apps/web/src/__tests__/prefer-derived.test.ts` +Expected: FAIL + +**Step 3: Write minimal implementation** + +- Create `apps/web/app/lib/playback.ts` implementing deterministic selection. +- Update viewer to: + - ask server for `kind=video_mp4&size=720` first + - fall back to `original` +- Keep existing poster behavior. + +**Step 4: Run test to verify it passes** + +Run: `bun test apps/web/src/__tests__/prefer-derived.test.ts` +Expected: PASS + +**Step 5: Commit** + +```bash +git add apps/web/app/api/assets/[id]/url/route.ts apps/web/app/lib/playback.ts \ + apps/web/app/components/MediaPanel.tsx apps/web/app/components/Viewer.tsx \ + apps/web/src/__tests__/prefer-derived.test.ts +git commit -m "feat: prefer derived mp4 playback with fallback" +``` + +## Phase 4: Tags + Albums + +### Task 4.1: Schema for tags/albums + audit log + +**Files:** + +- Create: `packages/db/migrations/0004_tags_albums_audit.sql` + +**Migration (example):** + +```sql +CREATE TABLE IF NOT EXISTS tags ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + name text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS asset_tags ( + asset_id uuid NOT NULL REFERENCES assets(id) ON DELETE CASCADE, + tag_id uuid NOT NULL REFERENCES tags(id) ON DELETE CASCADE, + PRIMARY KEY(asset_id, tag_id) +); + +CREATE TABLE IF NOT EXISTS albums ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + name text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS album_assets ( + album_id uuid NOT NULL REFERENCES albums(id) ON DELETE CASCADE, + asset_id uuid NOT NULL REFERENCES assets(id) ON DELETE CASCADE, + ord int, + PRIMARY KEY(album_id, asset_id) +); + +CREATE TABLE IF NOT EXISTS audit_log ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + actor text NOT NULL, + action text NOT NULL, + entity_type text NOT NULL, + entity_id uuid, + payload jsonb, + created_at timestamptz NOT NULL DEFAULT now() +); +``` + +**Verification:** run migrator (k8s migrate job or local script) and ensure no SQL errors. + +**Commit:** `git commit -m "feat: add tags, albums, and audit log tables"` + +### Task 4.2: Admin API for tags and albums + +**Files:** + +- Create: `apps/web/app/api/tags/route.ts` +- Create: `apps/web/app/api/albums/route.ts` +- Create: `apps/web/app/api/albums/[id]/assets/route.ts` +- Test: `apps/web/src/__tests__/tags-admin-auth.test.ts` + +**Steps:** + +- RED: test that POST without admin returns 401 +- GREEN: implement CRUD (minimal: list + create; album add/remove assets) +- REFACTOR: write audit_log rows on each mutation + +**Commit:** `feat: add admin tags and albums APIs` + +### Task 4.3: UI wiring for tags/albums + +**Files:** + +- Modify: `apps/web/app/admin/page.tsx` +- Modify: `apps/web/app/components/MediaPanel.tsx` + +**Steps:** + +- Add minimal admin form to set admin token in browser (sessionStorage) and to create/list tags and albums. +- Add UI on asset detail to assign tags, and to add asset to album. +- Keep UX resilient (errors render inline, don’t crash). + +**Commit:** `feat: add tags/albums UI` + +## Phase 5: Metadata Overrides + Timeline Uses Overrides + +### Task 5.1: Override table + API + +**Files:** + +- Create: `packages/db/migrations/0005_asset_overrides.sql` +- Create: `apps/web/app/api/assets/[id]/override-capture-ts/route.ts` +- Modify: `apps/web/app/api/tree/route.ts` +- Modify: `apps/web/app/api/assets/route.ts` + +**Migration:** table `asset_overrides(asset_id PK, capture_ts_utc_override timestamptz, capture_offset_minutes_override int, created_at...)`. + +**Steps:** + +- RED: test route rejects without admin +- GREEN: implement POST to set override and insert audit_log +- GREEN: update aggregation queries to use `COALESCE(overrides.capture_ts_utc_override, assets.capture_ts_utc)` + +**Commit:** `feat: add capture time overrides and apply in queries` + +### Task 5.2: UI for capture-time override + +**Files:** + +- Modify: `apps/web/app/components/Viewer.tsx` +- Modify: `apps/web/app/components/MediaPanel.tsx` + +**Steps:** + +- Add form to set ISO timestamp override and submit to API. +- Display current effective timestamp and base timestamp. + +**Commit:** `feat: add UI for capture time override` + +## Phase 6: GPS Extraction + Map UI (No Reverse Geocode) + +### Task 6.1: Add gps fields + extraction + +**Files:** + +- Create: `packages/db/migrations/0006_assets_gps.sql` +- Modify: `apps/worker/src/jobs.ts` + +**Steps:** + +- Add columns `gps_lat double precision`, `gps_lon double precision` (nullable) +- Parse ExifTool GPS fields for images (and where available for videos) and store them. + +**Commit:** `feat: extract and store GPS coords` + +### Task 6.2: Map UI + +**Files:** + +- Create: `apps/web/app/map/page.tsx` +- Modify: `apps/web/app/page.tsx` + +**Steps:** + +- Show a simple map view with markers for assets that have GPS. +- If tiles unavailable, show a clear fallback message. + +**Commit:** `feat: add map page for GPS assets` + +## Phase 7: Dedupe by Hash + Moments + +### Task 7.1: Hash table + compute sha256 + +**Files:** + +- Create: `packages/db/migrations/0007_asset_hashes.sql` +- Modify: `apps/worker/src/jobs.ts` + +**Steps:** + +- During download to temp file, compute sha256 and store it. +- Add unique index on `(bucket, sha256)` optionally (careful for partial/unknown). + +**Commit:** `feat: compute asset sha256 for dedupe` + +### Task 7.2: Dedupe detection + API + +**Files:** + +- Create: `apps/web/app/api/assets/[id]/dupes/route.ts` +- Modify: `apps/web/app/components/MediaPanel.tsx` + +**Steps:** + +- Endpoint returns assets with same sha256. +- UI indicates duplicates. + +**Commit:** `feat: expose and display duplicates` + +### Task 7.3: Moments clustering + +**Files:** + +- Create: `apps/web/app/api/moments/route.ts` +- Create: `apps/web/app/lib/moments.ts` +- Test: `apps/web/src/__tests__/moments.test.ts` + +**Steps:** + +- RED: test that assets within 30 minutes cluster together +- GREEN: implement clustering +- Wire UI to show moments as sub-groups + +**Commit:** `feat: add day moments clustering` + +## Phase 8: Presign Endpoint Selection (LAN vs Tailnet) + +### Task 8.1: Add endpoint mode to config and presign + +**Files:** + +- Modify: `packages/minio/src/index.ts` +- Modify: `packages/config/src/index.ts` +- Modify: `apps/web/app/api/assets/[id]/url/route.ts` + +**Steps:** + +- Add env for `MINIO_PUBLIC_ENDPOINT_LAN` and `MINIO_ENDPOINT_MODE=tailnet|lan|auto`. +- If `endpoint=lan|tailnet` query param is provided, force that. +- In `auto`, use tailnet as safe default. + +**Commit:** `feat: support lan/tailnet endpoint selection for presigned URLs` + +## Phase 9: Storage Policies (Derived Lifecycle) + CI Builds + +### Task 9.1: Optional MinIO lifecycle policy job + +**Files:** + +- Modify: `helm/porthole/values.yaml` +- Modify: `helm/porthole/templates/job-ensure-bucket.yaml.tpl` +- Create: `helm/porthole/templates/job-apply-lifecycle.yaml.tpl` + +**Steps:** + +- Add optional Job to apply lifecycle rules for prefixes `thumbs/` and `derived/` (expire after N days) without touching `originals/`. + +**Commit:** `feat: add optional lifecycle policy job` + +### Task 9.2: Add CI pipeline for multi-arch builds + +**Files:** + +- Create: `.gitea/workflows/build-images.yml` (or alternative supported by your CI) +- Modify: `README.md` + +**Steps:** + +- Build and push multi-arch images for `apps/web` and `apps/worker`. +- Run: `bun run typecheck`. +- Run: `bash run_tests.sh` (Go tests) to keep repo green. + +**Commit:** `ci: build and push multi-arch images` + +## Verification Checklist (Per Phase) + +- `bun test` +- `bun run typecheck` +- `bash run_tests.sh` +- Helm template renders: `helm template porthole helm/porthole -f your-values.yaml --namespace porthole`