feat(councils): add scaffold prompt hooks and checked-in scaffold files

docs/councils/ai-council-production-scaffold.json

@@ -0,0 +1,259 @@
+{
+  "generated_at": "2026-02-21T18:05:48.200896",
+  "version": "1.0.0",
+  "entities": {
+    "councils": {
+      "D": {
+        "name": "Deterministic",
+        "bias": [
+          "clarity",
+          "feasibility",
+          "low_drift"
+        ]
+      },
+      "P": {
+        "name": "Probabilistic",
+        "bias": [
+          "novelty",
+          "exploration",
+          "reframing"
+        ]
+      }
+    },
+    "roles": [
+      "arbiter",
+      "free_thinker",
+      "grounder",
+      "writer"
+    ],
+    "meta_roles": [
+      "meta_arbiter",
+      "meta_writer"
+    ]
+  },
+  "schemas": {
+    "input_package": {
+      "required": [
+        "problem_statement",
+        "success_definition",
+        "constraints",
+        "context",
+        "timebox"
+      ],
+      "optional": [
+        "stakeholders",
+        "non_goals",
+        "risk_tolerance",
+        "glossary"
+      ]
+    },
+    "idea_card": {
+      "id": "string",
+      "title": "string",
+      "one_liner": "string",
+      "mechanism": [
+        "string"
+      ],
+      "assumptions": [
+        "string"
+      ],
+      "constraints": [
+        "string"
+      ],
+      "mve": {
+        "steps": [
+          "string"
+        ],
+        "success_metric": "string",
+        "time_cost": "string",
+        "resources": [
+          "string"
+        ]
+      },
+      "risks": [
+        "string"
+      ],
+      "upside": "string",
+      "tags": [
+        "string"
+      ]
+    },
+    "bridge_packet": {
+      "top_ideas": [
+        "idea_card"
+      ],
+      "fragile_assumptions": [
+        "string"
+      ],
+      "key_questions": [
+        "string"
+      ],
+      "themes": [
+        "string"
+      ],
+      "do_not_do": [
+        "string"
+      ]
+    },
+    "score": {
+      "novelty": "0-5",
+      "feasibility": "0-5",
+      "leverage": "0-5",
+      "clarity": "0-5",
+      "evidence_alignment": "0-5",
+      "decision": [
+        "advance",
+        "hold",
+        "drop"
+      ]
+    },
+    "brief": {
+      "version": "string",
+      "summary": [
+        "string"
+      ],
+      "shortlist": [
+        {
+          "idea_id": "string",
+          "score": "score",
+          "notes": "string"
+        }
+      ],
+      "graph_delta": {
+        "nodes_added": [
+          "object"
+        ],
+        "edges_added": [
+          "object"
+        ],
+        "nodes_updated": [
+          "object"
+        ],
+        "edges_updated": [
+          "object"
+        ]
+      },
+      "open_questions": [
+        "string"
+      ],
+      "next_actions": [
+        "string"
+      ]
+    },
+    "final_pack": {
+      "primary_bets": [
+        "idea_card"
+      ],
+      "secondary_bets": [
+        "idea_card"
+      ],
+      "experiments": [
+        "object"
+      ],
+      "assumptions_to_validate": [
+        "string"
+      ],
+      "decision_log": [
+        "string"
+      ],
+      "kill_criteria": [
+        "string"
+      ],
+      "next_data": [
+        "string"
+      ]
+    }
+  },
+  "workflow": [
+    {
+      "phase": 0,
+      "name": "setup",
+      "artifacts_out": [
+        "context_snapshot_v0"
+      ]
+    },
+    {
+      "phase": 1,
+      "name": "independent_ideation",
+      "artifacts_out": [
+        "D_brief_v1",
+        "P_brief_v1"
+      ]
+    },
+    {
+      "phase": 2,
+      "name": "bridge",
+      "routes": [
+        "D.arbiter<->P.arbiter"
+      ],
+      "artifacts_out": [
+        "bridge_D_to_P",
+        "bridge_P_to_D"
+      ]
+    },
+    {
+      "phase": 3,
+      "name": "second_pass",
+      "artifacts_out": [
+        "D_brief_v2",
+        "P_brief_v2"
+      ]
+    },
+    {
+      "phase": 4,
+      "name": "meta_merge",
+      "artifacts_out": [
+        "final_idea_brief_pack"
+      ]
+    }
+  ],
+  "prompts": {
+    "D": {
+      "arbiter": "You are the Arbiter for the Deterministic Council. Manage phases, request Idea Cards, score with rubric, select and stop on convergence. Minimal speculation. Do not generate more than 2 original ideas yourself.",
+      "free_thinker": "Generate 10–15 feasible Idea Cards bounded by constraints. Include mechanisms. Keep each card short.",
+      "grounder": "Expand selected Idea Cards with Preconditions (X/Y/Z), dependencies, and MVEs with success metrics. Do not veto; propose cheap proxy tests if needed.",
+      "writer": "Maintain ideation graph and versioned briefs. No new ideas. Track deltas and tensions."
+    },
+    "P": {
+      "arbiter": "You are the Arbiter for the Probabilistic Council. Protect divergence early, enforce Idea Cards, require uncertainty labels, then request grounding and testability. Avoid premature convergence.",
+      "free_thinker": "Generate 15–25 novel Idea Cards with plausible mechanisms. Mark [SPECULATIVE] vs [GROUNDED]. Expand top 5.",
+      "grounder": "Preserve weirdness; extract kernel; define cheap MVEs with clear success metrics. Do not kill ideas.",
+      "writer": "Maintain ideation graph and versioned briefs. No new ideas. Track deltas and tensions."
+    },
+    "meta_arbiter": "Ingest D Brief v2 and P Brief v2. Apply rubric consistently. Select 1–3 primary bets and 2–4 secondary bets with MVEs and kill criteria. Produce rationale and decision log."
+  },
+  "graph_model": {
+    "node_types": [
+      "idea",
+      "assumption",
+      "constraint",
+      "experiment",
+      "evidence"
+    ],
+    "edge_types": [
+      "supports",
+      "depends_on",
+      "risks",
+      "tests",
+      "contradicts"
+    ]
+  },
+  "protocol": {
+    "allowed_routes": [
+      "D.arbiter<->D.free_thinker",
+      "D.arbiter<->D.grounder",
+      "D.arbiter<->D.writer",
+      "P.arbiter<->P.free_thinker",
+      "P.arbiter<->P.grounder",
+      "P.arbiter<->P.writer",
+      "D.arbiter<->P.arbiter (bridge only)",
+      "meta_arbiter reads D_brief_v2 and P_brief_v2"
+    ],
+    "hard_rules": [
+      "No cross-council messages except Bridge Packets.",
+      "Writers do not invent new ideas.",
+      "Grounders do not veto; they concretize.",
+      "Arbiters stop on convergence or diminishing novelty."
+    ]
+  }
+}

docs/councils/ai-council-production-scaffold.md

@@ -0,0 +1,296 @@
+# AI Council Scaffold (Production-Ready)
+Generated: 2026-02-21T18:05:48.200896
+
+This package defines:
+- Two councils (Deterministic, Probabilistic) with 4 roles each (Arbiter, FreeThinker, Grounder, Writer)
+- A controlled “Bridge Packet” exchange between Arbiters only
+- A Meta-Merge stage for final selection
+- Standard message schemas and state objects suitable for orchestration (LangGraph, Temporal, custom event loop, etc.)
+
+> Design principle: **protocol beats prompts**. Most failures are coordination failures, not “model intelligence” failures.
+
+---
+
+## 1) Entities
+
+### Councils
+- **Council D (Deterministic):** consistency, clarity, feasibility, low drift
+- **Council P (Probabilistic):** novelty, exploration, reframing, controlled speculation
+
+### Roles (per council)
+- **Arbiter (Lead):** requests work, scores, selects, stops
+- **Free Thinker:** divergent generator
+- **Grounder:** convergent concretizer (not a skeptic)
+- **Writer:** ideation graph + versioned briefs
+
+### Meta Roles
+- **Meta-Arbiter:** merges v2 briefs and selects winners
+- **Meta-Writer (optional):** final pack, decision log, next experiments
+
+---
+
+## 2) Core Artifacts
+
+### 2.1 Input Package (shared)
+A single bundle sent to both councils at the start.
+
+**Required fields**
+- Problem statement (1–3 paragraphs)
+- Success definition (what “good” looks like)
+- Constraints (budget/time/tech/legal/ethics)
+- Context (what’s been tried, known facts, prior art)
+- Timebox (round count and max output size)
+
+**Optional fields**
+- Stakeholders & priorities
+- Non-goals
+- Risk tolerance (low/med/high)
+- Domain glossary
+
+### 2.2 Idea Card (atomic unit)
+All agents output in this format to keep merging clean.
+
+**Idea Card**
+- **id:** `D-03` or `P-11`
+- **title:** 3–8 words
+- **one_liner:** 1 sentence
+- **mechanism:** 2–6 bullets (why it might work)
+- **assumptions:** 2–6 bullets
+- **constraints:** 2–6 bullets (dependencies, limits)
+- **mve:** minimal viable experiment
+  - **steps:** 3–7 bullets
+  - **success_metric:** 1–2 sentences
+  - **time_cost:** (e.g., “2 hours”, “1 week”)
+  - **resources:** bullets
+- **risks:** 2–6 bullets
+- **upside:** what “winning” looks like
+- **tags:** list of strings (domain, novelty, cost, etc.)
+
+### 2.3 Bridge Packet (Arbiter ↔ Arbiter only)
+Single exchange. No debate. Bandwidth-capped.
+
+- **top_ideas:** 3 Idea Cards (full)
+- **fragile_assumptions:** 3 bullets
+- **key_questions:** 3 bullets
+- **themes:** 1–2 bullets
+- **do_not_do:** 1–2 bullets (things that hurt independence, premature convergence)
+
+### 2.4 Brief (versioned)
+Produced by each council Writer after each phase.
+
+**Brief vX**
+- Summary (5–10 bullets)
+- Shortlist (3–5 ideas) with scores
+- Ideation graph delta (what changed since last version)
+- Open questions (prioritized)
+- Recommended next actions (MVEs)
+
+### 2.5 Final Idea Brief Pack
+Produced after Meta-Merge.
+
+- Primary bets (1–3)
+- Secondary bets (2–4)
+- MVEs for each
+- Assumptions to validate (ranked)
+- Decision log (why these won)
+- “Kill criteria” (what would make us drop an idea)
+- Next data to collect
+
+---
+
+## 3) Scoring Rubric (Arbiters & Meta-Arbiter)
+
+Score 0–5 on:
+- **novelty**
+- **feasibility**
+- **leverage** (impact if true)
+- **clarity** (can we explain/execute it)
+- **evidence_alignment** (fits known facts)
+
+Decision label:
+- **advance**
+- **hold**
+- **drop**
+
+Convergence guidance:
+- If **top-2 ideas** are stable across rounds and marginal novelty decreases, signal convergence.
+- If D and P independently converge on similar cores, treat that as strong signal.
+
+---
+
+## 4) Workflow (Islands + Thin Bridge)
+
+### Phase 0 — Setup
+1. User provides Input Package.
+2. Writers create **Context Snapshot v0** and empty ideation graph.
+
+### Phase 1 — Independent Ideation (no cross-council contact)
+Per council:
+1. Arbiter requests Idea Cards from Free Thinker (target 10–20).
+2. Arbiter selects top 5 and requests grounding.
+3. Grounder expands top 5 with X/Y/Z + MVEs.
+4. Writer compiles **Brief v1** + graph.
+
+### Phase 2 — Bridge (Arbiter ↔ Arbiter only)
+1. D-Arbiter sends Bridge Packet to P-Arbiter.
+2. P-Arbiter sends Bridge Packet to D-Arbiter.
+(No further discussion.)
+
+### Phase 3 — Second Pass (still mostly independent)
+Per council:
+1. Arbiter asks Free Thinker for **3 variants** inspired by received Bridge Packet.
+2. Arbiter asks Grounder to instrument the best 3 into clearer MVEs.
+3. Writer compiles **Brief v2**.
+
+### Phase 4 — Meta Merge
+1. Meta-Arbiter ingests both briefs.
+2. Selects final bets + experiments.
+3. Meta-Writer outputs Final Idea Brief Pack.
+
+---
+
+## 5) Interaction Protocol (message types)
+
+### Allowed message routes
+- Within a council: Arbiter ↔ (FreeThinker, Grounder, Writer)
+- Bridge: D-Arbiter ↔ P-Arbiter **only**
+- Meta: Meta-Arbiter reads both briefs; optional Meta-Writer
+
+### Hard rules
+- No cross-council messages except Bridge Packets.
+- No role switching inside an agent.
+- Writers do not “invent” new ideas; they only capture, structure, and diff.
+- Grounders do not veto; they convert into testable plans and surface constraints.
+
+---
+
+## 6) Prompts (Production Templates)
+
+These are role instructions you pin as **system prompts** or **role cards**.
+Fill `{INPUT_PACKAGE}` at runtime.
+
+### 6.1 D-Arbiter (Deterministic)
+**Role:** You are the Arbiter for the Deterministic Council.
+**Style:** crisp, bounded, structured. Minimal speculation.
+**Primary duties:**
+- Request outputs in **Idea Card** format.
+- Score ideas with the rubric.
+- Select top candidates and request grounding.
+- Stop when diminishing returns or convergence.
+
+**Operating rules:**
+- Do not generate more than 2 original ideas yourself.
+- Push for executable MVEs.
+- Enforce constraints strictly.
+
+**Start instructions:**
+1) Ask D-FreeThinker for 10–15 Idea Cards based on {INPUT_PACKAGE}.
+2) After receiving, shortlist 5 and ask D-Grounder to expand them with X/Y/Z + MVE.
+3) Ask D-Writer to produce Brief v1.
+
+### 6.2 P-Arbiter (Probabilistic)
+**Role:** You are the Arbiter for the Probabilistic Council.
+**Style:** theme-hunting, novelty-protecting, but still structured.
+**Primary duties:**
+- Maximize novelty density early.
+- Require speculative claims to be labeled.
+- Prevent premature convergence in Phase 1.
+
+**Operating rules:**
+- Do not terminate Phase 1 early unless redundancy is severe.
+- Enforce Idea Card format.
+- After Phase 1, move to grounding and testability.
+
+### 6.3 D-Free Thinker
+**Role:** Divergent generator under constraints.
+**Output:** 10–15 Idea Cards.
+**Rules:**
+- Keep ideas feasible within constraints.
+- Avoid vague visions; include mechanisms.
+- Keep each card short.
+
+### 6.4 P-Free Thinker
+**Role:** Divergent generator optimized for novelty.
+**Output:** 15–25 Idea Cards, then expand top 5 with richer mechanism.
+**Rules:**
+- Cross-domain analogies encouraged.
+- Mark uncertainty:
+  - `[SPECULATIVE]` for leaps
+  - `[GROUNDED]` for common/known patterns
+- Avoid pure fantasy: each idea must include a plausible mechanism.
+
+### 6.5 D-Grounder
+**Role:** Convergent concretizer.
+**Input:** 3–5 selected Idea Cards.
+**Output:** Updated cards with:
+- Preconditions (X/Y/Z)
+- Dependencies
+- MVE with success metric
+**Rules:**
+- Do not kill ideas.
+- If infeasible, propose a “cheap proxy” test.
+
+### 6.6 P-Grounder
+**Role:** Preserve weirdness; make it testable.
+**Input:** 3–5 selected Idea Cards.
+**Output:** MVEs that validate the “kernel” cheaply.
+**Rules:**
+- Keep the original spirit.
+- Convert big leaps into small experiments.
+
+### 6.7 Writer (both councils)
+**Role:** Scribe + graph curator.
+**Live duties:**
+- Track Idea Cards, scores, shortlist, assumptions, constraints.
+- Maintain ideation graph (nodes and edges).
+- Produce versioned briefs with deltas.
+
+**Rules:**
+- No new ideas.
+- If conflicting facts appear, record them as “tension edges”.
+- Keep briefs short and merge-friendly.
+
+### 6.8 Meta-Arbiter
+**Role:** Final selector.
+**Input:** D Brief v2 + P Brief v2.
+**Output:** Final selection + rationale.
+**Rules:**
+- Apply rubric consistently.
+- Prefer portfolios: 1–3 primary bets + 2–4 secondary.
+- Specify MVEs and kill criteria.
+
+---
+
+## 7) Ideation Graph Model (minimal)
+Nodes:
+- `idea`
+- `assumption`
+- `constraint`
+- `experiment`
+- `evidence`
+
+Edges:
+- `supports`
+- `depends_on`
+- `risks`
+- `tests`
+- `contradicts`
+
+Graph delta per version:
+- nodes_added, edges_added, nodes_updated, edges_updated
+
+---
+
+## 8) Quickstart Checklist
+1. Write Input Package.
+2. Run Phase 1 for Council D and Council P independently.
+3. Exchange Bridge Packets (Arbiter↔Arbiter only).
+4. Run Phase 3 short second pass.
+5. Meta-merge into Final Idea Brief Pack.
+
+---
+
+## 9) Optional Extensions (keep it lean)
+- Add “Historian” (cross-session memory) only if you run repeated projects.
+- Add “User Proxy” only if stakeholder conflicts are chronic.
+- Add “Devil’s Engineer” only **after** primary bets are chosen.

src/config/schema.ts

@@ -863,6 +863,8 @@ const agentConfigsSchema = z.record(z.string(), agentConfigEntrySchema).default(
 const councilsGroupConfigSchema = z.object({
   arbiter_agent: z.string().min(1),
   freethinker_agent: z.string().min(1),
+  grounder_agent: z.string().min(1).optional(),
+  writer_agent: z.string().min(1).optional(),
   group_prompt_prefix: z.string().min(1),
   novelty_bias: z.enum(['low', 'medium', 'high']).default('medium'),
   risk_tolerance: z.enum(['low', 'medium', 'high']).default('medium'),
@@ -883,6 +885,7 @@ const councilsSchema = z.object({
   }).default({}),
   strict_grounding: z.boolean().default(false),
   strict_meta_validation: z.boolean().default(true),
+  scaffold_path: z.string().optional(),
   groups: z.object({
     D: councilsGroupConfigSchema.default({
       arbiter_agent: 'council_d_arbiter',
@@ -910,6 +913,7 @@ const councilsSchema = z.object({
     }),
   }).default({}),
   meta_arbiter_agent: z.string().min(1).default('council_meta_arbiter'),
+  meta_writer_agent: z.string().min(1).optional(),
 }).default({});
 
 const routingSchema = z.object({

src/councils/orchestrator.ts

@@ -2,6 +2,7 @@ import type { AgentConfigRegistry } from '../agents/registry.js';
 import type { AgentOrchestrator } from '../backends/native/orchestrator.js';
 import type { ModelTier } from '../models/router.js';
 import type { TokenUsage } from '../models/types.js';
+import type { CouncilScaffold } from './scaffold.js';
 import {
   COUNCIL_PIPELINE_VERSION,
   COUNCIL_SCHEMA_VERSION,
@@ -60,11 +61,15 @@ export interface CouncilsConfig {
     P: CouncilGroupConfig;
   };
   meta_arbiter_agent: string;
+  meta_writer_agent?: string;
+  scaffold_path?: string;
 }
 
 interface CouncilGroupConfig {
   arbiter_agent: string;
   freethinker_agent: string;
+  grounder_agent?: string;
+  writer_agent?: string;
   group_prompt_prefix: string;
   novelty_bias: 'low' | 'medium' | 'high';
   risk_tolerance: 'low' | 'medium' | 'high';
@@ -223,16 +228,19 @@ export class CouncilsOrchestrator {
   private readonly _registry: AgentConfigRegistry;
   private readonly _delegateRunner: DelegateRunner;
   private readonly _config: CouncilsConfig;
+  private readonly _scaffold?: CouncilScaffold;
   private readonly _trace: CouncilTraceEvent[] = [];
 
   constructor(deps: {
     registry: AgentConfigRegistry;
     orchestrator: DelegateRunner;
     config: CouncilsConfig;
+    scaffold?: CouncilScaffold;
   }) {
     this._registry = deps.registry;
     this._delegateRunner = deps.orchestrator;
     this._config = deps.config;
+    this._scaffold = deps.scaffold;
   }
 
   async run(rawInput: unknown): Promise<CouncilRunResult> {
@@ -386,12 +394,14 @@ export class CouncilsOrchestrator {
     round?: number;
     promptPayload: unknown;
     modeDirective: string;
+    scaffoldPrompt?: string;
     maxTokens?: number;
   }): Promise<AgentCallResult> {
     const agent = this.getAgent(opts.agentName);
     const message = canonicalStringify(opts.promptPayload);
     const promptHash = hashCanonical(opts.promptPayload);
-    const systemPrompt = `${agent.systemPrompt}\n\n${opts.modeDirective}`;
+    const scaffoldPrompt = opts.scaffoldPrompt ? `${opts.scaffoldPrompt}\n\n` : '';
+    const systemPrompt = `${scaffoldPrompt}${agent.systemPrompt}\n\n${opts.modeDirective}`;
 
     const result = await this._delegateRunner.delegate({
       tier: agent.tier,
@@ -451,6 +461,7 @@ export class CouncilsOrchestrator {
       round,
       promptPayload: ideationPayload,
       modeDirective: 'Return JSON only: {"ideas":[IdeaContent,...]}. Do not include IDs. No prose.',
+      scaffoldPrompt: this._scaffold?.prompts[group].free_thinker,
     });
 
     const ideaOutput = parseJsonWithRepair(ideation.content, (value) => ideationOutputSchema.parse(value));
@@ -482,6 +493,7 @@ export class CouncilsOrchestrator {
       promptPayload: assessmentPayload,
       modeDirective:
         'Return JSON only. Assess provided idea IDs only. No new IDs. Include convergence_signal/novelty_score/repetition_rate.',
+      scaffoldPrompt: this._scaffold?.prompts[group].arbiter,
     });
 
     const assessmentOutput = parseJsonWithRepair(assessmentRaw.content, (value) => assessmentOutputSchema.parse(value));
@@ -524,11 +536,12 @@ export class CouncilsOrchestrator {
       constraints: input.constraints,
     };
 
+    const groundingAgent = groupConfig.grounder_agent ?? groupConfig.freethinker_agent;
     let groundingFailures = 0;
     let grounding = { grounded: [] as Array<{ idea_id: string; mve: string; constraints: string[]; falsifiability_checks: string[] }> };
     try {
       const groundingRaw = await this.callAgent({
-        agentName: groupConfig.freethinker_agent,
+        agentName: groundingAgent,
         callId: `${group}.r${round}.ft.ground`,
         phaseIndex: phaseBase + 2,
         group,
@@ -536,6 +549,7 @@ export class CouncilsOrchestrator {
         promptPayload: groundingPayload,
         modeDirective:
           'Grounder mode. Return JSON only: {"grounded":[{"idea_id", "mve", "constraints", "falsifiability_checks"}]}. No prose.',
+        scaffoldPrompt: this._scaffold?.prompts[group].grounder ?? this._scaffold?.prompts[group].free_thinker,
       });
       grounding = parseJsonWithRepair(groundingRaw.content, (value) => groundingOutputSchema.parse(value));
     } catch {
@@ -686,6 +700,7 @@ export class CouncilsOrchestrator {
       promptPayload: payload,
       modeDirective:
         'Return JSON only following schema with selected_primary/selected_secondary/merges/rejections/open_questions/next_experiments. Use only known idea IDs.',
+      scaffoldPrompt: this._scaffold?.prompts.meta_arbiter,
     });
 
     return parseJsonWithRepair(metaRaw.content, (value) => metaSelectionSchema.parse(value));

src/councils/scaffold.ts

@@ -0,0 +1,37 @@
+import { readFileSync } from 'fs';
+import { z } from 'zod';
+
+const councilPromptSetSchema = z.object({
+  arbiter: z.string().min(1),
+  free_thinker: z.string().min(1),
+  grounder: z.string().min(1).optional(),
+  writer: z.string().min(1).optional(),
+}).strict();
+
+export const councilScaffoldSchema = z.object({
+  generated_at: z.string().optional(),
+  version: z.string().optional(),
+  prompts: z.object({
+    D: councilPromptSetSchema,
+    P: councilPromptSetSchema,
+    meta_arbiter: z.string().min(1),
+  }).strict(),
+}).strict();
+
+export type CouncilScaffold = z.infer<typeof councilScaffoldSchema>;
+
+export function loadCouncilScaffold(path: string): CouncilScaffold {
+  const raw = readFileSync(path, 'utf8');
+  return councilScaffoldSchema.parse(JSON.parse(raw));
+}
+
+export function loadCouncilScaffoldSafe(path?: string): CouncilScaffold | undefined {
+  if (!path) {
+    return undefined;
+  }
+  try {
+    return loadCouncilScaffold(path);
+  } catch {
+    return undefined;
+  }
+}