Jev classifier in tandem with the LLM Sentry

- Jev (TypeSafe, direct) triages mode and task type with confidence
- Its call stands at >= 70% on both and no split; otherwise the LLM
  Sentry decides (unsure, split needs subtasks, or Jev down)
- Jev flights obey the same budget, flock cap, timeout and log
- Tests for every path; flight log is nil-safe
- start.command reads TYPESAFE_API_KEY and threshold from .env

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UQnSw4RRWp8uqpj3nsnnXF

README.md

@@ -53,6 +53,10 @@ curl -XPOST localhost:8080/api/lists -d '{"text":"- [ ] Should we launch on a Tu
 - `crow.flying` carries `borrowed: true` for a crow from outside the nest's core.
 - Names are 1–40 characters; `nest_id` is the name as a slug and must be unique (409 on a clash). 1–8 roster crows, no repeats, at most 20 nests. Nests are saved in `.crows/nests.json`.
 
+## Jev triage (optional)
+
+With `TYPESAFE_API_KEY` set, the Sentry asks Jev (TypeSafe's classifier) for mode and task type first. Jev's call stands when both answers reach `CROWS_JEV_THRESHOLD` (default 0.7) and no subtasks are needed. Otherwise the LLM Sentry decides: when Jev is unsure, when it calls a split (the LLM writes subtasks), or when Jev fails. The plan's reason shows Jev's confidence, and the flight log records each Jev call.
+
 ## How an item flies
 
 1. **Sentry** classifies it: `solo` (one crow), `split` (2–4 subtasks in parallel), or `murder` (3 crows answer independently).

flightlog.go

@@ -43,6 +43,9 @@ func OpenFlightLog(path string) (*FlightLog, error) {
 }
 
 func (l *FlightLog) Write(e Event) {
+	if l == nil {
+		return // no log configured (tests, embedded use)
+	}
 	l.mu.Lock()
 	defer l.mu.Unlock()
 	e.Time = time.Now()

flock.go

@@ -79,6 +79,7 @@ type Watcher interface {
 
 type Flock struct {
 	brain Brain
+	jev   *JevClient // optional classifier for triage; nil means the LLM Sentry triages alone
 	g     Guardrails
 	log   *FlightLog
 	sem   chan struct{}
@@ -163,7 +164,38 @@ func (f *Flock) ask(ctx context.Context, w Watcher, crow, system, prompt string,
 	return text, err
 }
 
+// SetClassifier pairs the Sentry with Jev. Pass nil to triage with the LLM alone.
+func (f *Flock) SetClassifier(j *JevClient) { f.jev = j }
+
+// Triage: Jev classifies first when it's configured. Its call stands when it is
+// confident and no subtasks are needed; otherwise the LLM Sentry decides, and
+// the reason records what Jev thought. Jev failing never blocks a list.
 func (f *Flock) Triage(ctx context.Context, w Watcher, item string, budgets ...*Budget) (Triage, error) {
+	if f.jev == nil {
+		return f.llmTriage(ctx, w, item, budgets...)
+	}
+	call, err := f.classify(ctx, w, item, budgets...)
+	if err != nil {
+		t, lerr := f.llmTriage(ctx, w, item, budgets...)
+		t.Reason = "Jev was unavailable; the LLM Sentry decided. " + t.Reason
+		return t, lerr
+	}
+	if call.confident(f.jev.threshold) && call.Mode != "split" {
+		return f.normalize(Triage{Mode: call.Mode, TaskType: call.TaskType, Reason: call.summary()}), nil
+	}
+	t, lerr := f.llmTriage(ctx, w, item, budgets...)
+	switch {
+	case !call.confident(f.jev.threshold):
+		t.Reason = fmt.Sprintf("%s; unsure, so the LLM Sentry decided. %s", call.summary(), t.Reason)
+	case t.Mode == "split":
+		t.Reason = fmt.Sprintf("%s; the LLM Sentry wrote the subtasks. %s", call.summary(), t.Reason)
+	default:
+		t.Reason = fmt.Sprintf("%s; the LLM Sentry disagreed and chose %s. %s", call.summary(), t.Mode, t.Reason)
+	}
+	return t, lerr
+}
+
+func (f *Flock) llmTriage(ctx context.Context, w Watcher, item string, budgets ...*Budget) (Triage, error) {
 	text, err := f.ask(ctx, w, "Sentry", sentrySystem, "ITEM: "+item, budgets...)
 	if err != nil {
 		return Triage{}, err
@@ -172,6 +204,11 @@ func (f *Flock) Triage(ctx context.Context, w Watcher, item string, budgets ...*
 	if i, j := strings.Index(text, "{"), strings.LastIndex(text, "}"); i < 0 || j < i || json.Unmarshal([]byte(text[i:j+1]), &t) != nil {
 		t = Triage{Mode: "solo", TaskType: "other", Reason: "Sentry's call was unreadable; defaulting to solo"}
 	}
+	return f.normalize(t), nil
+}
+
+// normalize keeps any triage, from Jev or the LLM, inside the guardrails.
+func (f *Flock) normalize(t Triage) Triage {
 	t.Mode = strings.ToLower(strings.TrimSpace(t.Mode))
 	t.TaskType = strings.ToLower(strings.TrimSpace(t.TaskType))
 	switch t.TaskType {
@@ -192,7 +229,7 @@ func (f *Flock) Triage(ctx context.Context, w Watcher, item string, budgets ...*
 	default:
 		t.Mode, t.Subtasks = "solo", nil
 	}
-	return t, nil
+	return t
 }
 
 func (f *Flock) Execute(ctx context.Context, w Watcher, item string, t Triage, nest Nest, budgets ...*Budget) (Outcome, error) {

jev.go

@@ -0,0 +1,169 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+)
+
+// Jev (TypeSafe) is a classifier, not a text model: it returns typed choices
+// with calibrated confidence. The Sentry uses it for triage, and escalates to
+// the LLM when Jev is unsure, when an item needs subtasks written, or when Jev
+// fails. Docs: https://docs.typesafe.ai/introduction/quickstart
+
+type JevClient struct {
+	url, key, model string
+	threshold       float64 // both answers must reach this confidence to stand alone
+	client          *http.Client
+}
+
+// JevCall is one classification: a label and a confidence per question.
+type JevCall struct {
+	Mode, TaskType         string
+	ModeConf, TaskTypeConf float64
+	Model                  string
+	Tokens                 int
+}
+
+// NewJev returns nil (no classifier) unless TYPESAFE_API_KEY is set.
+//
+//	TYPESAFE_API_KEY         the key from the TypeSafe console
+//	CROWS_JEV_MODEL          default jev-latest
+//	CROWS_JEV_THRESHOLD      default 0.7
+func NewJev() *JevClient {
+	key := os.Getenv("TYPESAFE_API_KEY")
+	if key == "" {
+		return nil
+	}
+	model := os.Getenv("CROWS_JEV_MODEL")
+	if model == "" {
+		model = "jev-latest"
+	}
+	threshold := 0.7
+	if v, err := strconv.ParseFloat(os.Getenv("CROWS_JEV_THRESHOLD"), 64); err == nil && v > 0 && v <= 1 {
+		threshold = v
+	}
+	return &JevClient{url: "https://api.typesafe.ai/v1/systemone", key: key, model: model, threshold: threshold, client: &http.Client{}}
+}
+
+func (j *JevClient) Name() string {
+	return fmt.Sprintf("Jev (%s, escalates below %.0f%%)", j.model, j.threshold*100)
+}
+
+var jevQuestions = map[string]any{
+	"mode": map[string]any{
+		"type":         "choice",
+		"instructions": "How should a team of AI agents handle this todo item?",
+		"criteria": map[string]string{
+			"solo":   "Simple or single-step: one agent can answer it directly.",
+			"split":  "Large, with 2 to 4 distinct parts that can be done independently and in parallel.",
+			"murder": "Ambiguous, high-stakes or a judgment call: several agents should answer independently, then reconcile.",
+		},
+	},
+	"task_type": map[string]any{
+		"type":         "choice",
+		"instructions": "What kind of task is this todo item?",
+		"criteria": map[string]string{
+			"retrieval": "Looking up or recalling facts.",
+			"writing":   "Drafting or editing text.",
+			"analysis":  "Comparing, evaluating, deciding or reasoning about options.",
+			"other":     "Anything else.",
+		},
+	},
+}
+
+func (j *JevClient) Classify(ctx context.Context, item string) (JevCall, error) {
+	body, _ := json.Marshal(map[string]any{"model": j.model, "state": item, "questions": jevQuestions})
+	req, err := http.NewRequestWithContext(ctx, "POST", j.url, bytes.NewReader(body))
+	if err != nil {
+		return JevCall{}, err
+	}
+	req.Header.Set("Authorization", "Bearer "+j.key)
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := j.client.Do(req)
+	if err != nil {
+		return JevCall{}, err
+	}
+	defer resp.Body.Close()
+	raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if resp.StatusCode != http.StatusOK {
+		return JevCall{}, fmt.Errorf("Jev returned %d: %s", resp.StatusCode, clip(raw, 300))
+	}
+	type answer struct {
+		Choice     string  `json:"choice"`
+		Confidence float64 `json:"confidence"`
+	}
+	var out struct {
+		Model   string            `json:"model"`
+		Answers map[string]answer `json:"answers"`
+		Usage   struct {
+			InputTokens  int `json:"input_tokens"`
+			OutputTokens int `json:"output_tokens"`
+		} `json:"usage"`
+	}
+	if err := json.Unmarshal(raw, &out); err != nil {
+		return JevCall{}, fmt.Errorf("Jev sent an unreadable reply: %w", err)
+	}
+	mode, task := out.Answers["mode"], out.Answers["task_type"]
+	if mode.Choice == "" || task.Choice == "" {
+		return JevCall{}, errors.New("Jev's reply was missing an answer")
+	}
+	model := out.Model
+	if model == "" {
+		model = j.model
+	}
+	return JevCall{
+		Mode: mode.Choice, TaskType: task.Choice, ModeConf: mode.Confidence, TaskTypeConf: task.Confidence,
+		Model: model, Tokens: out.Usage.InputTokens + out.Usage.OutputTokens,
+	}, nil
+}
+
+func (c JevCall) confident(threshold float64) bool {
+	return c.ModeConf >= threshold && c.TaskTypeConf >= threshold
+}
+
+func (c JevCall) summary() string {
+	return fmt.Sprintf("Jev: %s %.0f%%, %s %.0f%%", c.Mode, c.ModeConf*100, c.TaskType, c.TaskTypeConf*100)
+}
+
+// classify runs Jev under the same guardrails as any crow: budget, flock cap,
+// timeout, flight log, and flying/landed events for the Sentry.
+func (f *Flock) classify(ctx context.Context, w Watcher, item string, budgets ...*Budget) (JevCall, error) {
+	for _, b := range budgets {
+		if err := b.Check(); err != nil {
+			return JevCall{}, err
+		}
+	}
+	f.sem <- struct{}{}
+	defer func() { <-f.sem }()
+	if w != nil {
+		w.Flying("Sentry")
+	}
+	cctx, cancel := context.WithTimeout(ctx, f.g.CrowTimeout)
+	defer cancel()
+	start := time.Now()
+	call, err := f.jev.Classify(cctx, item)
+	for _, b := range budgets {
+		b.Add(call.Tokens)
+	}
+	e := Event{Kind: "flight", Crow: "Sentry", Model: call.Model, Tokens: call.Tokens, Ms: time.Since(start).Milliseconds()}
+	if err != nil {
+		e.Error = err.Error()
+	} else {
+		e.Note = call.summary()
+	}
+	if f.log != nil {
+		f.log.Write(e)
+	}
+	if w != nil {
+		w.Landed("Sentry", call.Tokens, err)
+	}
+	return call, err
+}

jev_test.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+)
+
+// countingBrain is the LLM Sentry: it counts calls and always answers split with subtasks.
+type countingBrain struct{ calls atomic.Int32 }
+
+func (b *countingBrain) Name() string { return "counting" }
+func (b *countingBrain) Ask(ctx context.Context, system, prompt string) (string, int, error) {
+	b.calls.Add(1)
+	return `{"mode":"split","task_type":"writing","subtasks":["a","b"],"reason":"two parts"}`, 10, nil
+}
+
+func fakeJev(t *testing.T, status int, mode string, modeConf float64, task string, taskConf float64) *JevClient {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "Bearer tk" {
+			t.Errorf("auth header = %q", r.Header.Get("Authorization"))
+		}
+		var body map[string]any
+		json.NewDecoder(r.Body).Decode(&body)
+		if body["state"] == nil || body["questions"] == nil {
+			t.Errorf("request missing state or questions: %v", body)
+		}
+		w.WriteHeader(status)
+		json.NewEncoder(w).Encode(map[string]any{
+			"model": "jev-1.13.0",
+			"answers": map[string]any{
+				"mode":      map[string]any{"type": "choice", "choice": mode, "confidence": modeConf},
+				"task_type": map[string]any{"type": "choice", "choice": task, "confidence": taskConf},
+			},
+			"usage": map[string]int{"input_tokens": 40, "output_tokens": 0},
+		})
+	}))
+	t.Cleanup(srv.Close)
+	return &JevClient{url: srv.URL, key: "tk", model: "jev-latest", threshold: 0.7, client: srv.Client()}
+}
+
+func triageWith(t *testing.T, j *JevClient) (Triage, int32) {
+	t.Helper()
+	brain := &countingBrain{}
+	f := NewFlock(brain, DefaultGuardrails(), nil)
+	f.SetClassifier(j)
+	tr, err := f.Triage(context.Background(), nil, "some item")
+	if err != nil {
+		t.Fatal(err)
+	}
+	return tr, brain.calls.Load()
+}
+
+func TestJevConfidentStandsAlone(t *testing.T) {
+	tr, llm := triageWith(t, fakeJev(t, 200, "murder", 0.91, "analysis", 0.88))
+	if llm != 0 || tr.Mode != "murder" || tr.TaskType != "analysis" || !strings.Contains(tr.Reason, "Jev: murder 91%") {
+		t.Fatalf("got %+v with %d LLM calls; want Jev's call alone", tr, llm)
+	}
+}
+
+func TestJevUnsureEscalates(t *testing.T) {
+	tr, llm := triageWith(t, fakeJev(t, 200, "solo", 0.55, "retrieval", 0.9))
+	if llm != 1 || !strings.Contains(tr.Reason, "unsure") {
+		t.Fatalf("got %+v with %d LLM calls; want escalation", tr, llm)
+	}
+}
+
+func TestJevSplitGetsSubtasksFromLLM(t *testing.T) {
+	tr, llm := triageWith(t, fakeJev(t, 200, "split", 0.95, "writing", 0.95))
+	if llm != 1 || tr.Mode != "split" || len(tr.Subtasks) != 2 || !strings.Contains(tr.Reason, "wrote the subtasks") {
+		t.Fatalf("got %+v with %d LLM calls; want LLM-written subtasks", tr, llm)
+	}
+}
+
+func TestJevDownFallsBackToLLM(t *testing.T) {
+	tr, llm := triageWith(t, fakeJev(t, 503, "solo", 0.9, "other", 0.9))
+	if llm != 1 || !strings.Contains(tr.Reason, "Jev was unavailable") {
+		t.Fatalf("got %+v with %d LLM calls; want fallback", tr, llm)
+	}
+}
+
+func TestNoJevKeyMeansNoClassifier(t *testing.T) {
+	t.Setenv("TYPESAFE_API_KEY", "")
+	if NewJev() != nil {
+		t.Fatal("expected no classifier without a key")
+	}
+}

main.go

@@ -90,6 +90,7 @@ func cmdRun(args []string) {
 
 	brain := NewBrain()
 	p.Flock = NewFlock(brain, g, flog)
+	p.Flock.SetClassifier(NewJev())
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer stop()
 

server.go

@@ -78,6 +78,11 @@ func cmdServe(args []string) {
 	brain := NewBrain()
 	approver := NewWebApprover(*timeout)
 	p := &Pipeline{Flock: NewFlock(brain, g, flog), Ledger: ledger, Log: flog, G: g, Approver: approver}
+	jev := NewJev()
+	p.Flock.SetClassifier(jev)
+	if jev != nil {
+		fmt.Printf("Sentry triage: %s, LLM fallback\n", jev.Name())
+	}
 	s := NewServer(ctx, p, approver, nests)
 
 	srv := &http.Server{Addr: *addr, Handler: s.Handler(), ReadHeaderTimeout: 10 * time.Second}

start.command

@@ -7,6 +7,8 @@
 #   ANTHROPIC_API_KEY=...       Claude direct from Anthropic (default: claude-sonnet-5-5)
 #   CROWS_PROVIDER=openrouter   optional: pick one when both keys are present
 #   CROWS_MODEL=...             optional: a specific model
+#   TYPESAFE_API_KEY=...        Jev classifier for the Sentry's triage (LLM is the fallback)
+#   CROWS_JEV_THRESHOLD=0.7     optional: below this confidence, triage escalates to the LLM
 # `./start.command --mock` forces offline mock crows.
 #
 # Listens on this Mac only (127.0.0.1:8080). State lives in .crows/ (delete it to start fresh).
@@ -17,9 +19,10 @@ chmod +x ./crows
 
 # Read only these lines from .env; the file is never executed.
 envval() { sed -n "s/^$1=//p" .env 2>/dev/null | head -n 1 | tr -d '[:space:]"'; }
-unset ANTHROPIC_API_KEY OPENROUTER_API_KEY CROWS_PROVIDER CROWS_MODEL
+VARS="ANTHROPIC_API_KEY OPENROUTER_API_KEY CROWS_PROVIDER CROWS_MODEL TYPESAFE_API_KEY CROWS_JEV_MODEL CROWS_JEV_THRESHOLD"
+unset $VARS
 if [ "$1" != "--mock" ]; then
-  for v in ANTHROPIC_API_KEY OPENROUTER_API_KEY CROWS_PROVIDER CROWS_MODEL; do
+  for v in $VARS; do
     val="$(envval "$v")"
     [ -n "$val" ] && export "$v=$val"
   done