From 449929123120894d8ddc06e184c75c4e5f285226 Mon Sep 17 00:00:00 2001 From: Smokey <150986568+jacisjake@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:13:48 -0600 Subject: [PATCH] Jev classifier in tandem with the LLM Sentry - Jev (TypeSafe, direct) triages mode and task type with confidence - Its call stands at >= 70% on both and no split; otherwise the LLM Sentry decides (unsure, split needs subtasks, or Jev down) - Jev flights obey the same budget, flock cap, timeout and log - Tests for every path; flight log is nil-safe - start.command reads TYPESAFE_API_KEY and threshold from .env Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UQnSw4RRWp8uqpj3nsnnXF --- README.md | 4 ++ flightlog.go | 3 + flock.go | 39 +++++++++++- jev.go | 169 ++++++++++++++++++++++++++++++++++++++++++++++++++ jev_test.go | 92 +++++++++++++++++++++++++++ main.go | 1 + server.go | 5 ++ start.command | 7 ++- 8 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 jev.go create mode 100644 jev_test.go diff --git a/README.md b/README.md index 4c1f83a..8b6f588 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,10 @@ curl -XPOST localhost:8080/api/lists -d '{"text":"- [ ] Should we launch on a Tu - `crow.flying` carries `borrowed: true` for a crow from outside the nest's core. - Names are 1–40 characters; `nest_id` is the name as a slug and must be unique (409 on a clash). 1–8 roster crows, no repeats, at most 20 nests. Nests are saved in `.crows/nests.json`. +## Jev triage (optional) + +With `TYPESAFE_API_KEY` set, the Sentry asks Jev (TypeSafe's classifier) for mode and task type first. Jev's call stands when both answers reach `CROWS_JEV_THRESHOLD` (default 0.7) and no subtasks are needed. Otherwise the LLM Sentry decides: when Jev is unsure, when it calls a split (the LLM writes subtasks), or when Jev fails. The plan's reason shows Jev's confidence, and the flight log records each Jev call. + ## How an item flies 1. **Sentry** classifies it: `solo` (one crow), `split` (2–4 subtasks in parallel), or `murder` (3 crows answer independently). diff --git a/flightlog.go b/flightlog.go index 0ea6452..fb71a0c 100644 --- a/flightlog.go +++ b/flightlog.go @@ -43,6 +43,9 @@ func OpenFlightLog(path string) (*FlightLog, error) { } func (l *FlightLog) Write(e Event) { + if l == nil { + return // no log configured (tests, embedded use) + } l.mu.Lock() defer l.mu.Unlock() e.Time = time.Now() diff --git a/flock.go b/flock.go index b352193..e8755a9 100644 --- a/flock.go +++ b/flock.go @@ -79,6 +79,7 @@ type Watcher interface { type Flock struct { brain Brain + jev *JevClient // optional classifier for triage; nil means the LLM Sentry triages alone g Guardrails log *FlightLog sem chan struct{} @@ -163,7 +164,38 @@ func (f *Flock) ask(ctx context.Context, w Watcher, crow, system, prompt string, return text, err } +// SetClassifier pairs the Sentry with Jev. Pass nil to triage with the LLM alone. +func (f *Flock) SetClassifier(j *JevClient) { f.jev = j } + +// Triage: Jev classifies first when it's configured. Its call stands when it is +// confident and no subtasks are needed; otherwise the LLM Sentry decides, and +// the reason records what Jev thought. Jev failing never blocks a list. func (f *Flock) Triage(ctx context.Context, w Watcher, item string, budgets ...*Budget) (Triage, error) { + if f.jev == nil { + return f.llmTriage(ctx, w, item, budgets...) + } + call, err := f.classify(ctx, w, item, budgets...) + if err != nil { + t, lerr := f.llmTriage(ctx, w, item, budgets...) + t.Reason = "Jev was unavailable; the LLM Sentry decided. " + t.Reason + return t, lerr + } + if call.confident(f.jev.threshold) && call.Mode != "split" { + return f.normalize(Triage{Mode: call.Mode, TaskType: call.TaskType, Reason: call.summary()}), nil + } + t, lerr := f.llmTriage(ctx, w, item, budgets...) + switch { + case !call.confident(f.jev.threshold): + t.Reason = fmt.Sprintf("%s; unsure, so the LLM Sentry decided. %s", call.summary(), t.Reason) + case t.Mode == "split": + t.Reason = fmt.Sprintf("%s; the LLM Sentry wrote the subtasks. %s", call.summary(), t.Reason) + default: + t.Reason = fmt.Sprintf("%s; the LLM Sentry disagreed and chose %s. %s", call.summary(), t.Mode, t.Reason) + } + return t, lerr +} + +func (f *Flock) llmTriage(ctx context.Context, w Watcher, item string, budgets ...*Budget) (Triage, error) { text, err := f.ask(ctx, w, "Sentry", sentrySystem, "ITEM: "+item, budgets...) if err != nil { return Triage{}, err @@ -172,6 +204,11 @@ func (f *Flock) Triage(ctx context.Context, w Watcher, item string, budgets ...* if i, j := strings.Index(text, "{"), strings.LastIndex(text, "}"); i < 0 || j < i || json.Unmarshal([]byte(text[i:j+1]), &t) != nil { t = Triage{Mode: "solo", TaskType: "other", Reason: "Sentry's call was unreadable; defaulting to solo"} } + return f.normalize(t), nil +} + +// normalize keeps any triage, from Jev or the LLM, inside the guardrails. +func (f *Flock) normalize(t Triage) Triage { t.Mode = strings.ToLower(strings.TrimSpace(t.Mode)) t.TaskType = strings.ToLower(strings.TrimSpace(t.TaskType)) switch t.TaskType { @@ -192,7 +229,7 @@ func (f *Flock) Triage(ctx context.Context, w Watcher, item string, budgets ...* default: t.Mode, t.Subtasks = "solo", nil } - return t, nil + return t } func (f *Flock) Execute(ctx context.Context, w Watcher, item string, t Triage, nest Nest, budgets ...*Budget) (Outcome, error) { diff --git a/jev.go b/jev.go new file mode 100644 index 0000000..eacd5b0 --- /dev/null +++ b/jev.go @@ -0,0 +1,169 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "strconv" + "time" +) + +// Jev (TypeSafe) is a classifier, not a text model: it returns typed choices +// with calibrated confidence. The Sentry uses it for triage, and escalates to +// the LLM when Jev is unsure, when an item needs subtasks written, or when Jev +// fails. Docs: https://docs.typesafe.ai/introduction/quickstart + +type JevClient struct { + url, key, model string + threshold float64 // both answers must reach this confidence to stand alone + client *http.Client +} + +// JevCall is one classification: a label and a confidence per question. +type JevCall struct { + Mode, TaskType string + ModeConf, TaskTypeConf float64 + Model string + Tokens int +} + +// NewJev returns nil (no classifier) unless TYPESAFE_API_KEY is set. +// +// TYPESAFE_API_KEY the key from the TypeSafe console +// CROWS_JEV_MODEL default jev-latest +// CROWS_JEV_THRESHOLD default 0.7 +func NewJev() *JevClient { + key := os.Getenv("TYPESAFE_API_KEY") + if key == "" { + return nil + } + model := os.Getenv("CROWS_JEV_MODEL") + if model == "" { + model = "jev-latest" + } + threshold := 0.7 + if v, err := strconv.ParseFloat(os.Getenv("CROWS_JEV_THRESHOLD"), 64); err == nil && v > 0 && v <= 1 { + threshold = v + } + return &JevClient{url: "https://api.typesafe.ai/v1/systemone", key: key, model: model, threshold: threshold, client: &http.Client{}} +} + +func (j *JevClient) Name() string { + return fmt.Sprintf("Jev (%s, escalates below %.0f%%)", j.model, j.threshold*100) +} + +var jevQuestions = map[string]any{ + "mode": map[string]any{ + "type": "choice", + "instructions": "How should a team of AI agents handle this todo item?", + "criteria": map[string]string{ + "solo": "Simple or single-step: one agent can answer it directly.", + "split": "Large, with 2 to 4 distinct parts that can be done independently and in parallel.", + "murder": "Ambiguous, high-stakes or a judgment call: several agents should answer independently, then reconcile.", + }, + }, + "task_type": map[string]any{ + "type": "choice", + "instructions": "What kind of task is this todo item?", + "criteria": map[string]string{ + "retrieval": "Looking up or recalling facts.", + "writing": "Drafting or editing text.", + "analysis": "Comparing, evaluating, deciding or reasoning about options.", + "other": "Anything else.", + }, + }, +} + +func (j *JevClient) Classify(ctx context.Context, item string) (JevCall, error) { + body, _ := json.Marshal(map[string]any{"model": j.model, "state": item, "questions": jevQuestions}) + req, err := http.NewRequestWithContext(ctx, "POST", j.url, bytes.NewReader(body)) + if err != nil { + return JevCall{}, err + } + req.Header.Set("Authorization", "Bearer "+j.key) + req.Header.Set("Content-Type", "application/json") + resp, err := j.client.Do(req) + if err != nil { + return JevCall{}, err + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusOK { + return JevCall{}, fmt.Errorf("Jev returned %d: %s", resp.StatusCode, clip(raw, 300)) + } + type answer struct { + Choice string `json:"choice"` + Confidence float64 `json:"confidence"` + } + var out struct { + Model string `json:"model"` + Answers map[string]answer `json:"answers"` + Usage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + } `json:"usage"` + } + if err := json.Unmarshal(raw, &out); err != nil { + return JevCall{}, fmt.Errorf("Jev sent an unreadable reply: %w", err) + } + mode, task := out.Answers["mode"], out.Answers["task_type"] + if mode.Choice == "" || task.Choice == "" { + return JevCall{}, errors.New("Jev's reply was missing an answer") + } + model := out.Model + if model == "" { + model = j.model + } + return JevCall{ + Mode: mode.Choice, TaskType: task.Choice, ModeConf: mode.Confidence, TaskTypeConf: task.Confidence, + Model: model, Tokens: out.Usage.InputTokens + out.Usage.OutputTokens, + }, nil +} + +func (c JevCall) confident(threshold float64) bool { + return c.ModeConf >= threshold && c.TaskTypeConf >= threshold +} + +func (c JevCall) summary() string { + return fmt.Sprintf("Jev: %s %.0f%%, %s %.0f%%", c.Mode, c.ModeConf*100, c.TaskType, c.TaskTypeConf*100) +} + +// classify runs Jev under the same guardrails as any crow: budget, flock cap, +// timeout, flight log, and flying/landed events for the Sentry. +func (f *Flock) classify(ctx context.Context, w Watcher, item string, budgets ...*Budget) (JevCall, error) { + for _, b := range budgets { + if err := b.Check(); err != nil { + return JevCall{}, err + } + } + f.sem <- struct{}{} + defer func() { <-f.sem }() + if w != nil { + w.Flying("Sentry") + } + cctx, cancel := context.WithTimeout(ctx, f.g.CrowTimeout) + defer cancel() + start := time.Now() + call, err := f.jev.Classify(cctx, item) + for _, b := range budgets { + b.Add(call.Tokens) + } + e := Event{Kind: "flight", Crow: "Sentry", Model: call.Model, Tokens: call.Tokens, Ms: time.Since(start).Milliseconds()} + if err != nil { + e.Error = err.Error() + } else { + e.Note = call.summary() + } + if f.log != nil { + f.log.Write(e) + } + if w != nil { + w.Landed("Sentry", call.Tokens, err) + } + return call, err +} diff --git a/jev_test.go b/jev_test.go new file mode 100644 index 0000000..36bf422 --- /dev/null +++ b/jev_test.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +// countingBrain is the LLM Sentry: it counts calls and always answers split with subtasks. +type countingBrain struct{ calls atomic.Int32 } + +func (b *countingBrain) Name() string { return "counting" } +func (b *countingBrain) Ask(ctx context.Context, system, prompt string) (string, int, error) { + b.calls.Add(1) + return `{"mode":"split","task_type":"writing","subtasks":["a","b"],"reason":"two parts"}`, 10, nil +} + +func fakeJev(t *testing.T, status int, mode string, modeConf float64, task string, taskConf float64) *JevClient { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer tk" { + t.Errorf("auth header = %q", r.Header.Get("Authorization")) + } + var body map[string]any + json.NewDecoder(r.Body).Decode(&body) + if body["state"] == nil || body["questions"] == nil { + t.Errorf("request missing state or questions: %v", body) + } + w.WriteHeader(status) + json.NewEncoder(w).Encode(map[string]any{ + "model": "jev-1.13.0", + "answers": map[string]any{ + "mode": map[string]any{"type": "choice", "choice": mode, "confidence": modeConf}, + "task_type": map[string]any{"type": "choice", "choice": task, "confidence": taskConf}, + }, + "usage": map[string]int{"input_tokens": 40, "output_tokens": 0}, + }) + })) + t.Cleanup(srv.Close) + return &JevClient{url: srv.URL, key: "tk", model: "jev-latest", threshold: 0.7, client: srv.Client()} +} + +func triageWith(t *testing.T, j *JevClient) (Triage, int32) { + t.Helper() + brain := &countingBrain{} + f := NewFlock(brain, DefaultGuardrails(), nil) + f.SetClassifier(j) + tr, err := f.Triage(context.Background(), nil, "some item") + if err != nil { + t.Fatal(err) + } + return tr, brain.calls.Load() +} + +func TestJevConfidentStandsAlone(t *testing.T) { + tr, llm := triageWith(t, fakeJev(t, 200, "murder", 0.91, "analysis", 0.88)) + if llm != 0 || tr.Mode != "murder" || tr.TaskType != "analysis" || !strings.Contains(tr.Reason, "Jev: murder 91%") { + t.Fatalf("got %+v with %d LLM calls; want Jev's call alone", tr, llm) + } +} + +func TestJevUnsureEscalates(t *testing.T) { + tr, llm := triageWith(t, fakeJev(t, 200, "solo", 0.55, "retrieval", 0.9)) + if llm != 1 || !strings.Contains(tr.Reason, "unsure") { + t.Fatalf("got %+v with %d LLM calls; want escalation", tr, llm) + } +} + +func TestJevSplitGetsSubtasksFromLLM(t *testing.T) { + tr, llm := triageWith(t, fakeJev(t, 200, "split", 0.95, "writing", 0.95)) + if llm != 1 || tr.Mode != "split" || len(tr.Subtasks) != 2 || !strings.Contains(tr.Reason, "wrote the subtasks") { + t.Fatalf("got %+v with %d LLM calls; want LLM-written subtasks", tr, llm) + } +} + +func TestJevDownFallsBackToLLM(t *testing.T) { + tr, llm := triageWith(t, fakeJev(t, 503, "solo", 0.9, "other", 0.9)) + if llm != 1 || !strings.Contains(tr.Reason, "Jev was unavailable") { + t.Fatalf("got %+v with %d LLM calls; want fallback", tr, llm) + } +} + +func TestNoJevKeyMeansNoClassifier(t *testing.T) { + t.Setenv("TYPESAFE_API_KEY", "") + if NewJev() != nil { + t.Fatal("expected no classifier without a key") + } +} diff --git a/main.go b/main.go index fa7e285..31681ee 100644 --- a/main.go +++ b/main.go @@ -90,6 +90,7 @@ func cmdRun(args []string) { brain := NewBrain() p.Flock = NewFlock(brain, g, flog) + p.Flock.SetClassifier(NewJev()) ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() diff --git a/server.go b/server.go index a0fa84b..8995dec 100644 --- a/server.go +++ b/server.go @@ -78,6 +78,11 @@ func cmdServe(args []string) { brain := NewBrain() approver := NewWebApprover(*timeout) p := &Pipeline{Flock: NewFlock(brain, g, flog), Ledger: ledger, Log: flog, G: g, Approver: approver} + jev := NewJev() + p.Flock.SetClassifier(jev) + if jev != nil { + fmt.Printf("Sentry triage: %s, LLM fallback\n", jev.Name()) + } s := NewServer(ctx, p, approver, nests) srv := &http.Server{Addr: *addr, Handler: s.Handler(), ReadHeaderTimeout: 10 * time.Second} diff --git a/start.command b/start.command index 28c2f01..c409025 100755 --- a/start.command +++ b/start.command @@ -7,6 +7,8 @@ # ANTHROPIC_API_KEY=... Claude direct from Anthropic (default: claude-sonnet-5-5) # CROWS_PROVIDER=openrouter optional: pick one when both keys are present # CROWS_MODEL=... optional: a specific model +# TYPESAFE_API_KEY=... Jev classifier for the Sentry's triage (LLM is the fallback) +# CROWS_JEV_THRESHOLD=0.7 optional: below this confidence, triage escalates to the LLM # `./start.command --mock` forces offline mock crows. # # Listens on this Mac only (127.0.0.1:8080). State lives in .crows/ (delete it to start fresh). @@ -17,9 +19,10 @@ chmod +x ./crows # Read only these lines from .env; the file is never executed. envval() { sed -n "s/^$1=//p" .env 2>/dev/null | head -n 1 | tr -d '[:space:]"'; } -unset ANTHROPIC_API_KEY OPENROUTER_API_KEY CROWS_PROVIDER CROWS_MODEL +VARS="ANTHROPIC_API_KEY OPENROUTER_API_KEY CROWS_PROVIDER CROWS_MODEL TYPESAFE_API_KEY CROWS_JEV_MODEL CROWS_JEV_THRESHOLD" +unset $VARS if [ "$1" != "--mock" ]; then - for v in ANTHROPIC_API_KEY OPENROUTER_API_KEY CROWS_PROVIDER CROWS_MODEL; do + for v in $VARS; do val="$(envval "$v")" [ -n "$val" ] && export "$v=$val" done