Merge pull request #14 from jacisjake/task/7-jev-score-noul-probe

(#7) Probe Jev score/noul; escape option on mode

JEV_STRATEGIES.md

@@ -12,6 +12,8 @@ _Research 2026-10-27. Sources at the bottom. Vendor numbers (latency, price, cal
 - **Reported cost and speed (vendor):** about 70–500 ms per call; input billed (about $0.042 per million input tokens for Jev 1.13), output free.
 - **Limits:** text only, English-first, no explanations, no generated text. Early access with changing rate limits. It receives whatever is in `state`, so secrets must stay out.
 
+**Verified 2026-10-27** (`TestProbeScoreAndNoulAgainstLiveAPI`, jev-1.14.0, 0.17 s, 466 tokens, one mixed call): `score` returns score/confidence/legend/probabilities as documented (severity 1.95 of 2, conf 0.93); `noul` returns only `noul` (0.94), **no confidence** — strategies 3 and 4 must threshold noul directly (docs suggest yes 0.7 / no 0.3, middle to review). Confidence is distribution concentration as warned: the split doc example (0.35) matches our probe's concentration reading.
+
 **Crows today** uses one call with two `choice` questions (mode, task type) and a single global threshold of 0.7 (`jev.go`). So **custom labels are supported**, which answers spec open question 2: expectation checks and orchestration checks are just more questions.
 
 ## Strategies for accuracy

jev.go

@@ -65,6 +65,7 @@ var jevQuestions = map[string]any{
 			"solo":   "Simple or single-step: one agent can answer it directly.",
 			"split":  "Large, with 2 to 4 distinct parts that can be done independently and in parallel.",
 			"murder": "Ambiguous, high-stakes or a judgment call: several agents should answer independently, then reconcile.",
+			"other":  "None of the above fits well.",
 		},
 	},
 	"task_type": map[string]any{
@@ -79,52 +80,81 @@ var jevQuestions = map[string]any{
 	},
 }
 
-func (j *JevClient) Classify(ctx context.Context, item string) (JevCall, error) {
-	body, _ := json.Marshal(map[string]any{"model": j.model, "state": item, "questions": jevQuestions})
+// Answer is one Jev verdict, any question type. Pointed fields are present only
+// for their type: choice sets Choice/Confidence, score sets Score/Confidence,
+// noul sets Noul (noul has no confidence — see JEV_STRATEGIES.md).
+type Answer struct {
+	Type          string             `json:"type"`
+	Choice        string             `json:"choice,omitempty"`
+	Confidence    *float64           `json:"confidence,omitempty"`
+	Score         *float64           `json:"score,omitempty"`
+	Noul          *float64           `json:"noul,omitempty"`
+	Probabilities map[string]float64 `json:"probabilities,omitempty"`
+	Legend        map[string]any     `json:"legend,omitempty"`
+}
+
+// SystemOne POSTs one state and a set of questions, mixing types freely, and
+// returns the answers keyed by question name. It is the single network path for
+// every Jev call (Classify and the probes), so auth, status and usage handling
+// stays in one place. Returns total tokens and the model that answered.
+func (j *JevClient) SystemOne(ctx context.Context, state any, questions map[string]any) (map[string]Answer, int, string, error) {
+	body, _ := json.Marshal(map[string]any{"model": j.model, "state": state, "questions": questions})
 	req, err := http.NewRequestWithContext(ctx, "POST", j.url, bytes.NewReader(body))
 	if err != nil {
-		return JevCall{}, err
+		return nil, 0, "", err
 	}
 	req.Header.Set("Authorization", "Bearer "+j.key)
 	req.Header.Set("Content-Type", "application/json")
 	resp, err := j.client.Do(req)
 	if err != nil {
-		return JevCall{}, err
+		return nil, 0, "", err
 	}
 	defer resp.Body.Close()
 	raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
 	if resp.StatusCode != http.StatusOK {
-		return JevCall{}, fmt.Errorf("Jev returned %d: %s", resp.StatusCode, clip(raw, 300))
-	}
-	type answer struct {
-		Choice     string  `json:"choice"`
-		Confidence float64 `json:"confidence"`
+		return nil, 0, "", fmt.Errorf("Jev returned %d: %s", resp.StatusCode, clip(raw, 300))
 	}
 	var out struct {
 		Model   string            `json:"model"`
-		Answers map[string]answer `json:"answers"`
+		Answers map[string]Answer `json:"answers"`
 		Usage   struct {
 			InputTokens  int `json:"input_tokens"`
 			OutputTokens int `json:"output_tokens"`
 		} `json:"usage"`
 	}
 	if err := json.Unmarshal(raw, &out); err != nil {
-		return JevCall{}, fmt.Errorf("Jev sent an unreadable reply: %w", err)
-	}
-	mode, task := out.Answers["mode"], out.Answers["task_type"]
-	if mode.Choice == "" || task.Choice == "" {
-		return JevCall{}, errors.New("Jev's reply was missing an answer")
+		return nil, 0, "", fmt.Errorf("Jev sent an unreadable reply: %w", err)
 	}
 	model := out.Model
 	if model == "" {
 		model = j.model
 	}
+	return out.Answers, out.Usage.InputTokens + out.Usage.OutputTokens, model, nil
+}
+
+func (j *JevClient) Classify(ctx context.Context, item string) (JevCall, error) {
+	answers, tokens, model, err := j.SystemOne(ctx, item, jevQuestions)
+	if err != nil {
+		return JevCall{}, err
+	}
+	mode, task := answers["mode"], answers["task_type"]
+	if mode.Choice == "" || task.Choice == "" {
+		return JevCall{}, errors.New("Jev's reply was missing an answer")
+	}
 	return JevCall{
-		Mode: mode.Choice, TaskType: task.Choice, ModeConf: mode.Confidence, TaskTypeConf: task.Confidence,
-		Model: model, Tokens: out.Usage.InputTokens + out.Usage.OutputTokens,
+		Mode: mode.Choice, TaskType: task.Choice,
+		ModeConf: deref(mode.Confidence), TaskTypeConf: deref(task.Confidence),
+		Model: model, Tokens: tokens,
 	}, nil
 }
 
+func deref(f *float64) float64 {
+	if f == nil {
+		return 0
+	}
+	return *f
+}
+
 func (c JevCall) confident(threshold float64) bool {
 	return c.ModeConf >= threshold && c.TaskTypeConf >= threshold
 }

jev_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"strings"
 	"sync/atomic"
 	"testing"
@@ -90,3 +91,91 @@ func TestNoJevKeyMeansNoClassifier(t *testing.T) {
 		t.Fatal("expected no classifier without a key")
 	}
 }
+
+var probeQuestions = map[string]any{
+	"kind": map[string]any{
+		"type":         "choice",
+		"instructions": "What kind of message is this?",
+		"criteria":     map[string]string{"ask": "Asks for work", "say": "Ordinary talk", "none": "Neither"},
+	},
+	"severity": map[string]any{
+		"type":         "score",
+		"instructions": "How severe is the problem in this message?",
+		"criteria":     []string{"No problem", "Annoying", "Blocking"},
+	},
+	"is_request": map[string]any{
+		"type":         "noul",
+		"instructions": "Is the sender asking someone to do something?",
+	},
+}
+
+func TestParseMixedAnswers(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(map[string]any{
+			"model": "jev-1.13.0",
+			"answers": map[string]any{
+				"kind":       map[string]any{"type": "choice", "choice": "ask", "confidence": 0.82},
+				"severity":   map[string]any{"type": "score", "score": 1.4, "confidence": 0.6, "legend": map[string]string{"0": "No problem", "1": "Annoying", "2": "Blocking"}, "probabilities": map[string]float64{"0": 0.1, "1": 0.5, "2": 0.4}},
+				"is_request": map[string]any{"type": "noul", "noul": 0.95},
+			},
+			"usage": map[string]int{"input_tokens": 100, "output_tokens": 20},
+		})
+	}))
+	t.Cleanup(srv.Close)
+	j := &JevClient{url: srv.URL, key: "tk", model: "jev-latest", client: srv.Client()}
+	a, tokens, model, err := j.SystemOne(context.Background(), "state text", probeQuestions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tokens != 120 || model != "jev-1.13.0" {
+		t.Errorf("got tokens=%d model=%s", tokens, model)
+	}
+	if a["kind"].Choice != "ask" || a["kind"].Confidence == nil || *a["kind"].Confidence != 0.82 {
+		t.Errorf("choice answer wrong: %+v", a["kind"])
+	}
+	if a["severity"].Score == nil || *a["severity"].Score != 1.4 || a["severity"].Legend["2"] != "Blocking" {
+		t.Errorf("score answer wrong: %+v", a["severity"])
+	}
+	if a["is_request"].Noul == nil || *a["is_request"].Noul != 0.95 {
+		t.Errorf("noul answer wrong: %+v", a["is_request"])
+	}
+	if a["is_request"].Confidence != nil {
+		t.Errorf("noul must have no confidence, got %v", *a["is_request"].Confidence)
+	}
+}
+
+func TestModeChoiceHasEscapeOption(t *testing.T) {
+	mode := jevQuestions["mode"].(map[string]any)["criteria"].(map[string]string)
+	if _, ok := mode["other"]; !ok {
+		t.Fatalf("mode has no escape option: %v", mode)
+	}
+}
+
+func TestClassifySurvivesOtherMode(t *testing.T) {
+	tr, _ := triageWith(t, fakeJev(t, 200, "other", 0.95, "analysis", 0.9))
+	if tr.Mode != "solo" {
+		t.Fatalf("Jev's other mode must normalize to solo, got %q", tr.Mode)
+	}
+}
+
+func TestProbeScoreAndNoulAgainstLiveAPI(t *testing.T) {
+	if os.Getenv("TYPESAFE_API_KEY") == "" {
+		t.Skip("no TYPESAFE_API_KEY")
+	}
+	j := NewJev()
+	a, tokens, model, err := j.SystemOne(context.Background(), "@Signals, classify these drops. The export crashes on Safari.", probeQuestions)
+	if err != nil {
+		t.Fatalf("live probe failed: %v", err)
+	}
+	dump, _ := json.MarshalIndent(a, "", "  ")
+	t.Logf("model=%s tokens=%d\n%s", model, tokens, dump)
+	if a["kind"].Choice == "" {
+		t.Errorf("choice answer missing: %+v", a["kind"])
+	}
+	if a["severity"].Score == nil {
+		t.Errorf("score answer missing: %+v", a["severity"])
+	}
+	if a["is_request"].Noul == nil {
+		t.Errorf("noul answer missing: %+v", a["is_request"])
+	}
+}