diff --git a/JEV_STRATEGIES.md b/JEV_STRATEGIES.md index 721c8cb..ca1f2df 100644 --- a/JEV_STRATEGIES.md +++ b/JEV_STRATEGIES.md @@ -12,6 +12,8 @@ _Research 2026-10-27. Sources at the bottom. Vendor numbers (latency, price, cal - **Reported cost and speed (vendor):** about 70–500 ms per call; input billed (about $0.042 per million input tokens for Jev 1.13), output free. - **Limits:** text only, English-first, no explanations, no generated text. Early access with changing rate limits. It receives whatever is in `state`, so secrets must stay out. +**Verified 2026-10-27** (`TestProbeScoreAndNoulAgainstLiveAPI`, jev-1.14.0, 0.17 s, 466 tokens, one mixed call): `score` returns score/confidence/legend/probabilities as documented (severity 1.95 of 2, conf 0.93); `noul` returns only `noul` (0.94), **no confidence** — strategies 3 and 4 must threshold noul directly (docs suggest yes 0.7 / no 0.3, middle to review). Confidence is distribution concentration as warned: the split doc example (0.35) matches our probe's concentration reading. + **Crows today** uses one call with two `choice` questions (mode, task type) and a single global threshold of 0.7 (`jev.go`). So **custom labels are supported**, which answers spec open question 2: expectation checks and orchestration checks are just more questions. ## Strategies for accuracy diff --git a/jev.go b/jev.go index eacd5b0..fe6f7cb 100644 --- a/jev.go +++ b/jev.go @@ -65,6 +65,7 @@ var jevQuestions = map[string]any{ "solo": "Simple or single-step: one agent can answer it directly.", "split": "Large, with 2 to 4 distinct parts that can be done independently and in parallel.", "murder": "Ambiguous, high-stakes or a judgment call: several agents should answer independently, then reconcile.", + "other": "None of the above fits well.", }, }, "task_type": map[string]any{ @@ -79,52 +80,81 @@ var jevQuestions = map[string]any{ }, } -func (j *JevClient) Classify(ctx context.Context, item string) (JevCall, error) { - body, _ := json.Marshal(map[string]any{"model": j.model, "state": item, "questions": jevQuestions}) +// Answer is one Jev verdict, any question type. Pointed fields are present only +// for their type: choice sets Choice/Confidence, score sets Score/Confidence, +// noul sets Noul (noul has no confidence — see JEV_STRATEGIES.md). +type Answer struct { + Type string `json:"type"` + Choice string `json:"choice,omitempty"` + Confidence *float64 `json:"confidence,omitempty"` + Score *float64 `json:"score,omitempty"` + Noul *float64 `json:"noul,omitempty"` + Probabilities map[string]float64 `json:"probabilities,omitempty"` + Legend map[string]any `json:"legend,omitempty"` +} + +// SystemOne POSTs one state and a set of questions, mixing types freely, and +// returns the answers keyed by question name. It is the single network path for +// every Jev call (Classify and the probes), so auth, status and usage handling +// stays in one place. Returns total tokens and the model that answered. +func (j *JevClient) SystemOne(ctx context.Context, state any, questions map[string]any) (map[string]Answer, int, string, error) { + body, _ := json.Marshal(map[string]any{"model": j.model, "state": state, "questions": questions}) req, err := http.NewRequestWithContext(ctx, "POST", j.url, bytes.NewReader(body)) if err != nil { - return JevCall{}, err + return nil, 0, "", err } req.Header.Set("Authorization", "Bearer "+j.key) req.Header.Set("Content-Type", "application/json") resp, err := j.client.Do(req) if err != nil { - return JevCall{}, err + return nil, 0, "", err } defer resp.Body.Close() raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) if resp.StatusCode != http.StatusOK { - return JevCall{}, fmt.Errorf("Jev returned %d: %s", resp.StatusCode, clip(raw, 300)) - } - type answer struct { - Choice string `json:"choice"` - Confidence float64 `json:"confidence"` + return nil, 0, "", fmt.Errorf("Jev returned %d: %s", resp.StatusCode, clip(raw, 300)) } var out struct { Model string `json:"model"` - Answers map[string]answer `json:"answers"` + Answers map[string]Answer `json:"answers"` Usage struct { InputTokens int `json:"input_tokens"` OutputTokens int `json:"output_tokens"` } `json:"usage"` } if err := json.Unmarshal(raw, &out); err != nil { - return JevCall{}, fmt.Errorf("Jev sent an unreadable reply: %w", err) - } - mode, task := out.Answers["mode"], out.Answers["task_type"] - if mode.Choice == "" || task.Choice == "" { - return JevCall{}, errors.New("Jev's reply was missing an answer") + return nil, 0, "", fmt.Errorf("Jev sent an unreadable reply: %w", err) } model := out.Model if model == "" { model = j.model } + return out.Answers, out.Usage.InputTokens + out.Usage.OutputTokens, model, nil +} + +func (j *JevClient) Classify(ctx context.Context, item string) (JevCall, error) { + answers, tokens, model, err := j.SystemOne(ctx, item, jevQuestions) + if err != nil { + return JevCall{}, err + } + mode, task := answers["mode"], answers["task_type"] + if mode.Choice == "" || task.Choice == "" { + return JevCall{}, errors.New("Jev's reply was missing an answer") + } return JevCall{ - Mode: mode.Choice, TaskType: task.Choice, ModeConf: mode.Confidence, TaskTypeConf: task.Confidence, - Model: model, Tokens: out.Usage.InputTokens + out.Usage.OutputTokens, + Mode: mode.Choice, TaskType: task.Choice, + ModeConf: deref(mode.Confidence), TaskTypeConf: deref(task.Confidence), + Model: model, Tokens: tokens, }, nil } +func deref(f *float64) float64 { + if f == nil { + return 0 + } + return *f +} + func (c JevCall) confident(threshold float64) bool { return c.ModeConf >= threshold && c.TaskTypeConf >= threshold } diff --git a/jev_test.go b/jev_test.go index 36bf422..f6e569f 100644 --- a/jev_test.go +++ b/jev_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" "strings" "sync/atomic" "testing" @@ -90,3 +91,91 @@ func TestNoJevKeyMeansNoClassifier(t *testing.T) { t.Fatal("expected no classifier without a key") } } + +var probeQuestions = map[string]any{ + "kind": map[string]any{ + "type": "choice", + "instructions": "What kind of message is this?", + "criteria": map[string]string{"ask": "Asks for work", "say": "Ordinary talk", "none": "Neither"}, + }, + "severity": map[string]any{ + "type": "score", + "instructions": "How severe is the problem in this message?", + "criteria": []string{"No problem", "Annoying", "Blocking"}, + }, + "is_request": map[string]any{ + "type": "noul", + "instructions": "Is the sender asking someone to do something?", + }, +} + +func TestParseMixedAnswers(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{ + "model": "jev-1.13.0", + "answers": map[string]any{ + "kind": map[string]any{"type": "choice", "choice": "ask", "confidence": 0.82}, + "severity": map[string]any{"type": "score", "score": 1.4, "confidence": 0.6, "legend": map[string]string{"0": "No problem", "1": "Annoying", "2": "Blocking"}, "probabilities": map[string]float64{"0": 0.1, "1": 0.5, "2": 0.4}}, + "is_request": map[string]any{"type": "noul", "noul": 0.95}, + }, + "usage": map[string]int{"input_tokens": 100, "output_tokens": 20}, + }) + })) + t.Cleanup(srv.Close) + j := &JevClient{url: srv.URL, key: "tk", model: "jev-latest", client: srv.Client()} + a, tokens, model, err := j.SystemOne(context.Background(), "state text", probeQuestions) + if err != nil { + t.Fatal(err) + } + if tokens != 120 || model != "jev-1.13.0" { + t.Errorf("got tokens=%d model=%s", tokens, model) + } + if a["kind"].Choice != "ask" || a["kind"].Confidence == nil || *a["kind"].Confidence != 0.82 { + t.Errorf("choice answer wrong: %+v", a["kind"]) + } + if a["severity"].Score == nil || *a["severity"].Score != 1.4 || a["severity"].Legend["2"] != "Blocking" { + t.Errorf("score answer wrong: %+v", a["severity"]) + } + if a["is_request"].Noul == nil || *a["is_request"].Noul != 0.95 { + t.Errorf("noul answer wrong: %+v", a["is_request"]) + } + if a["is_request"].Confidence != nil { + t.Errorf("noul must have no confidence, got %v", *a["is_request"].Confidence) + } +} + +func TestModeChoiceHasEscapeOption(t *testing.T) { + mode := jevQuestions["mode"].(map[string]any)["criteria"].(map[string]string) + if _, ok := mode["other"]; !ok { + t.Fatalf("mode has no escape option: %v", mode) + } +} + +func TestClassifySurvivesOtherMode(t *testing.T) { + tr, _ := triageWith(t, fakeJev(t, 200, "other", 0.95, "analysis", 0.9)) + if tr.Mode != "solo" { + t.Fatalf("Jev's other mode must normalize to solo, got %q", tr.Mode) + } +} + +func TestProbeScoreAndNoulAgainstLiveAPI(t *testing.T) { + if os.Getenv("TYPESAFE_API_KEY") == "" { + t.Skip("no TYPESAFE_API_KEY") + } + j := NewJev() + a, tokens, model, err := j.SystemOne(context.Background(), "@Signals, classify these drops. The export crashes on Safari.", probeQuestions) + if err != nil { + t.Fatalf("live probe failed: %v", err) + } + dump, _ := json.MarshalIndent(a, "", " ") + t.Logf("model=%s tokens=%d\n%s", model, tokens, dump) + if a["kind"].Choice == "" { + t.Errorf("choice answer missing: %+v", a["kind"]) + } + if a["severity"].Score == nil { + t.Errorf("score answer missing: %+v", a["severity"]) + } + if a["is_request"].Noul == nil { + t.Errorf("noul answer missing: %+v", a["is_request"]) + } +}