diff --git a/jev_test.go b/jev_test.go index 36bf422..f6e569f 100644 --- a/jev_test.go +++ b/jev_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" "strings" "sync/atomic" "testing" @@ -90,3 +91,91 @@ func TestNoJevKeyMeansNoClassifier(t *testing.T) { t.Fatal("expected no classifier without a key") } } + +var probeQuestions = map[string]any{ + "kind": map[string]any{ + "type": "choice", + "instructions": "What kind of message is this?", + "criteria": map[string]string{"ask": "Asks for work", "say": "Ordinary talk", "none": "Neither"}, + }, + "severity": map[string]any{ + "type": "score", + "instructions": "How severe is the problem in this message?", + "criteria": []string{"No problem", "Annoying", "Blocking"}, + }, + "is_request": map[string]any{ + "type": "noul", + "instructions": "Is the sender asking someone to do something?", + }, +} + +func TestParseMixedAnswers(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{ + "model": "jev-1.13.0", + "answers": map[string]any{ + "kind": map[string]any{"type": "choice", "choice": "ask", "confidence": 0.82}, + "severity": map[string]any{"type": "score", "score": 1.4, "confidence": 0.6, "legend": map[string]string{"0": "No problem", "1": "Annoying", "2": "Blocking"}, "probabilities": map[string]float64{"0": 0.1, "1": 0.5, "2": 0.4}}, + "is_request": map[string]any{"type": "noul", "noul": 0.95}, + }, + "usage": map[string]int{"input_tokens": 100, "output_tokens": 20}, + }) + })) + t.Cleanup(srv.Close) + j := &JevClient{url: srv.URL, key: "tk", model: "jev-latest", client: srv.Client()} + a, tokens, model, err := j.SystemOne(context.Background(), "state text", probeQuestions) + if err != nil { + t.Fatal(err) + } + if tokens != 120 || model != "jev-1.13.0" { + t.Errorf("got tokens=%d model=%s", tokens, model) + } + if a["kind"].Choice != "ask" || a["kind"].Confidence == nil || *a["kind"].Confidence != 0.82 { + t.Errorf("choice answer wrong: %+v", a["kind"]) + } + if a["severity"].Score == nil || *a["severity"].Score != 1.4 || a["severity"].Legend["2"] != "Blocking" { + t.Errorf("score answer wrong: %+v", a["severity"]) + } + if a["is_request"].Noul == nil || *a["is_request"].Noul != 0.95 { + t.Errorf("noul answer wrong: %+v", a["is_request"]) + } + if a["is_request"].Confidence != nil { + t.Errorf("noul must have no confidence, got %v", *a["is_request"].Confidence) + } +} + +func TestModeChoiceHasEscapeOption(t *testing.T) { + mode := jevQuestions["mode"].(map[string]any)["criteria"].(map[string]string) + if _, ok := mode["other"]; !ok { + t.Fatalf("mode has no escape option: %v", mode) + } +} + +func TestClassifySurvivesOtherMode(t *testing.T) { + tr, _ := triageWith(t, fakeJev(t, 200, "other", 0.95, "analysis", 0.9)) + if tr.Mode != "solo" { + t.Fatalf("Jev's other mode must normalize to solo, got %q", tr.Mode) + } +} + +func TestProbeScoreAndNoulAgainstLiveAPI(t *testing.T) { + if os.Getenv("TYPESAFE_API_KEY") == "" { + t.Skip("no TYPESAFE_API_KEY") + } + j := NewJev() + a, tokens, model, err := j.SystemOne(context.Background(), "@Signals, classify these drops. The export crashes on Safari.", probeQuestions) + if err != nil { + t.Fatalf("live probe failed: %v", err) + } + dump, _ := json.MarshalIndent(a, "", " ") + t.Logf("model=%s tokens=%d\n%s", model, tokens, dump) + if a["kind"].Choice == "" { + t.Errorf("choice answer missing: %+v", a["kind"]) + } + if a["severity"].Score == nil { + t.Errorf("score answer missing: %+v", a["severity"]) + } + if a["is_request"].Noul == nil { + t.Errorf("noul answer missing: %+v", a["is_request"]) + } +}