jev_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"strings"
 	"sync/atomic"
 	"testing"
@@ -90,3 +91,91 @@ func TestNoJevKeyMeansNoClassifier(t *testing.T) {
 		t.Fatal("expected no classifier without a key")
 	}
 }
+
+var probeQuestions = map[string]any{
+	"kind": map[string]any{
+		"type":         "choice",
+		"instructions": "What kind of message is this?",
+		"criteria":     map[string]string{"ask": "Asks for work", "say": "Ordinary talk", "none": "Neither"},
+	},
+	"severity": map[string]any{
+		"type":         "score",
+		"instructions": "How severe is the problem in this message?",
+		"criteria":     []string{"No problem", "Annoying", "Blocking"},
+	},
+	"is_request": map[string]any{
+		"type":         "noul",
+		"instructions": "Is the sender asking someone to do something?",
+	},
+}
+
+func TestParseMixedAnswers(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(map[string]any{
+			"model": "jev-1.13.0",
+			"answers": map[string]any{
+				"kind":       map[string]any{"type": "choice", "choice": "ask", "confidence": 0.82},
+				"severity":   map[string]any{"type": "score", "score": 1.4, "confidence": 0.6, "legend": map[string]string{"0": "No problem", "1": "Annoying", "2": "Blocking"}, "probabilities": map[string]float64{"0": 0.1, "1": 0.5, "2": 0.4}},
+				"is_request": map[string]any{"type": "noul", "noul": 0.95},
+			},
+			"usage": map[string]int{"input_tokens": 100, "output_tokens": 20},
+		})
+	}))
+	t.Cleanup(srv.Close)
+	j := &JevClient{url: srv.URL, key: "tk", model: "jev-latest", client: srv.Client()}
+	a, tokens, model, err := j.SystemOne(context.Background(), "state text", probeQuestions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tokens != 120 || model != "jev-1.13.0" {
+		t.Errorf("got tokens=%d model=%s", tokens, model)
+	}
+	if a["kind"].Choice != "ask" || a["kind"].Confidence == nil || *a["kind"].Confidence != 0.82 {
+		t.Errorf("choice answer wrong: %+v", a["kind"])
+	}
+	if a["severity"].Score == nil || *a["severity"].Score != 1.4 || a["severity"].Legend["2"] != "Blocking" {
+		t.Errorf("score answer wrong: %+v", a["severity"])
+	}
+	if a["is_request"].Noul == nil || *a["is_request"].Noul != 0.95 {
+		t.Errorf("noul answer wrong: %+v", a["is_request"])
+	}
+	if a["is_request"].Confidence != nil {
+		t.Errorf("noul must have no confidence, got %v", *a["is_request"].Confidence)
+	}
+}
+
+func TestModeChoiceHasEscapeOption(t *testing.T) {
+	mode := jevQuestions["mode"].(map[string]any)["criteria"].(map[string]string)
+	if _, ok := mode["other"]; !ok {
+		t.Fatalf("mode has no escape option: %v", mode)
+	}
+}
+
+func TestClassifySurvivesOtherMode(t *testing.T) {
+	tr, _ := triageWith(t, fakeJev(t, 200, "other", 0.95, "analysis", 0.9))
+	if tr.Mode != "solo" {
+		t.Fatalf("Jev's other mode must normalize to solo, got %q", tr.Mode)
+	}
+}
+
+func TestProbeScoreAndNoulAgainstLiveAPI(t *testing.T) {
+	if os.Getenv("TYPESAFE_API_KEY") == "" {
+		t.Skip("no TYPESAFE_API_KEY")
+	}
+	j := NewJev()
+	a, tokens, model, err := j.SystemOne(context.Background(), "@Signals, classify these drops. The export crashes on Safari.", probeQuestions)
+	if err != nil {
+		t.Fatalf("live probe failed: %v", err)
+	}
+	dump, _ := json.MarshalIndent(a, "", "  ")
+	t.Logf("model=%s tokens=%d\n%s", model, tokens, dump)
+	if a["kind"].Choice == "" {
+		t.Errorf("choice answer missing: %+v", a["kind"])
+	}
+	if a["severity"].Score == nil {
+		t.Errorf("score answer missing: %+v", a["severity"])
+	}
+	if a["is_request"].Noul == nil {
+		t.Errorf("noul answer missing: %+v", a["is_request"])
+	}
+}