so-many-crows/jev.go

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"time"
)

// Jev (TypeSafe) is a classifier, not a text model: it returns typed choices
// with calibrated confidence. The Sentry uses it for triage, and escalates to
// the LLM when Jev is unsure, when an item needs subtasks written, or when Jev
// fails. Docs: https://docs.typesafe.ai/introduction/quickstart

type JevClient struct {
	url, key, model string
	threshold       float64 // both answers must reach this confidence to stand alone
	client          *http.Client
}

// JevCall is one classification: a label and a confidence per question.
type JevCall struct {
	Mode, TaskType         string
	ModeConf, TaskTypeConf float64
	Model                  string
	Tokens                 int
}

// NewJev returns nil (no classifier) unless TYPESAFE_API_KEY is set.
//
//	TYPESAFE_API_KEY         the key from the TypeSafe console
//	CROWS_JEV_MODEL          default jev-latest
//	CROWS_JEV_THRESHOLD      default 0.7
func NewJev() *JevClient {
	key := os.Getenv("TYPESAFE_API_KEY")
	if key == "" {
		return nil
	}
	model := os.Getenv("CROWS_JEV_MODEL")
	if model == "" {
		model = "jev-latest"
	}
	threshold := 0.7
	if v, err := strconv.ParseFloat(os.Getenv("CROWS_JEV_THRESHOLD"), 64); err == nil && v > 0 && v <= 1 {
		threshold = v
	}
	return &JevClient{url: "https://api.typesafe.ai/v1/systemone", key: key, model: model, threshold: threshold, client: &http.Client{}}
}

func (j *JevClient) Name() string {
	return fmt.Sprintf("Jev (%s, escalates below %.0f%%)", j.model, j.threshold*100)
}

var jevQuestions = map[string]any{
	"mode": map[string]any{
		"type":         "choice",
		"instructions": "How should a team of AI agents handle this todo item?",
		"criteria": map[string]string{
			"solo":   "Simple or single-step: one agent can answer it directly.",
			"split":  "Large, with 2 to 4 distinct parts that can be done independently and in parallel.",
			"murder": "Ambiguous, high-stakes or a judgment call: several agents should answer independently, then reconcile.",
			"other":  "None of the above fits well.",
		},
	},
	"task_type": map[string]any{
		"type":         "choice",
		"instructions": "What kind of task is this todo item?",
		"criteria": map[string]string{
			"retrieval": "Looking up or recalling facts.",
			"writing":   "Drafting or editing text.",
			"analysis":  "Comparing, evaluating, deciding or reasoning about options.",
			"other":     "Anything else.",
		},
	},
}

// Answer is one Jev verdict, any question type. Pointed fields are present only
// for their type: choice sets Choice/Confidence, score sets Score/Confidence,
// noul sets Noul (noul has no confidence — see JEV_STRATEGIES.md).
type Answer struct {
	Type          string             `json:"type"`
	Choice        string             `json:"choice,omitempty"`
	Confidence    *float64           `json:"confidence,omitempty"`
	Score         *float64           `json:"score,omitempty"`
	Noul          *float64           `json:"noul,omitempty"`
	Probabilities map[string]float64 `json:"probabilities,omitempty"`
	Legend        map[string]any     `json:"legend,omitempty"`
}

// SystemOne POSTs one state and a set of questions, mixing types freely, and
// returns the answers keyed by question name. It is the single network path for
// every Jev call (Classify and the probes), so auth, status and usage handling
// stays in one place. Returns total tokens and the model that answered.
func (j *JevClient) SystemOne(ctx context.Context, state any, questions map[string]any) (map[string]Answer, int, string, error) {
	body, _ := json.Marshal(map[string]any{"model": j.model, "state": state, "questions": questions})
	req, err := http.NewRequestWithContext(ctx, "POST", j.url, bytes.NewReader(body))
	if err != nil {
		return nil, 0, "", err
	}
	req.Header.Set("Authorization", "Bearer "+j.key)
	req.Header.Set("Content-Type", "application/json")
	resp, err := j.client.Do(req)
	if err != nil {
		return nil, 0, "", err
	}
	defer resp.Body.Close()
	raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
	if resp.StatusCode != http.StatusOK {
		return nil, 0, "", fmt.Errorf("Jev returned %d: %s", resp.StatusCode, clip(raw, 300))
	}
	var out struct {
		Model   string            `json:"model"`
		Answers map[string]Answer `json:"answers"`
		Usage   struct {
			InputTokens  int `json:"input_tokens"`
			OutputTokens int `json:"output_tokens"`
		} `json:"usage"`
	}
	if err := json.Unmarshal(raw, &out); err != nil {
		return nil, 0, "", fmt.Errorf("Jev sent an unreadable reply: %w", err)
	}
	model := out.Model
	if model == "" {
		model = j.model
	}
	return out.Answers, out.Usage.InputTokens + out.Usage.OutputTokens, model, nil
}

func (j *JevClient) Classify(ctx context.Context, item string) (JevCall, error) {
	answers, tokens, model, err := j.SystemOne(ctx, item, jevQuestions)
	if err != nil {
		return JevCall{}, err
	}
	mode, task := answers["mode"], answers["task_type"]
	if mode.Choice == "" || task.Choice == "" {
		return JevCall{}, errors.New("Jev's reply was missing an answer")
	}
	return JevCall{
		Mode: mode.Choice, TaskType: task.Choice,
		ModeConf: deref(mode.Confidence), TaskTypeConf: deref(task.Confidence),
		Model: model, Tokens: tokens,
	}, nil
}

func deref(f *float64) float64 {
	if f == nil {
		return 0
	}
	return *f
}

// modeQuestions classifies how a request should be taken up (Epic C). The
// escape option "other" keeps Jev from forcing a wrong class; mode is discrete,
// so choice, not score (JEV_STRATEGIES.md strategy 3).
var modeQuestions = map[string]any{
	"mode": map[string]any{
		"type":         "choice",
		"instructions": "How should this chatroom request be taken up?",
		"criteria": map[string]string{
			"solo":       "Simple: one crow answers it directly.",
			"conspiracy": "A judgment call worth several distinct readings: several crows take the same request from their different strengths.",
			"split":      "Large with distinct parts: separate subtasks, one owner each.",
			"other":      "None of these fits well.",
		},
	},
}

// ClassifyMode returns the mode when Jev is set and its choice is at or above
// the band. "other" reads as unsure (not ok) so callers fall back to a header.
func (j *JevClient) ClassifyMode(ctx context.Context, text string) (string, map[string]float64, bool) {
	if j == nil {
		return "", nil, false
	}
	answers, _, _, err := j.SystemOne(ctx, text, modeQuestions)
	if err != nil {
		return "", nil, false
	}
	a := answers["mode"]
	if a.Choice == "" || a.Choice == "other" || a.Confidence == nil || *a.Confidence < j.threshold {
		return "", nil, false
	}
	return a.Choice, a.Probabilities, true
}

// ClassifyMode classifies a request under the same guardrails as any call:
// flock cap, timeout and flight log, with the probabilities recorded. Returns
// the mode and ok; below-band or absent Jev means not ok.
func (f *Flock) ClassifyMode(ctx context.Context, text string, budgets ...*Budget) (string, bool) {
	if f.jev == nil {
		return "", false
	}
	for _, b := range budgets {
		if err := b.Check(); err != nil {
			return "", false
		}
	}
	f.sem <- struct{}{}
	defer func() { <-f.sem }()
	cctx, cancel := context.WithTimeout(ctx, f.g.CrowTimeout)
	defer cancel()
	start := time.Now()
	mode, probs, ok := f.jev.ClassifyMode(cctx, text)
	e := Event{Kind: "mode", Item: text, Crow: "Sentry", Model: f.jev.model, Ms: time.Since(start).Milliseconds(), Prob: probs, Mode: mode}
	if f.log != nil {
		f.log.Write(e)
	}
	return mode, ok
}

func (c JevCall) confident(threshold float64) bool {
	return c.ModeConf >= threshold && c.TaskTypeConf >= threshold
}

// kindQuestions tags a nest message with its kind (A1: Jev first). The escape
// option "say" keeps Jev from forcing a wrong label: any message it isn't sure
// about falls back to the safest kind.
var kindQuestions = map[string]any{
	"kind": map[string]any{
		"type":         "choice",
		"instructions": "What kind of chatroom message is this?",
		"criteria": map[string]string{
			"request": "Asks a member or the room for work.",
			"claim":   "Takes ownership of a request.",
			"deliver": "Posts the result of a request.",
			"refer":   "Points to a file or artifact instead of pasting it.",
			"say":     "Ordinary talk, or unclear.",
		},
	},
}

// ClassifyKind returns the kind when Jev is set and its choice sits at or
// above the confidence band, with the choice probabilities for the log. Not ok
// means the caller falls back to the poster's header (EVENTS.md tagging rule).
func (j *JevClient) ClassifyKind(ctx context.Context, text string) (string, map[string]float64, bool) {
	if j == nil {
		return "", nil, false
	}
	answers, _, _, err := j.SystemOne(ctx, text, kindQuestions)
	if err != nil {
		return "", nil, false
	}
	a := answers["kind"]
	if a.Choice == "" || a.Confidence == nil || *a.Confidence < j.threshold {
		return "", nil, false
	}
	return a.Choice, a.Probabilities, true
}

// divergenceQuestions asks whether a message restates an earlier point. It is
// a noul: JEV_STRATEGIES strategy 3 (crisp yes/no) and noul has no confidence,
// so the probability itself is thresholded (band = threshold).
var divergenceQuestions = map[string]any{
	"restates_earlier": map[string]any{
		"type":         "noul",
		"instructions": "Does `this` make a point already made in `earlier`, adding no new angle?",
		"criteria": map[string]string{
			"true":  "It repeats an earlier point without a genuinely new angle.",
			"false": "It adds something different from what `earlier` already said.",
		},
	},
}

// ClassifyDivergence returns the probability that a message restates earlier
// ones, and ok when a value came back (noul always carries one; ok is false
// only on transport error or a missing answer).
func (j *JevClient) ClassifyDivergence(ctx context.Context, state any) (float64, bool) {
	if j == nil {
		return 0, false
	}
	answers, _, _, err := j.SystemOne(ctx, state, divergenceQuestions)
	if err != nil {
		return 0, false
	}
	a := answers["restates_earlier"]
	if a.Noul == nil {
		return 0, false
	}
	return *a.Noul, true
}

// ClassifyDivergence runs the restatement check under the flock guardrails
// (cap, timeout, flight log with the probability recorded).
func (f *Flock) ClassifyDivergence(ctx context.Context, state any, budgets ...*Budget) (float64, bool) {
	if f.jev == nil {
		return 0, false
	}
	for _, b := range budgets {
		if err := b.Check(); err != nil {
			return 0, false
		}
	}
	f.sem <- struct{}{}
	defer func() { <-f.sem }()
	cctx, cancel := context.WithTimeout(ctx, f.g.CrowTimeout)
	defer cancel()
	start := time.Now()
	p, ok := f.jev.ClassifyDivergence(cctx, state)
	if f.log != nil {
		f.log.Write(Event{Kind: "divergence", Crow: "Sentry", Model: f.jev.model, Ms: time.Since(start).Milliseconds(), Prob: map[string]float64{"restates_earlier": p}, Note: fmt.Sprint(ok && p >= f.jev.threshold)})
	}
	return p, ok
}

// ClassifyKind tags a nest message under the same guardrails as any call:
// flock cap, timeout and flight log, with the probabilities recorded.
func (f *Flock) ClassifyKind(ctx context.Context, text string, budgets ...*Budget) (string, map[string]float64, bool) {
	if f.jev == nil {
		return "", nil, false
	}
	for _, b := range budgets {
		if err := b.Check(); err != nil {
			return "", nil, false
		}
	}
	f.sem <- struct{}{}
	defer func() { <-f.sem }()
	cctx, cancel := context.WithTimeout(ctx, f.g.CrowTimeout)
	defer cancel()
	start := time.Now()
	kind, probs, ok := f.jev.ClassifyKind(cctx, text)
	e := Event{Kind: "tag", Item: text, Crow: "Sentry", Model: f.jev.model, Ms: time.Since(start).Milliseconds(), Prob: probs, Note: kind}
	if !ok {
		e.Note = "" // unsure: the header decides
	}
	if f.log != nil {
		f.log.Write(e)
	}
	return kind, probs, ok
}

func (c JevCall) summary() string {
	return fmt.Sprintf("Jev: %s %.0f%%, %s %.0f%%", c.Mode, c.ModeConf*100, c.TaskType, c.TaskTypeConf*100)
}

// classify runs Jev under the same guardrails as any crow: budget, flock cap,
// timeout, flight log, and flying/landed events for the Sentry.
func (f *Flock) classify(ctx context.Context, w Watcher, item string, budgets ...*Budget) (JevCall, error) {
	for _, b := range budgets {
		if err := b.Check(); err != nil {
			return JevCall{}, err
		}
	}
	f.sem <- struct{}{}
	defer func() { <-f.sem }()
	if w != nil {
		w.Flying("Sentry")
	}
	cctx, cancel := context.WithTimeout(ctx, f.g.CrowTimeout)
	defer cancel()
	start := time.Now()
	call, err := f.jev.Classify(cctx, item)
	for _, b := range budgets {
		b.Add(call.Tokens)
	}
	e := Event{Kind: "flight", Crow: "Sentry", Model: call.Model, Tokens: call.Tokens, Ms: time.Since(start).Milliseconds()}
	if err != nil {
		e.Error = err.Error()
	} else {
		e.Note = call.summary()
	}
	if f.log != nil {
		f.log.Write(e)
	}
	if w != nil {
		w.Landed("Sentry", call.Tokens, err)
	}
	return call, err
}