feat(jev): SystemOne parser for choice/score/noul, escape option on mode (#7)
This commit is contained in:
1 parent
3b573cea97
commit
ac95ab1480
1 file changed
+47
-17
@@ -65,6 +65,7 @@ var jevQuestions = map[string]any{
|
||||
"solo": "Simple or single-step: one agent can answer it directly.",
|
||||
"split": "Large, with 2 to 4 distinct parts that can be done independently and in parallel.",
|
||||
"murder": "Ambiguous, high-stakes or a judgment call: several agents should answer independently, then reconcile.",
|
||||
"other": "None of the above fits well.",
|
||||
},
|
||||
},
|
||||
"task_type": map[string]any{
|
||||
@@ -79,52 +80,81 @@ var jevQuestions = map[string]any{
|
||||
},
|
||||
}
|
||||
|
||||
func (j *JevClient) Classify(ctx context.Context, item string) (JevCall, error) {
|
||||
body, _ := json.Marshal(map[string]any{"model": j.model, "state": item, "questions": jevQuestions})
|
||||
// Answer is one Jev verdict, any question type. Pointed fields are present only
|
||||
// for their type: choice sets Choice/Confidence, score sets Score/Confidence,
|
||||
// noul sets Noul (noul has no confidence — see JEV_STRATEGIES.md).
|
||||
type Answer struct {
|
||||
Type string `json:"type"`
|
||||
Choice string `json:"choice,omitempty"`
|
||||
Confidence *float64 `json:"confidence,omitempty"`
|
||||
Score *float64 `json:"score,omitempty"`
|
||||
Noul *float64 `json:"noul,omitempty"`
|
||||
Probabilities map[string]float64 `json:"probabilities,omitempty"`
|
||||
Legend map[string]any `json:"legend,omitempty"`
|
||||
}
|
||||
|
||||
// SystemOne POSTs one state and a set of questions, mixing types freely, and
|
||||
// returns the answers keyed by question name. It is the single network path for
|
||||
// every Jev call (Classify and the probes), so auth, status and usage handling
|
||||
// stays in one place. Returns total tokens and the model that answered.
|
||||
func (j *JevClient) SystemOne(ctx context.Context, state any, questions map[string]any) (map[string]Answer, int, string, error) {
|
||||
body, _ := json.Marshal(map[string]any{"model": j.model, "state": state, "questions": questions})
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", j.url, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return JevCall{}, err
|
||||
return nil, 0, "", err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+j.key)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := j.client.Do(req)
|
||||
if err != nil {
|
||||
return JevCall{}, err
|
||||
return nil, 0, "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return JevCall{}, fmt.Errorf("Jev returned %d: %s", resp.StatusCode, clip(raw, 300))
|
||||
}
|
||||
type answer struct {
|
||||
Choice string `json:"choice"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
return nil, 0, "", fmt.Errorf("Jev returned %d: %s", resp.StatusCode, clip(raw, 300))
|
||||
}
|
||||
var out struct {
|
||||
Model string `json:"model"`
|
||||
Answers map[string]answer `json:"answers"`
|
||||
Answers map[string]Answer `json:"answers"`
|
||||
Usage struct {
|
||||
InputTokens int `json:"input_tokens"`
|
||||
OutputTokens int `json:"output_tokens"`
|
||||
} `json:"usage"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return JevCall{}, fmt.Errorf("Jev sent an unreadable reply: %w", err)
|
||||
}
|
||||
mode, task := out.Answers["mode"], out.Answers["task_type"]
|
||||
if mode.Choice == "" || task.Choice == "" {
|
||||
return JevCall{}, errors.New("Jev's reply was missing an answer")
|
||||
return nil, 0, "", fmt.Errorf("Jev sent an unreadable reply: %w", err)
|
||||
}
|
||||
model := out.Model
|
||||
if model == "" {
|
||||
model = j.model
|
||||
}
|
||||
return out.Answers, out.Usage.InputTokens + out.Usage.OutputTokens, model, nil
|
||||
}
|
||||
|
||||
func (j *JevClient) Classify(ctx context.Context, item string) (JevCall, error) {
|
||||
answers, tokens, model, err := j.SystemOne(ctx, item, jevQuestions)
|
||||
if err != nil {
|
||||
return JevCall{}, err
|
||||
}
|
||||
mode, task := answers["mode"], answers["task_type"]
|
||||
if mode.Choice == "" || task.Choice == "" {
|
||||
return JevCall{}, errors.New("Jev's reply was missing an answer")
|
||||
}
|
||||
return JevCall{
|
||||
Mode: mode.Choice, TaskType: task.Choice, ModeConf: mode.Confidence, TaskTypeConf: task.Confidence,
|
||||
Model: model, Tokens: out.Usage.InputTokens + out.Usage.OutputTokens,
|
||||
Mode: mode.Choice, TaskType: task.Choice,
|
||||
ModeConf: deref(mode.Confidence), TaskTypeConf: deref(task.Confidence),
|
||||
Model: model, Tokens: tokens,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func deref(f *float64) float64 {
|
||||
if f == nil {
|
||||
return 0
|
||||
}
|
||||
return *f
|
||||
}
|
||||
|
||||
func (c JevCall) confident(threshold float64) bool {
|
||||
return c.ModeConf >= threshold && c.TaskTypeConf >= threshold
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user