(#22) Jev mode classification for nest requests #29
No files matched your search
@@ -58,6 +58,9 @@ Every nest event carries `nest_id`, `ts` (RFC 3339) and one message record. The
|
||||
| `to` | string | The @-addressed member, or `""`. `deliver`/`claim` default to the requester they answer. |
|
||||
| `refers_to` | string | The `msg_id` this responds to (claim/deliver/nudge point at their request/expectation), or `""`. |
|
||||
| `tagged_by` | string | How `kind` was decided: `"jev"`, `"header"`, `"disagreement"` (Jev overrode the crow's header) or `"default"` (nothing fit, became `say`). |
|
||||
| `mode` | string | On a `request` only: how it should be taken up — `solo`, `conspiracy` (several crows, one task, distinct readings), `split`, or `""` (not yet classified). Tagged Jev-first with a `mode` header escape (`jev_test.go`, `Flock.ClassifyMode`), same rule shape as `kind`. |
|
||||
| `mode_tagged_by` | string | How `mode` was decided, same four sources as `tagged_by`. |
|
||||
| `invited` | int | Conspiracy only: how many crows the request expects to take it up (default from the `MurderSize` cap). |
|
||||
|
||||
### SSE event
|
||||
|
||||
|
||||
@@ -155,6 +155,64 @@ func deref(f *float64) float64 {
|
||||
return *f
|
||||
}
|
||||
|
||||
// modeQuestions classifies how a request should be taken up (Epic C). The
|
||||
// escape option "other" keeps Jev from forcing a wrong class; mode is discrete,
|
||||
// so choice, not score (JEV_STRATEGIES.md strategy 3).
|
||||
var modeQuestions = map[string]any{
|
||||
"mode": map[string]any{
|
||||
"type": "choice",
|
||||
"instructions": "How should this chatroom request be taken up?",
|
||||
"criteria": map[string]string{
|
||||
"solo": "Simple: one crow answers it directly.",
|
||||
"conspiracy": "A judgment call worth several distinct readings: several crows take the same request from their different strengths.",
|
||||
"split": "Large with distinct parts: separate subtasks, one owner each.",
|
||||
"other": "None of these fits well.",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// ClassifyMode returns the mode when Jev is set and its choice is at or above
|
||||
// the band. "other" reads as unsure (not ok) so callers fall back to a header.
|
||||
func (j *JevClient) ClassifyMode(ctx context.Context, text string) (string, map[string]float64, bool) {
|
||||
if j == nil {
|
||||
return "", nil, false
|
||||
}
|
||||
answers, _, _, err := j.SystemOne(ctx, text, modeQuestions)
|
||||
if err != nil {
|
||||
return "", nil, false
|
||||
}
|
||||
a := answers["mode"]
|
||||
if a.Choice == "" || a.Choice == "other" || a.Confidence == nil || *a.Confidence < j.threshold {
|
||||
return "", nil, false
|
||||
}
|
||||
return a.Choice, a.Probabilities, true
|
||||
}
|
||||
|
||||
// ClassifyMode classifies a request under the same guardrails as any call:
|
||||
// flock cap, timeout and flight log, with the probabilities recorded. Returns
|
||||
// the mode and ok; below-band or absent Jev means not ok.
|
||||
func (f *Flock) ClassifyMode(ctx context.Context, text string, budgets ...*Budget) (string, bool) {
|
||||
if f.jev == nil {
|
||||
return "", false
|
||||
}
|
||||
for _, b := range budgets {
|
||||
if err := b.Check(); err != nil {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
f.sem <- struct{}{}
|
||||
defer func() { <-f.sem }()
|
||||
cctx, cancel := context.WithTimeout(ctx, f.g.CrowTimeout)
|
||||
defer cancel()
|
||||
start := time.Now()
|
||||
mode, probs, ok := f.jev.ClassifyMode(cctx, text)
|
||||
e := Event{Kind: "mode", Item: text, Crow: "Sentry", Model: f.jev.model, Ms: time.Since(start).Milliseconds(), Prob: probs, Mode: mode}
|
||||
if f.log != nil {
|
||||
f.log.Write(e)
|
||||
}
|
||||
return mode, ok
|
||||
}
|
||||
|
||||
func (c JevCall) confident(threshold float64) bool {
|
||||
return c.ModeConf >= threshold && c.TaskTypeConf >= threshold
|
||||
}
|
||||
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func fakeModeJev(t *testing.T, choice string, conf float64) *JevClient {
|
||||
t.Helper()
|
||||
return fakeJevFor(t, map[string]struct {
|
||||
choice string
|
||||
conf float64
|
||||
}{"mode": {choice, conf}})
|
||||
}
|
||||
|
||||
// fakeJevFor answers only the named question keys; any other question in the
|
||||
// same request gets an empty answer (so callers fall back to headers).
|
||||
func fakeJevFor(t *testing.T, answers map[string]struct {
|
||||
choice string
|
||||
conf float64
|
||||
}) *JevClient {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Questions map[string]any `json:"questions"`
|
||||
}
|
||||
json.NewDecoder(r.Body).Decode(&body)
|
||||
out := map[string]any{}
|
||||
for q := range body.Questions {
|
||||
if a, ok := answers[q]; ok {
|
||||
out[q] = map[string]any{"type": "choice", "choice": a.choice, "confidence": a.conf,
|
||||
"probabilities": map[string]float64{a.choice: a.conf}}
|
||||
}
|
||||
}
|
||||
json.NewEncoder(w).Encode(map[string]any{"model": "jev-1.14.0", "answers": out,
|
||||
"usage": map[string]int{"input_tokens": 60, "output_tokens": 0}})
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
return &JevClient{url: srv.URL, key: "tk", model: "jev-latest", threshold: 0.7, client: srv.Client()}
|
||||
}
|
||||
|
||||
func TestModeQuestionsExist(t *testing.T) {
|
||||
m := modeQuestions["mode"].(map[string]any)
|
||||
if m["type"] != "choice" {
|
||||
t.Fatalf("mode must be a choice question, got %v", m["type"])
|
||||
}
|
||||
crit := m["criteria"].(map[string]string)
|
||||
for _, k := range []string{"solo", "conspiracy", "split", "other"} {
|
||||
if _, ok := crit[k]; !ok {
|
||||
t.Errorf("mode criteria missing %q escape/class: %v", k, crit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyMode(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
flog, err := OpenFlightLog(filepath.Join(dir, "flight.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer flog.Close()
|
||||
f := NewFlock(MockBrain{}, DefaultGuardrails(), flog)
|
||||
f.SetClassifier(fakeModeJev(t, "conspiracy", 0.9))
|
||||
got, ok := f.ClassifyMode(context.Background(), "Should we launch a CLI or a web app first?")
|
||||
if !ok || got != "conspiracy" {
|
||||
t.Fatalf("want conspiracy ok, got %q ok=%v", got, ok)
|
||||
}
|
||||
evs, err := ReadEvents(filepath.Join(dir, "flight.jsonl"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
modes := 0
|
||||
for _, e := range evs {
|
||||
if e.Kind == "mode" && e.Mode == "conspiracy" && e.Prob != nil {
|
||||
modes++
|
||||
}
|
||||
}
|
||||
if modes != 1 {
|
||||
t.Fatalf("want one logged mode verdict with probs, events: %+v", evs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifyModeUnsureNotOk(t *testing.T) {
|
||||
f := NewFlock(MockBrain{}, DefaultGuardrails(), nil)
|
||||
f.SetClassifier(fakeModeJev(t, "split", 0.4))
|
||||
if got, ok := f.ClassifyMode(context.Background(), "hi"); ok || got != "" {
|
||||
t.Fatalf("below-band confidence must not be ok, got %q ok=%v", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMessageModeFields(t *testing.T) {
|
||||
m := Message{NestID: "lab", Kind: "request", Mode: "conspiracy", Invited: 3}
|
||||
if m.Mode != "conspiracy" || m.Invited != 3 {
|
||||
t.Fatalf("Message must carry mode + invited, got %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostRequestJevMode(t *testing.T) {
|
||||
ts, s := newTestNestServer(t)
|
||||
s.p.Flock.SetClassifier(fakeModeJev(t, "conspiracy", 0.9))
|
||||
id := newNest(t, ts, "Lab", `"Hugin"`)
|
||||
code, req := post(t, ts.URL+"/api/nests/"+id+"/messages", `{"author":"owner","kind":"request","text":"should we launch a CLI or web app"}`)
|
||||
if code != http.StatusCreated || req["mode"] != "conspiracy" || req["mode_tagged_by"] != "jev" {
|
||||
t.Fatalf("want conspiracy/jev, got %d %v", code, req)
|
||||
}
|
||||
stored := s.msgs.All(id)[0]
|
||||
if stored.Mode != "conspiracy" || stored.ModeTagged != "jev" {
|
||||
t.Fatalf("record not persisted with mode: %+v", stored)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostRequestModeHeaderFallback(t *testing.T) {
|
||||
ts, _ := newTestNestServer(t) // no Jev set
|
||||
id := newNest(t, ts, "Lab", `"Hugin"`)
|
||||
code, req := post(t, ts.URL+"/api/nests/"+id+"/messages", `{"author":"owner","kind":"request","mode":"split","text":"do both parts"}`)
|
||||
if code != http.StatusCreated || req["mode"] != "split" || req["mode_tagged_by"] != "header" {
|
||||
t.Fatalf("want split/header, got %d %v", code, req)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSayHasNoMode(t *testing.T) {
|
||||
ts, s := newTestNestServer(t)
|
||||
s.p.Flock.SetClassifier(fakeModeJev(t, "conspiracy", 0.9))
|
||||
id := newNest(t, ts, "Lab", `"Hugin"`)
|
||||
code, say := post(t, ts.URL+"/api/nests/"+id+"/messages", `{"author":"owner","kind":"say","text":"nice weather"}`)
|
||||
if code != http.StatusCreated {
|
||||
t.Fatalf("say: %d", code)
|
||||
}
|
||||
if _, ok := say["mode"]; ok {
|
||||
t.Fatalf("a say must not carry a mode: %v", say)
|
||||
}
|
||||
}
|
||||
+61
-38
@@ -42,20 +42,33 @@ func (s *Server) getNestMessages(w http.ResponseWriter, r *http.Request) {
|
||||
all := s.msgs.All(nestID)
|
||||
out := make([]map[string]any, 0, len(all))
|
||||
for _, m := range all {
|
||||
out = append(out, map[string]any{
|
||||
"msg_id": m.MsgID(),
|
||||
"kind": m.Kind,
|
||||
"author": m.Author,
|
||||
"text": m.Text,
|
||||
"to": m.To,
|
||||
"refers_to": m.RefersTo,
|
||||
"tagged_by": m.TaggedBy,
|
||||
"ts": m.Timestamp,
|
||||
})
|
||||
out = append(out, nestRecord(m))
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// nestRecord is the EVENTS.md wire shape for one message.
|
||||
func nestRecord(m Message) map[string]any {
|
||||
rec := map[string]any{
|
||||
"msg_id": m.MsgID(),
|
||||
"kind": m.Kind,
|
||||
"author": m.Author,
|
||||
"text": m.Text,
|
||||
"to": m.To,
|
||||
"refers_to": m.RefersTo,
|
||||
"tagged_by": m.TaggedBy,
|
||||
"ts": m.Timestamp,
|
||||
}
|
||||
if m.Mode != "" {
|
||||
rec["mode"] = m.Mode
|
||||
rec["mode_tagged_by"] = m.ModeTagged
|
||||
}
|
||||
if m.Invited > 0 {
|
||||
rec["invited"] = m.Invited
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
func (s *Server) postNestMessage(w http.ResponseWriter, r *http.Request) {
|
||||
nestID := r.PathValue("nest_id")
|
||||
nest, found := s.nests.Get(nestID)
|
||||
@@ -68,6 +81,7 @@ func (s *Server) postNestMessage(w http.ResponseWriter, r *http.Request) {
|
||||
Author string `json:"author"`
|
||||
Text string `json:"text"`
|
||||
Kind string `json:"kind"`
|
||||
Mode string `json:"mode"`
|
||||
To string `json:"to"`
|
||||
RefersTo string `json:"refers_to"`
|
||||
}
|
||||
@@ -115,15 +129,20 @@ func (s *Server) postNestMessage(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// Mode (Epic C, same rule shape as kind): Jev first, header escape, "" when unsure.
|
||||
mode, modeBy := s.tagMode(kind, body.Mode, body.Text)
|
||||
|
||||
msg := Message{
|
||||
NestID: nestID,
|
||||
Kind: kind,
|
||||
Author: body.Author,
|
||||
Text: body.Text,
|
||||
To: body.To,
|
||||
RefersTo: body.RefersTo,
|
||||
TaggedBy: taggedBy,
|
||||
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
||||
NestID: nestID,
|
||||
Kind: kind,
|
||||
Author: body.Author,
|
||||
Text: body.Text,
|
||||
To: body.To,
|
||||
RefersTo: body.RefersTo,
|
||||
TaggedBy: taggedBy,
|
||||
Mode: mode,
|
||||
ModeTagged: modeBy,
|
||||
Timestamp: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
record, err := s.msgs.Append(msg)
|
||||
if err != nil {
|
||||
@@ -139,16 +158,27 @@ func (s *Server) postNestMessage(w http.ResponseWriter, r *http.Request) {
|
||||
go s.respondToRequest(nest, record)
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusCreated, map[string]any{
|
||||
"msg_id": record.MsgID(),
|
||||
"kind": record.Kind,
|
||||
"author": record.Author,
|
||||
"text": record.Text,
|
||||
"to": record.To,
|
||||
"refers_to": record.RefersTo,
|
||||
"tagged_by": record.TaggedBy,
|
||||
"ts": record.Timestamp,
|
||||
})
|
||||
writeJSON(w, http.StatusCreated, nestRecord(record))
|
||||
}
|
||||
|
||||
var modePostable = map[string]bool{"solo": true, "conspiracy": true, "split": true}
|
||||
|
||||
// tagMode classifies a request's mode (Jev first, header escape). Only requests
|
||||
// carry a mode; other kinds get "".
|
||||
func (s *Server) tagMode(kind, header, text string) (string, string) {
|
||||
if kind != "request" {
|
||||
return "", ""
|
||||
}
|
||||
if m, ok := s.p.Flock.ClassifyMode(s.ctx, text); ok {
|
||||
if modePostable[header] && header != m {
|
||||
return m, "disagreement"
|
||||
}
|
||||
return m, "jev"
|
||||
}
|
||||
if modePostable[header] {
|
||||
return header, "header"
|
||||
}
|
||||
return "", "default"
|
||||
}
|
||||
|
||||
// tagKind runs the A1 rule: Jev first; Jev unsure -> the poster's header;
|
||||
@@ -169,16 +199,9 @@ func (s *Server) tagKind(header, text string) (string, string) {
|
||||
}
|
||||
|
||||
func (s *Server) emitNestMessage(m Message) {
|
||||
s.hub.PublishNest(m.NestID, "nest.message", map[string]any{
|
||||
"nest_id": m.NestID,
|
||||
"msg_id": m.MsgID(),
|
||||
"kind": m.Kind,
|
||||
"author": m.Author,
|
||||
"text": m.Text,
|
||||
"to": m.To,
|
||||
"refers_to": m.RefersTo,
|
||||
"tagged_by": m.TaggedBy,
|
||||
})
|
||||
rec := nestRecord(m)
|
||||
rec["nest_id"] = m.NestID
|
||||
s.hub.PublishNest(m.NestID, "nest.message", rec)
|
||||
}
|
||||
|
||||
// respondToRequest is the must-respond rule: the addressed crow answers with
|
||||
|
||||
+12
-9
@@ -13,15 +13,18 @@ import (
|
||||
// contract. IDs are assigned by the store; MsgID is "<nest_id>-<n>" and is
|
||||
// what the contract exposes.
|
||||
type Message struct {
|
||||
NestID string `json:"nest_id"`
|
||||
ID int `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Author string `json:"author"`
|
||||
Text string `json:"text"`
|
||||
To string `json:"to,omitempty"`
|
||||
RefersTo string `json:"refers_to,omitempty"`
|
||||
TaggedBy string `json:"tagged_by,omitempty"`
|
||||
Timestamp string `json:"ts"`
|
||||
NestID string `json:"nest_id"`
|
||||
ID int `json:"id"`
|
||||
Kind string `json:"kind"`
|
||||
Author string `json:"author"`
|
||||
Text string `json:"text"`
|
||||
To string `json:"to,omitempty"`
|
||||
RefersTo string `json:"refers_to,omitempty"`
|
||||
TaggedBy string `json:"tagged_by,omitempty"`
|
||||
Mode string `json:"mode,omitempty"` // solo | conspiracy | split (requests only, Epic C)
|
||||
ModeTagged string `json:"mode_tagged_by,omitempty"`
|
||||
Invited int `json:"invited,omitempty"`
|
||||
Timestamp string `json:"ts"`
|
||||
}
|
||||
|
||||
func (m Message) MsgID() string { return fmt.Sprintf("%s-%d", m.NestID, m.ID) }
|
||||
|
||||
Reference in new issue
Block a user