Jev classifier in tandem with the LLM Sentry
- Jev (TypeSafe, direct) triages mode and task type with confidence - Its call stands at >= 70% on both and no split; otherwise the LLM Sentry decides (unsure, split needs subtasks, or Jev down) - Jev flights obey the same budget, flock cap, timeout and log - Tests for every path; flight log is nil-safe - start.command reads TYPESAFE_API_KEY and threshold from .env Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01UQnSw4RRWp8uqpj3nsnnXF
This commit is contained in:
8 files changed
+317
-3
No files matched your search
@@ -79,6 +79,7 @@ type Watcher interface {
|
||||
|
||||
type Flock struct {
|
||||
brain Brain
|
||||
jev *JevClient // optional classifier for triage; nil means the LLM Sentry triages alone
|
||||
g Guardrails
|
||||
log *FlightLog
|
||||
sem chan struct{}
|
||||
@@ -163,7 +164,38 @@ func (f *Flock) ask(ctx context.Context, w Watcher, crow, system, prompt string,
|
||||
return text, err
|
||||
}
|
||||
|
||||
// SetClassifier pairs the Sentry with Jev. Pass nil to triage with the LLM alone.
|
||||
func (f *Flock) SetClassifier(j *JevClient) { f.jev = j }
|
||||
|
||||
// Triage: Jev classifies first when it's configured. Its call stands when it is
|
||||
// confident and no subtasks are needed; otherwise the LLM Sentry decides, and
|
||||
// the reason records what Jev thought. Jev failing never blocks a list.
|
||||
func (f *Flock) Triage(ctx context.Context, w Watcher, item string, budgets ...*Budget) (Triage, error) {
|
||||
if f.jev == nil {
|
||||
return f.llmTriage(ctx, w, item, budgets...)
|
||||
}
|
||||
call, err := f.classify(ctx, w, item, budgets...)
|
||||
if err != nil {
|
||||
t, lerr := f.llmTriage(ctx, w, item, budgets...)
|
||||
t.Reason = "Jev was unavailable; the LLM Sentry decided. " + t.Reason
|
||||
return t, lerr
|
||||
}
|
||||
if call.confident(f.jev.threshold) && call.Mode != "split" {
|
||||
return f.normalize(Triage{Mode: call.Mode, TaskType: call.TaskType, Reason: call.summary()}), nil
|
||||
}
|
||||
t, lerr := f.llmTriage(ctx, w, item, budgets...)
|
||||
switch {
|
||||
case !call.confident(f.jev.threshold):
|
||||
t.Reason = fmt.Sprintf("%s; unsure, so the LLM Sentry decided. %s", call.summary(), t.Reason)
|
||||
case t.Mode == "split":
|
||||
t.Reason = fmt.Sprintf("%s; the LLM Sentry wrote the subtasks. %s", call.summary(), t.Reason)
|
||||
default:
|
||||
t.Reason = fmt.Sprintf("%s; the LLM Sentry disagreed and chose %s. %s", call.summary(), t.Mode, t.Reason)
|
||||
}
|
||||
return t, lerr
|
||||
}
|
||||
|
||||
func (f *Flock) llmTriage(ctx context.Context, w Watcher, item string, budgets ...*Budget) (Triage, error) {
|
||||
text, err := f.ask(ctx, w, "Sentry", sentrySystem, "ITEM: "+item, budgets...)
|
||||
if err != nil {
|
||||
return Triage{}, err
|
||||
@@ -172,6 +204,11 @@ func (f *Flock) Triage(ctx context.Context, w Watcher, item string, budgets ...*
|
||||
if i, j := strings.Index(text, "{"), strings.LastIndex(text, "}"); i < 0 || j < i || json.Unmarshal([]byte(text[i:j+1]), &t) != nil {
|
||||
t = Triage{Mode: "solo", TaskType: "other", Reason: "Sentry's call was unreadable; defaulting to solo"}
|
||||
}
|
||||
return f.normalize(t), nil
|
||||
}
|
||||
|
||||
// normalize keeps any triage, from Jev or the LLM, inside the guardrails.
|
||||
func (f *Flock) normalize(t Triage) Triage {
|
||||
t.Mode = strings.ToLower(strings.TrimSpace(t.Mode))
|
||||
t.TaskType = strings.ToLower(strings.TrimSpace(t.TaskType))
|
||||
switch t.TaskType {
|
||||
@@ -192,7 +229,7 @@ func (f *Flock) Triage(ctx context.Context, w Watcher, item string, budgets ...*
|
||||
default:
|
||||
t.Mode, t.Subtasks = "solo", nil
|
||||
}
|
||||
return t, nil
|
||||
return t
|
||||
}
|
||||
|
||||
func (f *Flock) Execute(ctx context.Context, w Watcher, item string, t Triage, nest Nest, budgets ...*Budget) (Outcome, error) {
|
||||
|
||||
Reference in new issue
Block a user