OpenRouter crows; log which model answered each flight

- ChatBrain speaks OpenAI-style chat completions (OpenRouter now)
- CROWS_PROVIDER picks anthropic or openrouter; default model
  openrouter/free; CROWS_MODEL overrides
- Flight log records the model that actually answered
- Token budgets fall back to a length estimate if usage is missing
- start.command reads either key from .env

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01UQnSw4RRWp8uqpj3nsnnXF
This commit is contained in:
SmokeyandClaude Opus 5.5 committed 2026-10-27 16:23:59 -06:00
1 parent 69ecabcb8c
commit 73cf298fee
5 files changed
+245 -24

No files matched your search

+142 -5
View File
@@ -17,17 +17,69 @@ type Brain interface {
Ask(ctx context.Context, system, prompt string) (text string, tokens int, err error) Ask(ctx context.Context, system, prompt string) (text string, tokens int, err error)
} }
// NewBrain uses Claude when ANTHROPIC_API_KEY is set, otherwise an offline mock. // modelKey lets a brain report which model actually answered a call
// (routers like openrouter/free pick a different model each time).
type modelKey struct{}
func withModelSlot(ctx context.Context, slot *string) context.Context {
return context.WithValue(ctx, modelKey{}, slot)
}
func reportModel(ctx context.Context, model string) {
if slot, ok := ctx.Value(modelKey{}).(*string); ok && slot != nil {
*slot = model
}
}
// NewBrain picks the crows' model from the environment:
//
// CROWS_PROVIDER=openrouter OPENROUTER_API_KEY (default model openrouter/free)
// CROWS_PROVIDER=anthropic ANTHROPIC_API_KEY (default model claude-sonnet-5-5)
// CROWS_MODEL overrides the model for either.
//
// With no provider set, whichever key is present wins (Anthropic first).
// No key at all means the offline mock.
func NewBrain() Brain { func NewBrain() Brain {
key := os.Getenv("ANTHROPIC_API_KEY") provider := strings.ToLower(os.Getenv("CROWS_PROVIDER"))
if key == "" { anthropicKey, openrouterKey := os.Getenv("ANTHROPIC_API_KEY"), os.Getenv("OPENROUTER_API_KEY")
return MockBrain{} if provider == "" {
switch {
case anthropicKey != "":
provider = "anthropic"
case openrouterKey != "":
provider = "openrouter"
}
} }
model := os.Getenv("CROWS_MODEL") model := os.Getenv("CROWS_MODEL")
switch provider {
case "anthropic":
if anthropicKey == "" {
fmt.Fprintln(os.Stderr, "CROWS_PROVIDER=anthropic but ANTHROPIC_API_KEY is empty; using the offline mock")
return MockBrain{}
}
if model == "" { if model == "" {
model = "claude-sonnet-5-5" model = "claude-sonnet-5-5"
} }
return &AnthropicBrain{key: key, model: model, client: &http.Client{}} return &AnthropicBrain{key: anthropicKey, model: model, client: &http.Client{}}
case "openrouter":
if openrouterKey == "" {
fmt.Fprintln(os.Stderr, "CROWS_PROVIDER=openrouter but OPENROUTER_API_KEY is empty; using the offline mock")
return MockBrain{}
}
if model == "" {
model = "openrouter/free"
}
return &ChatBrain{
provider: "OpenRouter", url: "https://openrouter.ai/api/v1/chat/completions",
key: openrouterKey, model: model, client: &http.Client{},
headers: map[string]string{"X-OpenRouter-Title": "So Many Crows"},
}
case "":
return MockBrain{}
default:
fmt.Fprintf(os.Stderr, "unknown CROWS_PROVIDER %q; using the offline mock\n", provider)
return MockBrain{}
}
} }
type AnthropicBrain struct { type AnthropicBrain struct {
@@ -80,9 +132,93 @@ func (b *AnthropicBrain) Ask(ctx context.Context, system, prompt string) (string
sb.WriteString(c.Text) sb.WriteString(c.Text)
} }
} }
reportModel(ctx, b.model)
return sb.String(), out.Usage.InputTokens + out.Usage.OutputTokens, nil return sb.String(), out.Usage.InputTokens + out.Usage.OutputTokens, nil
} }
// ChatBrain speaks the OpenAI-style chat completions format (OpenRouter, and
// gateways like OpenCode Zen for open models).
type ChatBrain struct {
provider, url, key, model string
headers map[string]string
client *http.Client
}
func (b *ChatBrain) Name() string { return b.provider + " (" + b.model + ")" }
func (b *ChatBrain) Ask(ctx context.Context, system, prompt string) (string, int, error) {
body, _ := json.Marshal(map[string]any{
"model": b.model,
"max_tokens": 2048,
"messages": []map[string]string{
{"role": "system", "content": system},
{"role": "user", "content": prompt},
},
})
req, err := http.NewRequestWithContext(ctx, "POST", b.url, bytes.NewReader(body))
if err != nil {
return "", 0, err
}
req.Header.Set("Authorization", "Bearer "+b.key)
req.Header.Set("Content-Type", "application/json")
for k, v := range b.headers {
req.Header.Set(k, v)
}
resp, err := b.client.Do(req)
if err != nil {
return "", 0, err
}
defer resp.Body.Close()
raw, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if resp.StatusCode != http.StatusOK {
return "", 0, fmt.Errorf("%s returned %d: %s", b.provider, resp.StatusCode, clip(raw, 300))
}
var out struct {
Model string `json:"model"`
Choices []struct {
Message struct {
Content string `json:"content"`
} `json:"message"`
} `json:"choices"`
Usage *struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
} `json:"usage"`
Error *struct {
Message string `json:"message"`
} `json:"error"`
}
if err := json.Unmarshal(raw, &out); err != nil {
return "", 0, fmt.Errorf("%s sent an unreadable reply: %w", b.provider, err)
}
if out.Error != nil {
return "", 0, fmt.Errorf("%s error: %s", b.provider, out.Error.Message)
}
if len(out.Choices) == 0 || strings.TrimSpace(out.Choices[0].Message.Content) == "" {
return "", 0, fmt.Errorf("%s sent an empty reply", b.provider)
}
text := out.Choices[0].Message.Content
// Budgets must still bite if a provider leaves out usage: estimate ~4 characters per token.
tokens := (len(system) + len(prompt) + len(text)) / 4
if out.Usage != nil && out.Usage.PromptTokens+out.Usage.CompletionTokens > 0 {
tokens = out.Usage.PromptTokens + out.Usage.CompletionTokens
}
model := out.Model
if model == "" {
model = b.model
}
reportModel(ctx, model)
return text, tokens, nil
}
func clip(b []byte, n int) string {
s := strings.TrimSpace(string(b))
if len(s) > n {
return s[:n] + "…"
}
return s
}
// MockBrain lets the whole flow run offline with no API key. // MockBrain lets the whole flow run offline with no API key.
type MockBrain struct{} type MockBrain struct{}
@@ -92,6 +228,7 @@ func (MockBrain) Ask(ctx context.Context, system, prompt string) (string, int, e
if err := ctx.Err(); err != nil { if err := ctx.Err(); err != nil {
return "", 0, err return "", 0, err
} }
reportModel(ctx, "mock")
tokens := (len(system)+len(prompt))/4 + 40 tokens := (len(system)+len(prompt))/4 + 40
switch { switch {
case strings.Contains(system, markerSentry): case strings.Contains(system, markerSentry):
+81
View File
@@ -0,0 +1,81 @@
package main
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func fakeChat(t *testing.T, status int, reply string) (*ChatBrain, *http.Request) {
t.Helper()
var got http.Request
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
got = *r
var body map[string]any
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
t.Errorf("request body is not JSON: %v", err)
}
if body["model"] != "openrouter/free" {
t.Errorf("model = %v", body["model"])
}
w.WriteHeader(status)
w.Write([]byte(reply))
}))
t.Cleanup(srv.Close)
return &ChatBrain{provider: "OpenRouter", url: srv.URL, key: "k", model: "openrouter/free", client: srv.Client(),
headers: map[string]string{"X-OpenRouter-Title": "So Many Crows"}}, &got
}
func TestChatBrainReply(t *testing.T) {
b, req := fakeChat(t, 200, `{"model":"some/model:free","choices":[{"message":{"content":"caw"}}],"usage":{"prompt_tokens":10,"completion_tokens":5}}`)
var model string
text, tokens, err := b.Ask(withModelSlot(context.Background(), &model), "sys", "hi")
if err != nil || text != "caw" || tokens != 15 || model != "some/model:free" {
t.Fatalf("got %q %d %q %v", text, tokens, model, err)
}
if req.Header.Get("Authorization") != "Bearer k" || req.Header.Get("X-OpenRouter-Title") != "So Many Crows" {
t.Fatalf("headers: %v", req.Header)
}
}
func TestChatBrainEstimatesTokensWhenUsageMissing(t *testing.T) {
b, _ := fakeChat(t, 200, `{"choices":[{"message":{"content":"`+strings.Repeat("a", 400)+`"}}]}`)
_, tokens, err := b.Ask(context.Background(), "", "")
if err != nil || tokens != 100 {
t.Fatalf("tokens = %d, err %v; want the 4-chars-per-token estimate so budgets still bite", tokens, err)
}
}
func TestChatBrainErrors(t *testing.T) {
for name, c := range map[string]struct {
status int
body string
want string
}{
"rate limit": {429, `{"error":{"message":"Rate limit exceeded"}}`, "429"},
"error body": {200, `{"error":{"message":"no free models"}}`, "no free models"},
"empty": {200, `{"choices":[{"message":{"content":" "}}]}`, "empty reply"},
} {
b, _ := fakeChat(t, c.status, c.body)
if _, _, err := b.Ask(context.Background(), "s", "p"); err == nil || !strings.Contains(err.Error(), c.want) {
t.Errorf("%s: err = %v, want %q", name, err, c.want)
}
}
}
func TestNewBrainPicksProvider(t *testing.T) {
t.Setenv("ANTHROPIC_API_KEY", "")
t.Setenv("OPENROUTER_API_KEY", "or")
t.Setenv("CROWS_PROVIDER", "")
t.Setenv("CROWS_MODEL", "")
if got := NewBrain().Name(); got != "OpenRouter (openrouter/free)" {
t.Fatalf("got %q", got)
}
t.Setenv("CROWS_PROVIDER", "anthropic")
if got := NewBrain().Name(); got != "offline mock" {
t.Fatalf("anthropic with no key should fall back to mock, got %q", got)
}
}
+1
View File
@@ -17,6 +17,7 @@ type Event struct {
Crow string `json:"crow,omitempty"` Crow string `json:"crow,omitempty"`
Mode string `json:"mode,omitempty"` Mode string `json:"mode,omitempty"`
TaskType string `json:"task_type,omitempty"` TaskType string `json:"task_type,omitempty"`
Model string `json:"model,omitempty"` // which model actually answered (routers vary)
Tokens int `json:"tokens,omitempty"` Tokens int `json:"tokens,omitempty"`
Ms int64 `json:"ms,omitempty"` Ms int64 `json:"ms,omitempty"`
Verdict string `json:"verdict,omitempty"` Verdict string `json:"verdict,omitempty"`
+3 -2
View File
@@ -144,14 +144,15 @@ func (f *Flock) ask(ctx context.Context, w Watcher, crow, system, prompt string,
cctx, cancel := context.WithTimeout(ctx, f.g.CrowTimeout) cctx, cancel := context.WithTimeout(ctx, f.g.CrowTimeout)
defer cancel() defer cancel()
start := time.Now() start := time.Now()
text, tokens, err := f.brain.Ask(cctx, system, prompt) var model string
text, tokens, err := f.brain.Ask(withModelSlot(cctx, &model), system, prompt)
for _, b := range budgets { for _, b := range budgets {
b.Add(tokens) b.Add(tokens)
} }
if errors.Is(err, context.DeadlineExceeded) { if errors.Is(err, context.DeadlineExceeded) {
err = fmt.Errorf("%s timed out after %s and was dropped", crow, f.g.CrowTimeout) err = fmt.Errorf("%s timed out after %s and was dropped", crow, f.g.CrowTimeout)
} }
e := Event{Kind: "flight", Crow: crow, Tokens: tokens, Ms: time.Since(start).Milliseconds(), Output: text} e := Event{Kind: "flight", Crow: crow, Model: model, Tokens: tokens, Ms: time.Since(start).Milliseconds(), Output: text}
if err != nil { if err != nil {
e.Error = err.Error() e.Error = err.Error()
} }
+18 -17
View File
@@ -2,26 +2,27 @@
# So Many Crows: local server. # So Many Crows: local server.
# Double-click to start. Close this window (or Ctrl-C) to stop. # Double-click to start. Close this window (or Ctrl-C) to stop.
# #
# - Real crows (Claude) when .env in this folder holds ANTHROPIC_API_KEY=...; mock crows otherwise. # Which crows fly is set in .env in this folder (never committed):
# - `./start.command --mock` forces mock crows. # OPENROUTER_API_KEY=... OpenRouter (default model: openrouter/free)
# - Listens on this Mac only (127.0.0.1:8080). State lives in .crows/ (delete it to start fresh). # ANTHROPIC_API_KEY=... Claude direct from Anthropic (default: claude-sonnet-5-5)
# - Guardrails: 20,000 tokens per item, 100,000 per list; you approve everything until crows earn trust. # CROWS_PROVIDER=openrouter optional: pick one when both keys are present
# CROWS_MODEL=... optional: a specific model
# `./start.command --mock` forces offline mock crows.
#
# Listens on this Mac only (127.0.0.1:8080). State lives in .crows/ (delete it to start fresh).
# Guardrails: 20,000 tokens per item, 100,000 per list; you approve everything until crows earn trust.
cd "$(dirname "$0")" || exit 1 cd "$(dirname "$0")" || exit 1
xattr -d com.apple.quarantine ./crows 2>/dev/null xattr -d com.apple.quarantine ./crows 2>/dev/null
chmod +x ./crows chmod +x ./crows
unset ANTHROPIC_API_KEY # Read only these lines from .env; the file is never executed.
if [ "$1" != "--mock" ] && [ -f .env ]; then envval() { sed -n "s/^$1=//p" .env 2>/dev/null | head -n 1 | tr -d '[:space:]"'; }
# Read only the key line; the file is never executed. unset ANTHROPIC_API_KEY OPENROUTER_API_KEY CROWS_PROVIDER CROWS_MODEL
ANTHROPIC_API_KEY="$(sed -n 's/^ANTHROPIC_API_KEY=//p' .env | head -n 1 | tr -d '[:space:]')" if [ "$1" != "--mock" ]; then
CROWS_MODEL_FROM_ENV="$(sed -n 's/^CROWS_MODEL=//p' .env | head -n 1 | tr -d '[:space:]')" for v in ANTHROPIC_API_KEY OPENROUTER_API_KEY CROWS_PROVIDER CROWS_MODEL; do
[ -n "$CROWS_MODEL_FROM_ENV" ] && export CROWS_MODEL="$CROWS_MODEL_FROM_ENV" val="$(envval "$v")"
[ -n "$ANTHROPIC_API_KEY" ] && export ANTHROPIC_API_KEY [ -n "$val" ] && export "$v=$val"
fi done
if [ -n "$ANTHROPIC_API_KEY" ]; then
echo "Real crows: Claude (${CROWS_MODEL:-claude-sonnet-5-5}). Each list is capped at 100,000 tokens."
else
echo "Mock crows (no key in .env, or --mock)."
fi fi
if lsof -iTCP:8080 -sTCP:LISTEN >/dev/null 2>&1; then if lsof -iTCP:8080 -sTCP:LISTEN >/dev/null 2>&1; then
@@ -29,4 +30,4 @@ if lsof -iTCP:8080 -sTCP:LISTEN >/dev/null 2>&1; then
exit 1 exit 1
fi fi
(sleep 1; open "http://127.0.0.1:8080") & (sleep 1; open "http://127.0.0.1:8080") &
exec ./crows serve -addr 127.0.0.1:8080 exec ./crows serve -addr 127.0.0.1:8080 # prints which brain is flying