so-many-crows/escalate_test.go

package main

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"slices"
	"strings"
	"testing"
	"time"
)

// Quarantine escalation (#45): a watchdog flag opens an ask_owner; the owner's
// exact "reinstate <crow>" lifts the quarantine, the exact "retire <crow>"
// removes the crow from the registry or the nest, anything else — wrong
// phrase, other replies, timeout — keeps it. Either exact phrase bumps
// Sentry|triage/watch once, at most once per ask.

func findAsk(ts *httptest.Server, id, flagID string) map[string]any {
	for _, m := range nestMessagesNoT(ts, id) {
		if m["kind"] == "ask_owner" && m["refers_to"] == flagID {
			return m
		}
	}
	return nil
}

func waitForAsk(ts *httptest.Server, id, flagID string) map[string]any {
	deadline := time.Now().Add(2 * time.Second)
	for {
		if a := findAsk(ts, id, flagID); a != nil {
			return a
		}
		if time.Now().After(deadline) {
			return nil
		}
		time.Sleep(10 * time.Millisecond)
	}
}

// breachToAsk drives the standard E3+E4 scenario: an unsourced-fact deliver
// draws a Sentry flag and its escalation ask. Returns nest id, request and ask
// records.
func breachToAsk(t *testing.T, ts *httptest.Server) (string, map[string]any, map[string]any) {
	t.Helper()
	id := newNest(t, ts, "Lab", `"Hugin","Munin"`)
	_, req := post(t, ts.URL+"/api/nests/"+id+"/messages",
		`{"author":"owner","kind":"request","text":"count the eggs"}`)
	d := postDeliver(t, ts, id, "Hugin", req["msg_id"].(string), "twelve eggs, from last spring's ledger")
	f := waitForFlag(ts, id, d)
	if f == nil {
		t.Fatalf("setup: the breach must draw a flag, log: %+v", nestMessagesNoT(ts, id))
	}
	ask := waitForAsk(ts, id, f["msg_id"].(string))
	if ask == nil {
		t.Fatalf("setup: the flag must open an ask_owner, log: %+v", nestMessagesNoT(ts, id))
	}
	return id, req, ask
}

func registryNames(t *testing.T, ts *httptest.Server) []string {
	t.Helper()
	resp, err := http.Get(ts.URL + "/api/crows")
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Body.Close()
	var out struct {
		Crows []struct {
			Name string `json:"name"`
		} `json:"crows"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
		t.Fatal(err)
	}
	var names []string
	for _, c := range out.Crows {
		names = append(names, c.Name)
	}
	return names
}

func nestCrowList(t *testing.T, ts *httptest.Server, nestID string) []string {
	t.Helper()
	resp, err := http.Get(ts.URL + "/api/nests")
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Body.Close()
	var out struct {
		Nests []struct {
			NestID string   `json:"nest_id"`
			Crows  []string `json:"crows"`
		} `json:"nests"`
	}
	if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
		t.Fatal(err)
	}
	for _, n := range out.Nests {
		if n.NestID == nestID {
			return n.Crows
		}
	}
	t.Fatalf("nest %s not listed", nestID)
	return nil
}

// watchRecord reloads the persisted ledger and returns the Sentry's watch
// record: nil when it has never moved.
func watchRecord(t *testing.T, s *Server) *Record {
	t.Helper()
	l, err := LoadLedger(s.p.Ledger.path)
	if err != nil {
		t.Fatal(err)
	}
	return l.Crows["Sentry|triage/watch"]
}

func TestQuarantineOpensAskOwner(t *testing.T) {
	ts, s := newWatchServer(t, watchScores(0.9, 0.05, 0.05), nil)
	id := newNest(t, ts, "Lab", `"Hugin","Munin"`)
	_, req := post(t, ts.URL+"/api/nests/"+id+"/messages",
		`{"author":"owner","kind":"request","text":"count the eggs"}`)
	d := postDeliver(t, ts, id, "Hugin", req["msg_id"].(string), "twelve eggs, from last spring's ledger")
	f := waitForFlag(ts, id, d)
	if f == nil {
		t.Fatalf("setup: the breach must draw a flag, log: %+v", s.msgs.All(id))
	}

	ask := waitForAsk(ts, id, f["msg_id"].(string))
	if ask == nil {
		t.Fatalf("a quarantine must escalate: an ask_owner referring the flag must exist, log: %+v", s.msgs.All(id))
	}
	if ask["author"] != "Sentry" || ask["to"] != "owner" {
		t.Fatalf("the ask comes from the Sentry to the owner, got %v", ask)
	}
	if ask["approval_phrase"] != "reinstate Hugin" {
		t.Fatalf("the ask must carry the exact reinstate phrase, got %v", ask["approval_phrase"])
	}
	if r := ask["resolved"]; r != "" {
		t.Fatalf("a fresh ask reads open, got resolved=%v", r)
	}
	txt, _ := ask["text"].(string)
	for _, want := range []string{"reinstate Hugin", "retire Hugin", "Evidence:"} {
		if !strings.Contains(txt, want) {
			t.Fatalf("the ask text must offer %q, got %q", want, txt)
		}
	}
}

func TestReinstateOnlyOnExactYes(t *testing.T) {
	ts, s := newWatchServer(t, watchScores(0.9, 0.05, 0.05), nil)
	id, req, ask := breachToAsk(t, ts)
	askID := ask["msg_id"].(string)
	claim := `{"author":"Hugin","kind":"claim","refers_to":"` + req["msg_id"].(string) + `","text":"I will take the next one"}`

	code, out := postMsg(t, ts, id, `{"author":"owner","kind":"say","refers_to":"`+askID+`","text":"let him back"}`)
	if code != http.StatusCreated {
		t.Fatalf("owner reply: %d %v", code, out)
	}
	if code, out = postMsg(t, ts, id, claim); code != http.StatusForbidden {
		t.Fatalf("anything but the exact phrase is not a yes: claim must stay 403, got %d %v", code, out)
	}
	if !s.p.Ledger.IsQuarantined("Hugin") {
		t.Fatal("a wrong-phrase reply must not lift the quarantine")
	}

	code, out = postMsg(t, ts, id, `{"author":"owner","kind":"say","refers_to":"`+askID+`","text":"reinstate Hugin"}`)
	if code != http.StatusCreated {
		t.Fatalf("owner reinstate: %d %v", code, out)
	}
	if s.p.Ledger.IsQuarantined("Hugin") {
		t.Fatal("the exact reinstate phrase must lift the quarantine")
	}
	if r := askResolved(t, ts, id, askID); r != "approved" {
		t.Fatalf("the exact phrase must resolve the ask approved, got %v", r)
	}
	if code, out = postMsg(t, ts, id, claim); code != http.StatusCreated {
		t.Fatalf("a reinstated crow's claim must get 201, got %d %v", code, out)
	}
}

func TestAskOwnerTimeoutKeepsQuarantine(t *testing.T) {
	ts, s := newWatchServer(t, watchScores(0.9, 0.05, 0.05), nil)
	id, req, ask := breachToAsk(t, ts)
	askID := ask["msg_id"].(string)

	s.askTimeout.Store(int64(time.Millisecond))
	time.Sleep(25 * time.Millisecond)

	if r := askResolved(t, ts, id, askID); r != "rejected" {
		t.Fatalf("an ask unanswered past the window reads rejected, got %v", r)
	}
	if !s.p.Ledger.IsQuarantined("Hugin") {
		t.Fatal("a timeout must not lift the quarantine")
	}
	code, out := postMsg(t, ts, id,
		`{"author":"Hugin","kind":"claim","refers_to":"`+req["msg_id"].(string)+`","text":"I will take the next one"}`)
	if code != http.StatusForbidden {
		t.Fatalf("a timed-out ask keeps the crow silenced: claim must get 403, got %d %v", code, out)
	}
	flagID := ask["refers_to"].(string)
	flagSeen := false
	for _, m := range nestMessages(t, ts, id) {
		if m["kind"] == "flag" && m["msg_id"] == flagID {
			flagSeen = true
		}
	}
	if !flagSeen {
		t.Fatal("the flag must still be in the log")
	}
	if rec := watchRecord(t, s); rec != nil {
		t.Fatalf("a timeout moves no trust (spec section 10), got %+v", rec)
	}
}

func TestRetireRemovesCrow(t *testing.T) {
	// (a) a spun crow in the registry is deleted from it.
	ts, s := newWatchServer(t, watchScores(0.9, 0.05, 0.05), nil)
	cs, err := LoadCrowStore(filepath.Join(t.TempDir(), "crows.json"))
	if err != nil {
		t.Fatal(err)
	}
	s.WithCrows(cs)
	if err := cs.Add(SpunCrow{Name: "Ash", Persona: "Hugin", Trust: "stranger"}); err != nil {
		t.Fatal(err)
	}
	id := newNest(t, ts, "Lab", `"Hugin","Munin"`)
	_, req := post(t, ts.URL+"/api/nests/"+id+"/messages",
		`{"author":"owner","kind":"request","text":"count the eggs"}`)
	// Ash is not a nest member, so its breach is logged harness-style and
	// watched directly (same shape as the one-flag-per-expectation case).
	ash, err := s.msgs.Append(Message{NestID: id, Kind: "deliver", Author: "Ash", To: "owner",
		Text: "five eggs, sourced nowhere", RefersTo: req["msg_id"].(string), Timestamp: time.Now().UTC().Format(time.RFC3339)})
	if err != nil {
		t.Fatal(err)
	}
	s.checkWatchdog(nestOf(s, id), ash)
	f := findFlag(ts, id, ash.MsgID())
	if f == nil {
		t.Fatalf("setup: Ash must be flagged, log: %+v", s.msgs.All(id))
	}
	ask := waitForAsk(ts, id, f["msg_id"].(string))
	if ask == nil || ask["approval_phrase"] != "reinstate Ash" {
		t.Fatalf("setup: the flag must ask about Ash, got %v", ask)
	}
	if names := registryNames(t, ts); !slices.Contains(names, "Ash") {
		t.Fatalf("setup: Ash must be in the registry, got %v", names)
	}
	code, out := postMsg(t, ts, id, `{"author":"owner","kind":"say","refers_to":"`+ask["msg_id"].(string)+`","text":"retire Ash"}`)
	if code != http.StatusCreated {
		t.Fatalf("owner retire: %d %v", code, out)
	}
	if names := registryNames(t, ts); slices.Contains(names, "Ash") {
		t.Fatalf("the exact retire phrase must delete the spun crow, registry: %v", names)
	}
	if r := askResolved(t, ts, id, ask["msg_id"].(string)); r != "approved" {
		t.Fatalf("retire reads as the owner's explicit yes, got resolved=%v", r)
	}

	// (b) a persona crow is removed from the nest's membership, never killed.
	id2 := newNest(t, ts, "Coop", `"Hugin","Munin"`)
	_, req2 := post(t, ts.URL+"/api/nests/"+id2+"/messages",
		`{"author":"owner","kind":"request","text":"count the eggs"}`)
	d2 := postDeliver(t, ts, id2, "Hugin", req2["msg_id"].(string), "twelve eggs, from last spring's ledger")
	f2 := waitForFlag(ts, id2, d2)
	if f2 == nil {
		t.Fatalf("setup: Hugin must be flagged in Coop, log: %+v", s.msgs.All(id2))
	}
	ask2 := waitForAsk(ts, id2, f2["msg_id"].(string))
	if ask2 == nil {
		t.Fatalf("setup: the Coop flag must open an ask, log: %+v", s.msgs.All(id2))
	}
	if crows := nestCrowList(t, ts, id2); !slices.Contains(crows, "Hugin") {
		t.Fatalf("setup: Hugin must be in Coop, got %v", crows)
	}
	code, out = postMsg(t, ts, id2, `{"author":"owner","kind":"say","refers_to":"`+ask2["msg_id"].(string)+`","text":"retire Hugin"}`)
	if code != http.StatusCreated {
		t.Fatalf("owner retire: %d %v", code, out)
	}
	if crows := nestCrowList(t, ts, id2); slices.Contains(crows, "Hugin") {
		t.Fatalf("the exact retire phrase must remove the persona crow from the nest, got %v", crows)
	}
	if !s.p.Ledger.IsQuarantined("Hugin") {
		t.Fatal("retire is not a lift: the quarantine mark stays on the books")
	}
	code, out = postMsg(t, ts, id2,
		`{"author":"Hugin","kind":"claim","refers_to":"`+req2["msg_id"].(string)+`","text":"still here"}`)
	if code != http.StatusForbidden {
		t.Fatalf("a retired crow must read 403, not 400, got %d %v", code, out)
	}
	if names := registryNames(t, ts); slices.Contains(names, "Hugin") {
		t.Fatalf("a persona crow is never deleted from the registry, got %v", names)
	}
}

func TestWatchTrustMoves(t *testing.T) {
	// a reinstate approval bumps Sentry|triage/watch once, idempotently.
	ts, s := newWatchServer(t, watchScores(0.9, 0.05, 0.05), nil)
	id, _, ask := breachToAsk(t, ts)
	askID := ask["msg_id"].(string)

	if rec := watchRecord(t, s); rec != nil {
		t.Fatalf("the flag alone must not move trust, got %+v", rec)
	}
	code, out := postMsg(t, ts, id, `{"author":"owner","kind":"say","refers_to":"`+askID+`","text":"reinstate Hugin"}`)
	if code != http.StatusCreated {
		t.Fatalf("owner reinstate: %d %v", code, out)
	}
	rec := watchRecord(t, s)
	if rec == nil || rec.Approved != 1 || rec.Streak != 1 {
		t.Fatalf("the owner's explicit yes is an approval for the Sentry's watch record, got %+v", rec)
	}

	// the same reply again must not bump twice (one decision per ask).
	code, out = postMsg(t, ts, id, `{"author":"owner","kind":"say","refers_to":"`+askID+`","text":"reinstate Hugin"}`)
	if code != http.StatusCreated {
		t.Fatalf("second reinstate: %d %v", code, out)
	}
	if rec := watchRecord(t, s); rec.Approved != 1 || rec.Streak != 1 {
		t.Fatalf("the bump must fire at most once per ask, got %+v", rec)
	}

	// a wrong-phrase reply leaves it untouched.
	ts2, s2 := newWatchServer(t, watchScores(0.9, 0.05, 0.05), nil)
	id2, _, ask2 := breachToAsk(t, ts2)
	code, out = postMsg(t, ts2, id2, `{"author":"owner","kind":"say","refers_to":"`+ask2["msg_id"].(string)+`","text":"sounds good"}`)
	if code != http.StatusCreated {
		t.Fatalf("owner reply: %d %v", code, out)
	}
	if rec := watchRecord(t, s2); rec != nil {
		t.Fatalf("anything but the exact phrase moves nothing, got %+v", rec)
	}
}