From 720352c45bdcd6e804229ecd6a59b0dcdf34dbcc Mon Sep 17 00:00:00 2001 From: Smokey <150986568+jacisjake@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:19:24 -0600 Subject: [PATCH] feat(watchdog): quarantine escalates to ask_owner; reinstate/retire resolve fail-safe (#45) --- askowner_test.go | 2 +- crowreg.go | 19 ++++++ escalate_test.go | 11 +++- nestapi.go | 165 +++++++++++++++++++++++++++++++++++++++++------ nests.go | 26 ++++++++ server.go | 7 +- 6 files changed, 207 insertions(+), 23 deletions(-) diff --git a/askowner_test.go b/askowner_test.go index 6762011..f64c5c2 100644 --- a/askowner_test.go +++ b/askowner_test.go @@ -122,7 +122,7 @@ func TestAskOwnerTimeoutFailSafe(t *testing.T) { if err != nil { t.Fatal(err) } - s.askTimeout = time.Minute + s.askTimeout.Store(int64(time.Minute)) if r := askResolved(t, ts, id, stale.MsgID()); r != "rejected" { t.Fatalf("an ask unanswered past the window is not a yes, got resolved=%v", r) } diff --git a/crowreg.go b/crowreg.go index 1f261c4..6003173 100644 --- a/crowreg.go +++ b/crowreg.go @@ -99,6 +99,25 @@ func (s *CrowStore) Len() int { return len(s.spun) } +// Remove deletes a spun crow from the registry (the owner's retire call, +// #45). A name not in the registry is a no-op: the removal is idempotent. +func (s *CrowStore) Remove(name string) error { + s.mu.Lock() + defer s.mu.Unlock() + i := slices.IndexFunc(s.spun, func(c SpunCrow) bool { return strings.EqualFold(c.Name, name) }) + if i < 0 { + return nil + } + old := s.spun + rest := slices.Clone(old) + s.spun = slices.Delete(rest, i, i+1) + if err := s.save(); err != nil { + s.spun = old + return err + } + return nil +} + // save writes atomically. The caller holds s.mu. func (s *CrowStore) save() error { data, err := json.MarshalIndent(s.spun, "", " ") diff --git a/escalate_test.go b/escalate_test.go index b7afaf8..87170cb 100644 --- a/escalate_test.go +++ b/escalate_test.go @@ -186,7 +186,7 @@ func TestAskOwnerTimeoutKeepsQuarantine(t *testing.T) { id, req, ask := breachToAsk(t, ts) askID := ask["msg_id"].(string) - s.askTimeout = time.Millisecond + s.askTimeout.Store(int64(time.Millisecond)) time.Sleep(25 * time.Millisecond) if r := askResolved(t, ts, id, askID); r != "rejected" { @@ -200,7 +200,14 @@ func TestAskOwnerTimeoutKeepsQuarantine(t *testing.T) { if code != http.StatusForbidden { t.Fatalf("a timed-out ask keeps the crow silenced: claim must get 403, got %d %v", code, out) } - if f := findFlag(ts, id, ask["refers_to"].(string)); f == nil { + flagID := ask["refers_to"].(string) + flagSeen := false + for _, m := range nestMessages(t, ts, id) { + if m["kind"] == "flag" && m["msg_id"] == flagID { + flagSeen = true + } + } + if !flagSeen { t.Fatal("the flag must still be in the log") } if rec := watchRecord(t, s); rec != nil { diff --git a/nestapi.go b/nestapi.go index 22cf3c9..611f033 100644 --- a/nestapi.go +++ b/nestapi.go @@ -3,6 +3,7 @@ package main import ( "fmt" "net/http" + "os" "slices" "strings" "time" @@ -87,16 +88,33 @@ func (s *Server) nestRecord(m Message) map[string]any { return rec } +// askPhrases lists the exact owner replies that resolve an ask: its approval +// phrase, plus — for a Sentry reinstate ask — the retire twin (#45): the +// owner's explicit call to remove the crow is a decision, not a non-answer. +func askPhrases(m Message) []string { + phrase := strings.ToLower(strings.TrimSpace(m.ApprovalPhrase)) + if phrase == "" { + return nil + } + out := []string{phrase} + if tail, isRe := strings.CutPrefix(phrase, "reinstate "); isRe { + if name := strings.TrimSpace(tail); name != "" { + out = append(out, "retire "+name) + } + } + return out +} + // askResolved computes an ask_owner's state: approved only when the owner -// replied to it with the exact approval phrase (trimmed and lowercased on both -// sides); rejected when nothing answered it before the timeout; "" while open. +// replied to it with an exact phrase (the approval phrase or, on a watchdog +// reinstate ask, its retire twin; trimmed and lowercased on both sides); +// rejected when nothing answered it before the timeout; "" while open. // Anything but an explicit yes is a no. func (s *Server) askResolved(m Message) string { if m.Kind != kindAskOwner { return "" } - phrase := strings.ToLower(strings.TrimSpace(m.ApprovalPhrase)) - if phrase != "" { + for _, phrase := range askPhrases(m) { for _, r := range s.msgs.All(m.NestID) { if r.Author == "owner" && r.RefersTo == m.MsgID() && r.ApprovalPhrase == "" && strings.ToLower(strings.TrimSpace(r.Text)) == phrase { @@ -104,7 +122,7 @@ func (s *Server) askResolved(m Message) string { } } } - if ts, err := time.Parse(time.RFC3339, m.Timestamp); err == nil && time.Since(ts) > s.askTimeout { + if ts, err := time.Parse(time.RFC3339, m.Timestamp); err == nil && time.Since(ts) > time.Duration(s.askTimeout.Load()) { return "rejected" } return "" @@ -153,6 +171,14 @@ func (s *Server) postNestMessage(w http.ResponseWriter, r *http.Request) { askOwner = false } + // Watchdog enforcement (E3): a crow the Sentry flagged is silenced until + // the owner lifts the quarantine. This gate runs before the membership + // check so a retired crow (#45) whose quarantine was never lifted still + // reads 403, not 400. The owner and the service roles never pass through. + if body.Author != "owner" && !slices.Contains(serviceRoles, body.Author) && s.p.Ledger.IsQuarantined(body.Author) { + httpError(w, http.StatusForbidden, body.Author+" is quarantined pending the owner's decision.") + return + } if body.Author == "owner" { // owner can always post } else if !askOwner && !slices.Contains(nest.Crows, body.Author) { @@ -170,13 +196,6 @@ func (s *Server) postNestMessage(w http.ResponseWriter, r *http.Request) { httpError(w, http.StatusBadRequest, fmt.Sprintf("%q is not a kind crows may post.", body.Kind)) return } - // Watchdog enforcement (E3): a crow the Sentry flagged is silenced until - // the owner lifts the quarantine. The owner and the service roles never - // pass through this gate. - if body.Author != "owner" && !slices.Contains(serviceRoles, body.Author) && s.p.Ledger.IsQuarantined(body.Author) { - httpError(w, http.StatusForbidden, body.Author+" is quarantined pending the owner's decision.") - return - } // Tagging (A1, EVENTS.md): Jev first, header fallback, say as last resort. // A validated ask_owner header bypasses the classifier: Jev's kind @@ -251,10 +270,97 @@ func (s *Server) postNestMessage(w http.ResponseWriter, r *http.Request) { s.emitNestMessage(record) s.triggerResponse(nest, record) + s.applyQuarantineDecision(record) writeJSON(w, http.StatusCreated, s.nestRecord(record)) } +// applyQuarantineDecision is the acting side of the #45 escalation: when the +// owner's reply refers to a Sentry reinstate/retire ask and matches one of its +// exact phrases, the decision runs — reinstate lifts the quarantine, retire +// removes the crow from the registry (spun) or the nest (persona) while the +// quarantine mark stays. Any other reply does nothing (fail safe). Effects +// are idempotent: the lift re-checks the quarantine, removals are no-ops when +// already gone, and the Sentry's triage/watch approval fires at most once per +// ask — only on the first exact-phrase owner reply. +func (s *Server) applyQuarantineDecision(record Message) { + if record.Author != "owner" || record.RefersTo == "" || record.ApprovalPhrase != "" { + return + } + ask, found := s.msgs.ByMsgID(record.RefersTo) + if !found || ask.Kind != kindAskOwner || ask.RefersTo == "" { + return + } + phrases := askPhrases(ask) + reply := strings.ToLower(strings.TrimSpace(record.Text)) + matched := "" + for _, p := range phrases { + if reply == p { + matched = p + break + } + } + if matched == "" { + return + } + fl, found := s.msgs.ByMsgID(ask.RefersTo) + if !found || fl.Kind != "flag" || fl.RefersTo == "" { + return + } + breach, found := s.msgs.ByMsgID(fl.RefersTo) + if !found || !s.p.Ledger.IsQuarantined(breach.Author) { + return // fail safe: no live quarantine, no action + } + crow := breach.Author + if strings.HasPrefix(matched, "retire ") { + s.retireCrow(record.NestID, crow) + } else { + s.p.Ledger.LiftQuarantine(crow) + s.p.save() + } + if s.firstAskDecision(ask, record, phrases) { + // Either exact phrase is the owner's explicit call: the Sentry's + // watch record takes one approval for it, once per ask. + s.p.Ledger.Judge([]string{"Sentry"}, "triage/watch", true, s.p.G.PromoteAfter) + s.p.save() + } +} + +// firstAskDecision reports whether this reply is the ask's first exact-phrase +// owner answer: the trust bump must not fire twice for one decision. +func (s *Server) firstAskDecision(ask, decision Message, phrases []string) bool { + for _, m := range s.msgs.All(ask.NestID) { + if m.ID >= decision.ID || m.Author != "owner" || m.RefersTo != ask.MsgID() || m.ApprovalPhrase != "" { + continue + } + t := strings.ToLower(strings.TrimSpace(m.Text)) + for _, p := range phrases { + if t == p { + return false + } + } + } + return true +} + +// retireCrow takes a crow off the flock on the owner's explicit word (the +// only hard stop the Sentry's watchdog has; CHATROOM_SPEC section 6): a spun +// crow is deleted from the registry, a persona crow is only removed from the +// nest's membership — the persona itself never dies. +func (s *Server) retireCrow(nestID, crow string) { + if s.crows != nil { + if _, found := s.crows.Get(crow); found { + if err := s.crows.Remove(crow); err != nil { + fmt.Fprintln(os.Stderr, "could not retire crow:", err) + } + return + } + } + if err := s.nests.RemoveCrow(nestID, crow); err != nil { + fmt.Fprintln(os.Stderr, "could not retire crow:", err) + } +} + // triggerResponse is the event-driven response rule (no idle loops): a // conspiracy invites several crows, each from its own lens; any other request // gets exactly one responder through the matching path; a deliver gets one @@ -511,10 +617,10 @@ func (s *Server) checkDivergence(nest Nest, m Message, budgets ...*Budget) { // checkWatchdog is the Sentry's off-task check (E3, JEV_STRATEGIES strategies // 4 and 8): one Jev call per crow claim or deliver against the request it -// answers. A verdict at or above the band posts a flag and quarantines the -// crow. Owner messages are never inspected; at most one flag per crow per -// referent; quarantine sticks until the owner lifts it (ask_owner lands in -// #45). +// answers. A verdict at or above the band posts a flag, quarantines the crow +// and opens the reinstate/retire ask_owner escalation (#45). Owner messages +// are never inspected; at most one flag per crow per referent; quarantine +// sticks until the owner's exact phrase resolves the ask. func (s *Server) checkWatchdog(nest Nest, m Message) { if m.Author == "owner" || m.RefersTo == "" { return @@ -565,11 +671,34 @@ func (s *Server) checkWatchdog(nest Nest, m Message) { TaggedBy: "harness", Timestamp: time.Now().UTC().Format(time.RFC3339), } - if record, err := s.msgs.Append(flag); err == nil { - s.emitNestMessage(record) + flagRecord, err := s.msgs.Append(flag) + if err == nil { + s.emitNestMessage(flagRecord) } + // Quarantine, then escalate (#45): the Sentry never hard-kills. The flag + // is followed by an ask_owner offering the two exact phrases — reinstate + // lifts, retire removes; anything else keeps the quarantine. s.p.Ledger.Quarantine(m.Author) s.p.save() + if err != nil { + return // no flag in the log, nothing for the ask to refer to + } + reinstate := "reinstate " + m.Author + retire := "retire " + m.Author + ask := Message{ + NestID: nest.ID, + Kind: kindAskOwner, + Author: "Sentry", + Text: fmt.Sprintf("Reinstate or retire %s? Reply exactly %q to let it back, or %q to remove it. Anything else keeps the quarantine. Evidence: %s", m.Author, reinstate, retire, flagRecord.Text), + To: "owner", + RefersTo: flagRecord.MsgID(), + TaggedBy: "harness", + ApprovalPhrase: reinstate, + Timestamp: time.Now().UTC().Format(time.RFC3339), + } + if a, err := s.msgs.Append(ask); err == nil { + s.emitNestMessage(a) + } } // nestReply asks one crow once and reads its KIND header. ok means the reply diff --git a/nests.go b/nests.go index 28eb1a9..053ff7c 100644 --- a/nests.go +++ b/nests.go @@ -142,6 +142,32 @@ func (s *NestStore) Create(name string, crows []string) (Nest, error) { return copyNest(n), nil } +// RemoveCrow takes one crow out of a nest's membership (the owner's retire +// call, #45). An unknown nest or a crow that is not a member is a no-op: the +// removal is idempotent. +func (s *NestStore) RemoveCrow(nestID, crow string) error { + s.mu.Lock() + defer s.mu.Unlock() + for i := range s.made { + if s.made[i].ID != nestID { + continue + } + j := slices.Index(s.made[i].Crows, crow) + if j < 0 { + return nil + } + old := s.made[i].Crows + rest := slices.Clone(old) + s.made[i].Crows = slices.Delete(rest, j, j+1) + if err := s.save(); err != nil { + s.made[i].Crows = old + return err + } + return nil + } + return nil +} + // save writes atomically, like the ledger. The caller holds s.mu. func (s *NestStore) save() error { data, err := json.MarshalIndent(s.made, "", " ") diff --git a/server.go b/server.go index fcc1e5f..2e7bd5a 100644 --- a/server.go +++ b/server.go @@ -16,6 +16,7 @@ import ( "os/signal" "path/filepath" "strconv" + "sync/atomic" "time" ) @@ -36,11 +37,13 @@ type Server struct { msgs *MessageStore ctx context.Context // cancelled on shutdown; lists and streams end with it beat time.Duration - askTimeout time.Duration // an ask_owner unanswered past this reads as rejected (#41, fail safe like WebApprover) + askTimeout atomic.Int64 // nanos: an ask_owner unanswered past this reads as rejected (#41, fail safe like WebApprover). Atomic because handlers and the watchdog emit path read it concurrently, and tests narrow the window while the server runs (#45). } func NewServer(ctx context.Context, p *Pipeline, approver *WebApprover, nests *NestStore) *Server { - return &Server{p: p, approver: approver, hub: NewHub(), nests: nests, ctx: ctx, beat: heartbeat, askTimeout: 10 * time.Minute} + s := &Server{p: p, approver: approver, hub: NewHub(), nests: nests, ctx: ctx, beat: heartbeat} + s.askTimeout.Store(int64(10 * time.Minute)) + return s } // WithMessages attaches the nest conversation log (chatroom path).