package health import ( "encoding/json" "os" "reflect" "strings" "testing" "time" "git.novox.be/novox/mesh-sdk/go/catalogue" ) // ADR 0564 clause 3 (issue 1952 S2): the bodies of condition-raised, condition-changed and condition-cleared read the // events the live controller said unchanged. testdata/condition-events.json holds five transitions of the controller's // condition history (mesh-controller_condition-history, whose entries are the events' bodies, events.go Event), captured // on 2026-10-29 with `mesh-controller.conditions history=true days=1`: a raising, a reopening with gaps, a resolver and a // severity change with tried and was, and a clearing with why. Never written by hand. func TestTheControllersConditionEventsReadUnchanged(t *testing.T) { raw, err := os.ReadFile("testdata/condition-events.json") if err != nil { t.Fatal(err) } var captured []json.RawMessage if err := json.Unmarshal(raw, &captured); err != nil { t.Fatal(err) } changes := map[string]bool{} for _, body := range captured { var e Event if err := json.Unmarshal(body, &e); err != nil { t.Fatalf("%s: %v", body, err) } changes[e.Change] = true again, err := json.Marshal(e) if err != nil { t.Fatal(err) } var want, got map[string]any _ = json.Unmarshal(body, &want) _ = json.Unmarshal(again, &got) if !reflect.DeepEqual(want, got) { t.Errorf("the controller's %s of %s reads back as\n%s\nnot\n%s", e.Change, e.Key, again, body) } if EventFor(e.Change) != e.Event || e.Key != Key(e.Subject.Scope, e.Subject.ID, e.Key[strings.LastIndex(e.Key, ".")+1:]) { t.Errorf("%s: said as %s for a %s, or its key is not its subject's", e.Key, e.Event, e.Change) } } for _, c := range []string{ChangeRaised, ChangeReopened, ChangeResolver, ChangeSeverity, ChangeCleared} { if !changes[c] { t.Errorf("no captured %s", c) } } // The keeper's own body (Limits 9, 10): epoch 0, said and not left out, silenced null, and its show and silence // naming the new seat. c := Condition{Key: "machine.g14.silent", Actions: []Action{SilenceAction("machine.g14.silent")}} b, _ := json.Marshal(Event{Condition: c, Event: EventRaised, Change: ChangeRaised, Show: ShowKeeper(c.Key)}) for _, says := range []string{`"epoch":0`, `"silenced":null`, `"show":"mesh-conditions.conditions key=machine.g14.silent"`, `"verb":"mesh-conditions.silence"`} { if !strings.Contains(string(b), says) { t.Errorf("the keeper's body %s does not say %s", b, says) } } } // ADR 0564 clauses 2, 7, 8, 9 (issue 1952 S3, S4): the layouts between the observers, the healers and the keeper, and the // take-over key the readers follow. func TestTheHealthLayoutsHoldTheKeepersRules(t *testing.T) { at := time.Date(2026, 10, 29, 12, 0, 0, 0, time.UTC) // ADR 0571 clause 3: a reading keeps its seven-day peak and is written only when it changes; a partial size is no peak. var prev *Reading for i, step := range []struct { day, size int64 partial, wrote bool peak int64 }{{20, 100, false, true, 100}, {20, 100, false, false, 100}, {21, 300, false, true, 300}, {22, 50, false, true, 300}, {23, 9000, true, true, 300}, {29, 10, false, true, 50}, {30, 10, false, true, 10}} { now := time.Date(2026, 10, int(step.day), 1, 0, 0, 0, time.UTC) size := step.size m := Measured{SizeBytes: &size, MeasuredAt: &now, Precision: "exact"} if step.partial { m.Precision = "partial: stopped" } r, wrote := NextReading(prev, "ace", catalogue.PlacedDataItem{Module: "pg", ID: "data", Path: "/srv/pg"}, m, now) raw, _ := json.Marshal(r) back, err := DecodeReading(raw) if wrote != step.wrote || r.Peak == nil || *r.Peak != step.peak || err != nil || back.ID != "data" { t.Fatalf("reading %d: wrote %v, peak %v, read back %v", i, wrote, r.Peak, err) } prev = &r } // The first reading sets FirstSeen and every one after keeps it, as it keeps a retirement; an older reading reads as before. if first := time.Date(2026, 10, 20, 1, 0, 0, 0, time.UTC); !prev.FirstSeen.Equal(first) { t.Fatalf("first seen %s, not %s", prev.FirstSeen, first) } retired := *prev retired.RetiredAt, retired.RetiredWhy = at, "its module left ace" if r, _ := NextReading(&retired, "ace", retired.PlacedDataItem, retired.Measured, at.Add(time.Hour)); !r.RetiredAt.Equal(at) || r.RetiredWhy != "its module left ace" || !r.FirstSeen.Equal(prev.FirstSeen) { t.Fatalf("a retired reading moved on as %+v", r) } if old, err := DecodeReading([]byte(`{"v":1,"node":"ace","module":"pg","item":"data"}`)); err != nil || !old.FirstSeen.IsZero() { t.Fatalf("a reading of before read as %+v: %v", old, err) } if raw, _ := json.Marshal(Reading{V: ReadingVersion}); strings.Contains(string(raw), "first_seen") || strings.Contains(string(raw), "retired") { t.Fatalf("zero fields were written: %s", raw) } if ReadingKey("ace", "pg", "data") != "ace.pg.data" || ReadingsBucket != "disk-load_readings" { t.Fatal("the readings' key or bucket moved") } // And restic's last good night per item, keyed as a reading. goodRow, _ := json.Marshal(GoodRow{V: GoodVersion, Node: "ace", Module: "pg", Item: "data", CoveredBy: "/srv/pg", LastGood: at}) if g, err := DecodeGoodRow(goodRow); err != nil || !g.LastGood.Equal(at) || GoodKey("ace", "pg", "data") != "ace.pg.data" || GoodBucket != "restic_good" || GoodRead != "restic.good" { t.Fatalf("a last good night read back as %+v: %v", g, err) } if _, err := DecodeGoodRow([]byte(`{"v":2}`)); err == nil { t.Fatal("a last good night of another version was read") } good := Finding{Key: "provider.keycloak.ace.mesh-mail.failing", Kind: "provider-failing", Token: "failing", Subject: Subject{Scope: ScopeProvider, ID: "keycloak.ace.mesh-mail", Machine: "ace"}, Severity: Warning, Summary: "keycloak on ace fails mesh-mail", Source: "S8", Observer: WatchmanModule, At: at} for _, c := range []struct { name string edit func(*Finding) fails string }{ {name: "a finding keyed by the controller's key rule"}, // The key rule is the controller's (condition.go Key): a character the bus refuses becomes `_`, so the finding // and the condition it raises share one key. {name: "an unsafe id", edit: func(f *Finding) { f.Subject.ID, f.Key = "keycloak.ace.mesh mail*", "provider.keycloak.ace.mesh_mail_.failing" }}, {name: "a key not its parts'", edit: func(f *Finding) { f.Key = "provider.keycloak.failing" }, fails: "not"}, {name: "an unknown scope", edit: func(f *Finding) { f.Subject.Scope = "thing" }, fails: "scope"}, {name: "no severity", edit: func(f *Finding) { f.Severity = "" }, fails: "urgent or a warning"}, {name: "no observer", edit: func(f *Finding) { f.Observer = "" }, fails: "observer"}, {name: "eleven observations", edit: func(f *Finding) { f.Evidence = make([]Evidence, KeptEvidence+1) }, fails: "at most"}, } { f := good if c.edit != nil { c.edit(&f) } b, err := EncodeFinding(f) if c.fails != "" { if err == nil || !strings.Contains(err.Error(), c.fails) { t.Errorf("%s: encoded (%v), want refused saying %q", c.name, err, c.fails) } continue } back, err := DecodeFinding(b) if err != nil || back.Key != f.Key || back.V != FindingVersion || back.Evidence == nil { t.Errorf("%s: read back as %+v, %v", c.name, back, err) } } if FindingsBucket(SelfCheckModule) != "mesh-self-check_findings" || Bucket(HealersModule, AttemptsState) != "mesh-healers_attempts" || Bucket(KeeperModule, TakeoverState) != "mesh-conditions_takeover" { t.Errorf("the buckets are named %s, %s", FindingsBucket(SelfCheckModule), Bucket(HealersModule, AttemptsState)) } // An attempt reaches the condition's tried as the controller wrote it: by the healer, its outcome and what it said. if tr := (Attempt{Healer: "H3", Act: "asserted the grants", Outcome: OutcomeEscalated, Said: "budget spent", Begun: at}).Tried(); tr != (Tried{At: at, What: "asserted the grants", Outcome: "escalated: budget spent", By: "healer H3"}) { t.Errorf("an attempt is tried as %+v", tr) } // The take-over key (clause 9): absent follows the controller; the keeper's names who chose; anything else refused. for _, c := range []struct { raw string found bool seat, events string fails bool }{ {found: false, seat: "mesh-controller", events: "mesh-controller.doctor-heartbeat"}, {raw: `{"v":1,"follow":"controller","by":"the operator"}`, found: true, seat: "mesh-controller", events: "mesh-controller.doctor-heartbeat"}, {raw: `{"v":1,"follow":"keeper","by":"the operator","ask":"ask-1"}`, found: true, seat: "mesh-conditions", events: "mesh-self-check.checked"}, {raw: `{"v":1,"follow":"keeper"}`, found: true, fails: true}, {raw: `{"v":1,"follow":"both","by":"x"}`, found: true, fails: true}, {raw: `{"v":2,"follow":"keeper","by":"x"}`, found: true, fails: true}, } { got, err := DecodeTakeover([]byte(c.raw), c.found) if c.fails != (err != nil) { t.Errorf("%q: read as %+v, %v", c.raw, got, err) continue } if c.fails { continue } seat, event := got.SelfCheckEvent() if got.Seat() != c.seat || seat+"."+event != c.events { t.Errorf("%q follows %s and %s.%s", c.raw, got.Seat(), seat, event) } if b, err := EncodeTakeover(got); err != nil || !strings.Contains(string(b), `"v":1`) { t.Errorf("%q encodes as %s, %v", c.raw, b, err) } } }