jschoubben
d6db334840
mesh/repo-check head d6db334840a5 merged into trunk 98155e327b6d beside mesh-host#129@09c77d57 — pass: its merge-check.sh passed
mesh/merge-gate head d6db334840a5 merged into trunk 98155e327b6d — pass: group fix/1732-held-moves-asked composed: every machine composes with the change a…
mesh/delivery delivered
mesh/delivery-group group fix/1732-held-moves-asked delivered: every member is delivered
238 lines
11 KiB
Go
238 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// **A move held for a person is asked of the person** (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259).
|
|
|
|
// aHeldMailu is novox holding mailu's new build because it would recreate a container: the walk's own words.
|
|
func aHeldMailu() heldFacts {
|
|
return heldFacts{node: "novox", moves: []heldMove{{Module: "mailu", From: "b2f5ebb2aaaa", To: "3259765dbbbb",
|
|
Why: "waits: recreates 1 of mailu's 12 container(s) (with a new image: automx)"}}}
|
|
}
|
|
|
|
func heldConditionOn(f *signalFacts) conditions.Condition {
|
|
o := watchMovesHeld(f)[0]
|
|
return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
|
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
|
}
|
|
|
|
// 1732/AC1: a machine holding a move of a recorded module for a person raises one condition for the operator, with
|
|
// Release as its only answer: the machine's push, approved, naming the modules whose moves wait there.
|
|
func TestAHeldMoveIsRaisedWithReleaseAsItsOnlyAction(t *testing.T) {
|
|
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
|
|
f.held = []heldFacts{aHeldMailu()}
|
|
got := watchMovesHeld(f)
|
|
if len(got) != 1 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
o := got[0]
|
|
if o.Kind != "moves-held" || o.Scope != conditions.ScopeMachine || o.ID != "novox" || o.Severity != conditions.Warning ||
|
|
o.Resolver != conditions.ResolverOperator || o.Key() != "machine.novox.moves-held" {
|
|
t.Fatalf("kind, scope, subject, severity or resolver: %+v (%s)", o, o.Key())
|
|
}
|
|
if len(o.Actions) != 1 {
|
|
t.Fatalf("actions: %+v", o.Actions)
|
|
}
|
|
a := o.Actions[0]
|
|
if a.Label != "Release" || a.Verb != "mesh-controller.push" || a.Level != conditions.LevelApprove ||
|
|
a.Arguments["node"] != "novox" || a.Arguments["move"] != "mailu" || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
|
|
t.Fatalf("the action is not the push of novox: %+v", a)
|
|
}
|
|
if _, takes := a.Arguments["why"]; !takes {
|
|
t.Fatal("the push takes a why, which the asker fills in with the operator's answer")
|
|
}
|
|
for _, want := range []string{"mailu", "novox"} {
|
|
if !strings.Contains(o.Headline+o.Explanation, want) {
|
|
t.Errorf("the words do not name %q: %q / %q", want, o.Headline, o.Explanation)
|
|
}
|
|
}
|
|
if !strings.Contains(o.Said, "recreates 1 of mailu's 12") {
|
|
t.Errorf("the evidence does not carry the walk's own words: %q", o.Said)
|
|
}
|
|
if w := (conditions.Observation{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs, Resolved: o.Resolved}); w.Headline == "" || len(o.Headline) > conditions.HeadlineMax {
|
|
t.Errorf("headline: %q", o.Headline)
|
|
}
|
|
}
|
|
|
|
// 1732/AC2: a machine with nothing held raises nothing, so the condition ends the look after the move is sent.
|
|
func TestAMachineHoldingNothingRaisesNoHeldMoveCondition(t *testing.T) {
|
|
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
|
|
if got := watchMovesHeld(f); len(got) != 0 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
}
|
|
|
|
// 1732/AC3: the condition is asked on a channel at the approve level in words that say what Release does, and the
|
|
// operator's warrant calls the controller's own push with the machine and the modules, once.
|
|
func TestAHeldMovesReleaseIsAskedAndAWarrantPushesTheMachine(t *testing.T) {
|
|
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
|
|
f.held = []heldFacts{aHeldMailu()}
|
|
c := heldConditionOn(f)
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{c}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 1 || len(sent[0].Options) != 1 {
|
|
t.Fatalf("asked %+v", sent)
|
|
}
|
|
o := sent[0].Options[0]
|
|
if o.Label != "Release" || o.Level != asks.Approve || o.Does != "the held builds are sent to novox" {
|
|
t.Errorf("the option is %+v", o)
|
|
}
|
|
if err := sent[0].Check(r.now); err != nil {
|
|
t.Errorf("the ask is refused as made: %v", err)
|
|
}
|
|
w := r.warrantFor(t, c.Key, "Release")
|
|
answerWith(t, r, w)
|
|
answerWith(t, r, w)
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.push@ ") ||
|
|
!strings.Contains(r.called[0], `"node":"novox"`) || !strings.Contains(r.called[0], `"move":"mailu"`) ||
|
|
!strings.Contains(r.called[0], `"cause":"`+conditions.CauseOperatorAnswer+`"`) ||
|
|
!strings.Contains(r.called[0], "chose Release") {
|
|
t.Fatalf("called %v", r.called)
|
|
}
|
|
}
|
|
|
|
// 1732/AC4: the controller may call its own push and upgrade verbs on a warrant: the grant names them.
|
|
func TestTheControllerMayActOnAWarrantForPushAndUpgrade(t *testing.T) {
|
|
for _, verb := range []string{"push", "upgrade"} {
|
|
granted := false
|
|
for _, v := range broker.VerbsTheControllerActsOnAWarrant {
|
|
granted = granted || (v.Seat == broker.ControllerSeat && v.Verb == verb)
|
|
}
|
|
if !granted {
|
|
t.Errorf("the controller may not act on a warrant for %s", verb)
|
|
}
|
|
}
|
|
}
|
|
|
|
func aDoneMailuPlan() inventory.Plan {
|
|
return inventory.Plan{ID: "plan-1792585970446304628", Repository: "novox/mesh-catalog", State: inventory.PlanDone,
|
|
Tiers: [][]string{{"a"}, {"b"}, {"mailu"}},
|
|
Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", Commit: "3259765dbbbbcccc",
|
|
Recorded: map[string]string{"novox": "waits: recreates 1 of mailu's 12 container(s)"}}}}
|
|
}
|
|
|
|
// 1732/AC5: a done plan that carries a move a machine still holds for a person says so on its line; once the machine
|
|
// no longer holds it, or holds another build of the module, the line says nothing more.
|
|
func TestADonePlanSaysWhatItHoldsForAPerson(t *testing.T) {
|
|
p := aDoneMailuPlan()
|
|
line := planLine(p, time.Date(2026, 10, 21, 16, 0, 0, 0, time.UTC)) + heldSaid(p, []heldFacts{aHeldMailu()})
|
|
if !strings.Contains(line, "done, 3 tier(s)") || !strings.Contains(line, "1 held for a person: mailu on novox") {
|
|
t.Errorf("the line: %q", line)
|
|
}
|
|
if said := heldSaid(p, nil); said != "" {
|
|
t.Errorf("a plan whose moves were sent says %q", said)
|
|
}
|
|
other := aHeldMailu()
|
|
other.moves[0].To = "9999999ffff"
|
|
if said := heldSaid(p, []heldFacts{other}); said != "" {
|
|
t.Errorf("a plan older than the build now held says %q", said)
|
|
}
|
|
p.Modules["mailu"].Recorded = map[string]string{"novox": recordedRollsWhy}
|
|
if said := heldSaid(p, []heldFacts{aHeldMailu()}); said != "" {
|
|
t.Errorf("a move the plan sent says %q", said)
|
|
}
|
|
}
|
|
|
|
// 1732/AC6: the builds held after a failed release are asked of the operator: the held-release condition offers
|
|
// Release, which calls the controller's own `upgrade` with release-backlog and nothing else, once, on a warrant.
|
|
func TestTheBuildsHeldAfterAFailedReleaseAreAskedAndReleasedOnAWarrant(t *testing.T) {
|
|
saved := backlogNow
|
|
t.Cleanup(func() { backlogNow = saved })
|
|
backlogNow.held = "release-1792595552499982311 failed (failed on novox); what waits is released again by a person"
|
|
backlogNow.waiting = map[string][]inventory.CarriedMove{"novox": {{Module: "indexer"}, {Module: "nextcloud"}}}
|
|
got := backlogObservation()
|
|
if len(got) != 1 || len(got[0].Actions) != 1 {
|
|
t.Fatalf("raised %+v", got)
|
|
}
|
|
a := got[0].Actions[0]
|
|
if a.Label != "Release" || a.Verb != "mesh-controller.upgrade" || a.Level != conditions.LevelApprove ||
|
|
a.Arguments["release-backlog"] != "true" || a.Arguments["cause"] != conditions.CauseOperatorAnswer {
|
|
t.Fatalf("the action: %+v", a)
|
|
}
|
|
o := got[0]
|
|
c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline,
|
|
Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}
|
|
r := newAskerRig(t)
|
|
r.open = []conditions.Condition{c}
|
|
if err := r.a.reconcile(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sent := r.asksSent(t)
|
|
if len(sent) != 1 || len(sent[0].Options) != 1 || sent[0].Options[0].Does != "the builds held after the failed delivery are released" {
|
|
t.Fatalf("asked %+v", sent)
|
|
}
|
|
w := r.warrantFor(t, c.Key, "Release")
|
|
answerWith(t, r, w)
|
|
answerWith(t, r, w)
|
|
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.upgrade@ ") ||
|
|
!strings.Contains(r.called[0], `"release-backlog":"true"`) || !strings.Contains(r.called[0], "chose Release") {
|
|
t.Fatalf("called %v", r.called)
|
|
}
|
|
}
|
|
|
|
// 1732/AC7: a plan whose send waits for a gate while the builds are held after a failed release reads as waiting for
|
|
// a person to release them, not "building ... LATE"; with nothing held it still reads as the wait it is.
|
|
func TestAPlanWaitingForAGateBehindAFailedReleaseIsNotLate(t *testing.T) {
|
|
saved := backlogNow
|
|
t.Cleanup(func() { backlogNow = saved })
|
|
now := time.Date(2026, 10, 21, 18, 50, 0, 0, time.UTC)
|
|
p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-sdk", Commit: "c0ffee00", State: inventory.PlanBuilding,
|
|
Tiers: [][]string{{"indexer"}}, TierEntered: now.Add(-100 * time.Minute), Created: now.Add(-110 * time.Minute),
|
|
Modules: map[string]*inventory.PlanModule{"indexer": {State: "built"}},
|
|
Note: refusalNote(0, errors.New("a build waits there for its gate: novox"))}
|
|
backlogNow.held = ""
|
|
if line := planLine(p, now); !strings.Contains(line, "a build waits there for its gate: novox") {
|
|
t.Errorf("the wait is not said: %q", line)
|
|
}
|
|
backlogNow.held = "release-1 failed (failed on novox); what waits is released again by a person"
|
|
line := planLine(p, now)
|
|
if strings.Contains(line, "LATE") || strings.Contains(line, "building") ||
|
|
!strings.Contains(line, "until a person releases it") {
|
|
t.Errorf("a plan held behind a failed release reads %q", line)
|
|
}
|
|
}
|
|
|
|
// 1732/AC10: a machine that also holds a move a person chose to hold gets no Release, since a push would send that move
|
|
// too; the condition names both and says the push is the controller's terminal. A machine with only asked moves keeps it.
|
|
func TestAMachineAlsoHoldingAPersonsOwnHoldIsOfferedNoRelease(t *testing.T) {
|
|
f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC))
|
|
h := aHeldMailu()
|
|
h.moves = append(h.moves, heldMove{Module: "nftables", From: "n1aaaa", To: "n2bbbb",
|
|
Why: "waits: a person set it to record, and holds every move of it until they push", Chosen: true})
|
|
f.held = []heldFacts{h}
|
|
got := watchMovesHeld(f)
|
|
if len(got) != 1 || len(got[0].Actions) != 0 {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
o := got[0]
|
|
text := o.Headline + o.Explanation + o.Needs + o.Summary + o.Said
|
|
for _, want := range []string{"mailu", "nftables", "terminal", "push novox"} {
|
|
if !strings.Contains(text, want) {
|
|
t.Errorf("does not say %q: %q", want, text)
|
|
}
|
|
}
|
|
w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs}
|
|
if why, ok := conditions.PlainWords(w, "novox"); !ok {
|
|
t.Errorf("not plain: %s", why)
|
|
}
|
|
f.held = []heldFacts{aHeldMailu()}
|
|
if got := watchMovesHeld(f); len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["move"] != "mailu" {
|
|
t.Fatalf("a machine with only asked moves: %+v", got)
|
|
}
|
|
}
|