Put a broken module back at once, and excuse a wait only for a move that added an account group (hq issue 318 review)
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer delivery to the same trunk took over its walk
mesh/merge-gate pass: builds build-agent, mesh-controller, route-proxy → ace, g14, novox, shanks; no bus step; every machine composes with the change as it…
mesh/repo-check pass: its merge-check.sh passed
mesh/delivery superseded: a newer delivery to the same trunk took over its walk
This commit is contained in:
@@ -119,6 +119,9 @@ type gateFacts struct {
|
||||
healthErr error
|
||||
// heldOn is, per "<module>@<machine>", the provider its findings are held under (ADR 0240 rule 5).
|
||||
heldOn map[string]string
|
||||
// groupsAdded is, per module, whether the move judged puts an account in a group its previous build did
|
||||
// not (issue 318 review): the only move whose wait for a new login is excused.
|
||||
groupsAdded map[string]bool
|
||||
}
|
||||
|
||||
// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A
|
||||
@@ -430,6 +433,7 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf)
|
||||
// **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each
|
||||
// machine judged, apart from what is wrong with a module there, and never pinned on the module the
|
||||
// gate happens to be kept on.
|
||||
@@ -544,7 +548,8 @@ func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs
|
||||
fail(g.BrokenWhy)
|
||||
break
|
||||
}
|
||||
// What broke fails the send; what is beside it is judged to its own verdict first, within the bound.
|
||||
// What broke fails the send, and is put back at once by the caller (putBackBroken); what is beside
|
||||
// it is judged to its own verdict first, within the bound.
|
||||
g.Passes, g.LastPass, g.Failing = 0, nil, failing
|
||||
g.Last = fmt.Sprintf("%s; %s judged to its own verdict before the send's", g.BrokenWhy, strings.Join(judging, ", "))
|
||||
case worst == healthWaiting:
|
||||
@@ -607,6 +612,37 @@ func joinSaid(a, b string) string {
|
||||
return a + "; " + b
|
||||
}
|
||||
|
||||
// movesAddingGroups is, per module a gate judges, whether its move puts an account in a group the build it
|
||||
// moved from did not: read from the builds' manifests. A module whose move is not known adds none.
|
||||
func movesAddingGroups(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged,
|
||||
shelf map[string]catalogue.Manifest) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, j := range pairs {
|
||||
if _, done := out[j.module]; done {
|
||||
continue
|
||||
}
|
||||
from, to := g.From, g.To
|
||||
for _, c := range g.Carried {
|
||||
if c.Module == j.module {
|
||||
from, to = c.From, c.To
|
||||
break
|
||||
}
|
||||
}
|
||||
target, found, err := inv.ManifestAt(ctx, j.module, to)
|
||||
if err != nil || !found {
|
||||
target = shelf[j.module]
|
||||
}
|
||||
before, had, err := inv.ManifestAt(ctx, j.module, from)
|
||||
if err != nil || (from != "" && !had) {
|
||||
// What it moved from is not known: no wait is excused, rather than one the move did not bring.
|
||||
out[j.module] = false
|
||||
continue
|
||||
}
|
||||
out[j.module] = addsAccountGroups(before, had, target)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// decide sets a gate's verdict.
|
||||
func decide(g *inventory.PlanGate, verdict, why string, now time.Time) {
|
||||
g.Verdict, g.Why, g.JudgedAt = verdict, why, &now
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -277,3 +278,146 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said)
|
||||
}
|
||||
}
|
||||
|
||||
// **A broken module is put back at once** (issue 318 review): the laptop's witness put the node tools back
|
||||
// within a minute of a send that also moved `app`, and `app` reads not yet healthy because of it. The node
|
||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
withConditionsInMemory(t)
|
||||
was := doctorFrom
|
||||
doctorFrom = nil
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute)
|
||||
build := func(module, commit string, asked time.Time) {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
|
||||
Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + commit}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
|
||||
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, m := range []string{"app", broker_runtime} {
|
||||
build(m, "c1", old)
|
||||
if _, err := inv.Assign(ctx, "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop", map[string]string{"app": "c1", broker_runtime: "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
build("app", "c2", recent)
|
||||
build(broker_runtime, "c2", recent)
|
||||
|
||||
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
|
||||
t.Cleanup(func() {
|
||||
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
||||
})
|
||||
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
||||
modules, _ := open.inventory.Assigned(ctx, node)
|
||||
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f.moves(node, modules, sent, known, all), nil
|
||||
}
|
||||
releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true}, nil }
|
||||
var sends []string
|
||||
n := 0
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
current, err := open.inventory.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, node := range names {
|
||||
n++
|
||||
carried := map[string]string{"app": current["app"].Commit, broker_runtime: current[broker_runtime].Commit}
|
||||
sends = append(sends, node+":"+carried[broker_runtime])
|
||||
digest := "d-" + node + "-" + time.Now().Format("150405.000000") + string(rune('a'+n))
|
||||
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
||||
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
var seen []string // the node tools' registered build at each judging
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
now := time.Now()
|
||||
f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
||||
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
||||
reports, _ := open.inventory.LastReports(ctx)
|
||||
for _, r := range reports {
|
||||
f.reports[r.Node] = r
|
||||
}
|
||||
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||
seen = append(seen, current[broker_runtime].Commit)
|
||||
app := healthyContainer("app")
|
||||
if current[broker_runtime].Commit == "c2" {
|
||||
// The witness reverted the node tools; app cannot reach them yet.
|
||||
f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack,
|
||||
From: "c2", To: "c1", At: now, Why: "the node tools did not answer"}}
|
||||
app.State, app.Reason = link.StateUnhealthy, "down"
|
||||
}
|
||||
f.served["laptop"] = served{runtime: current[broker_runtime].Commit == "c1", tools: map[string]bool{}}
|
||||
f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}}
|
||||
return f, nil
|
||||
}
|
||||
withGateBounds(t, 100*time.Millisecond, 0, 10*time.Second)
|
||||
|
||||
release := func() inventory.Plan {
|
||||
t.Helper()
|
||||
plans, _ := inv.RecentPlans(ctx, 10)
|
||||
for _, p := range plans {
|
||||
if p.Release != nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
t.Fatal("no walk")
|
||||
return inventory.Plan{}
|
||||
}
|
||||
// Judged until the first judging has found the node tools broken, and one more.
|
||||
for i := 0; i < 20 && len(seen) < 2; i++ {
|
||||
advancePlans(ctx, open)
|
||||
}
|
||||
if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" {
|
||||
t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at the very next judging", seen)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-"+broker_runtime+"-c2"); !failed {
|
||||
t.Fatal("the node tools' build is not marked failed at once")
|
||||
}
|
||||
if p := release(); p.State != inventory.PlanRolling || p.Release.Gate == nil || p.Release.Gate.Verdict != "" {
|
||||
t.Fatalf("the walk is %s with gate %+v; want it still judging app", p.State, p.Release.Gate)
|
||||
}
|
||||
if !slices.Contains(sends, "laptop:c1") {
|
||||
t.Fatalf("sends %v; want the laptop sent the node tools' earlier build at once", sends)
|
||||
}
|
||||
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); {
|
||||
advancePlans(ctx, open)
|
||||
if release().State != inventory.PlanRolling {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
p := release()
|
||||
if p.State != inventory.PlanFailed {
|
||||
t.Fatalf("the walk is %s: %s; want failed, for the node tools", p.State, p.Note)
|
||||
}
|
||||
if v, found, _ := inv.GateOf(ctx, "build-app-c2"); !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") {
|
||||
t.Fatalf("app's verdict: %+v; want its own pass", v)
|
||||
}
|
||||
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
||||
t.Fatalf("app is registered at %s; want it kept at c2", current["app"].Commit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -492,17 +492,12 @@ func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (hea
|
||||
return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module)
|
||||
}
|
||||
wait, waits := personWait(module, machine, h.Resources)
|
||||
// **Only the build whose own send put the account into its wait is excused** (issue 318 review): an
|
||||
// account already waiting before this send was not brought to it by this build, and a later build of
|
||||
// the module sent while the login is still owed is judged as before. Two seconds of skew between the
|
||||
// machine's clock and the controller's, as the witness allows.
|
||||
if waits {
|
||||
for _, r := range h.Resources {
|
||||
if r.Module == module && r.Kind == link.KindAccount && r.State == link.StateUnhealthy &&
|
||||
strings.HasPrefix(r.Reason, link.ReasonRelogin) && r.Since.Before(since.Add(-2*time.Second)) {
|
||||
waits = false
|
||||
}
|
||||
}
|
||||
// **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from
|
||||
// what the controller sent, never from when the machine says the wait began — that time is the engine's
|
||||
// memory, reset by its restart and moved by a change of words. A build that adds no account group cannot
|
||||
// have brought a wait, so a later build sent while the login is still owed is judged as before.
|
||||
if waits && !f.groupsAdded[module] {
|
||||
waits = false
|
||||
}
|
||||
for _, r := range h.Resources {
|
||||
if r.Module != module {
|
||||
@@ -565,3 +560,38 @@ func healthLines(h inventory.NodeHealth, had bool, now time.Time) []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// accountGroups is every account group a manifest declares, as "<account>/<group>": a user resource's
|
||||
// groups (ADR 0252).
|
||||
func accountGroups(m catalogue.Manifest) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, r := range m.Resources {
|
||||
if t, _ := r["type"].(string); t != "user" {
|
||||
continue
|
||||
}
|
||||
name, _ := r["name"].(string)
|
||||
groups, _ := r["groups"].([]any)
|
||||
for _, g := range groups {
|
||||
if group, ok := g.(string); ok && group != "" {
|
||||
out[name+"/"+group] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// addsAccountGroups is whether a move from one manifest of a module to another puts an account in a group the
|
||||
// earlier one did not (issue 318 review): the only send that can bring a wait for a new login. A module new to
|
||||
// the machine (no earlier manifest) adds every group it declares.
|
||||
func addsAccountGroups(from catalogue.Manifest, hadFrom bool, to catalogue.Manifest) bool {
|
||||
before := map[string]bool{}
|
||||
if hadFrom {
|
||||
before = accountGroups(from)
|
||||
}
|
||||
for g := range accountGroups(to) {
|
||||
if !before[g] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -76,23 +77,19 @@ func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
account := relogin("lights", "operator")
|
||||
account.Since = since.Add(10 * time.Second) // the send put it in its wait
|
||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}}}
|
||||
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}},
|
||||
groupsAdded: map[string]bool{"lights": true}}
|
||||
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") {
|
||||
t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why)
|
||||
}
|
||||
// **Only the send that put the account in its wait is excused** (issue 318 review): an account waiting
|
||||
// since before the send is judged as before, so a later build of the module is not passed on an old wait.
|
||||
older := f.health["laptop"]
|
||||
older.Resources = []inventory.ResourceHealth{relogin("lights", "operator"), userUnit("lights", "operator")}
|
||||
older.Resources[0].Since = since.Add(-time.Hour)
|
||||
f.health["laptop"] = older
|
||||
// **Only a move that put the account in a new group is excused** (issue 318 review): a later build that
|
||||
// adds no group did not bring the wait, and is judged as before.
|
||||
f.groupsAdded["lights"] = false
|
||||
if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthNotYet {
|
||||
t.Fatalf("a wait from before the send reads %v %q; want not yet", h, why)
|
||||
t.Fatalf("a wait the move did not bring reads %v %q; want not yet", h, why)
|
||||
}
|
||||
f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now,
|
||||
Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}
|
||||
f.groupsAdded["lights"] = true
|
||||
h := f.health["laptop"]
|
||||
h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container",
|
||||
Target: "lights", State: link.StateUnhealthy, Reason: "down"})
|
||||
@@ -273,6 +270,12 @@ func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
||||
for _, e := range told.Said() {
|
||||
if e.Change == conditions.ChangeCleared {
|
||||
resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved)
|
||||
// The clearing line is held to the plain rule too (ADR 0253).
|
||||
w := conditions.Words{Headline: e.Condition.Headline, Explanation: e.Condition.Explanation,
|
||||
Resolved: e.Condition.Resolved, Needs: e.Condition.Needs}
|
||||
if why, ok := conditions.PlainWords(w, "laptop"); !ok {
|
||||
t.Errorf("%s: its clearing words are not plain: %s", e.Condition.Key, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -280,3 +283,30 @@ func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
||||
t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved)
|
||||
}
|
||||
}
|
||||
|
||||
// Which moves add an account group, read from the builds' manifests (issue 318 review).
|
||||
func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) {
|
||||
user := func(groups ...any) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
|
||||
"name": "operator", "groups": groups}}}
|
||||
}
|
||||
none := catalogue.Manifest{Module: "lights"}
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
from catalogue.Manifest
|
||||
had bool
|
||||
to catalogue.Manifest
|
||||
addsSome bool
|
||||
}{
|
||||
{"a group added", none, true, user("lights"), true},
|
||||
{"the same group kept", user("lights"), true, user("lights"), false},
|
||||
{"a second group added", user("lights"), true, user("lights", "video"), true},
|
||||
{"a group taken away", user("lights", "video"), true, user("lights"), false},
|
||||
{"new to the machine, with a group", none, false, user("lights"), true},
|
||||
{"new to the machine, no group", none, false, none, false},
|
||||
} {
|
||||
if got := addsAccountGroups(c.from, c.had, c.to); got != c.addsSome {
|
||||
t.Errorf("%s: adds %v; want %v", c.name, got, c.addsSome)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -334,6 +334,9 @@ func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *invent
|
||||
notes = append(notes, p.Note)
|
||||
}
|
||||
done := map[string]bool{except: true}
|
||||
for _, m := range g.Returned {
|
||||
done[m] = true // put back at once when it broke
|
||||
}
|
||||
// **What passed on its own keeps its pass** (novox/hq ADR 0254, issue 318): healthy for the passes the
|
||||
// gate asks while another module of the send failed, it is neither put back nor left unjudged — a build
|
||||
// left without a verdict is one more move every other walk would wait for.
|
||||
@@ -390,6 +393,65 @@ func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *invent
|
||||
return kept
|
||||
}
|
||||
|
||||
// putBackBroken puts back, at once, every module a gate found broken and has not put back yet (issue 318
|
||||
// review): a witness that reverted a core component, or a machine that refused or failed its send, is not
|
||||
// left registered at the build that broke for as long as the modules beside it take to be judged — a resend
|
||||
// meanwhile would declare it again. The send goes on judging the others to their own verdicts; its own
|
||||
// verdict is still failed. lead and leadState are the module a plan's gate is kept on, when it is a plan's.
|
||||
// Answers whether anything was put back.
|
||||
func putBackBroken(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, lead string,
|
||||
leadState *inventory.PlanModule) bool {
|
||||
var todo []string
|
||||
for _, m := range g.Broken {
|
||||
if !slices.Contains(g.Returned, m) {
|
||||
todo = append(todo, m)
|
||||
}
|
||||
}
|
||||
if len(todo) == 0 || g.Verdict != "" {
|
||||
return false
|
||||
}
|
||||
stateWas, noteWas := p.State, p.Note
|
||||
batched, back := batchingRollbacks(ctx)
|
||||
var said []string
|
||||
for _, m := range todo {
|
||||
g.Returned = append(g.Returned, m)
|
||||
machines := g.Machines
|
||||
var state *inventory.PlanModule
|
||||
if m == lead && leadState != nil {
|
||||
judged := *leadState
|
||||
judged.Gate = nil // its own record of the failure; the send's gate goes on judging
|
||||
state = &judged
|
||||
} else {
|
||||
i := slices.IndexFunc(g.Carried, func(c inventory.CarriedMove) bool { return c.Module == m })
|
||||
if i < 0 {
|
||||
continue
|
||||
}
|
||||
c := g.Carried[i]
|
||||
state = &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To}
|
||||
if sent, err := sentTheBuild(ctx, open, m, c.To); err == nil && len(sent) > 0 {
|
||||
machines = sent
|
||||
} else {
|
||||
machines = []string{c.Node}
|
||||
}
|
||||
}
|
||||
p.Note = ""
|
||||
gateFailed(batched, open, p, m, state, machines, g.BrokenWhy)
|
||||
if m == lead && leadState != nil {
|
||||
leadState.Why = "put back at once, its send still judged: " + g.BrokenWhy
|
||||
}
|
||||
said = append(said, m)
|
||||
}
|
||||
sendRollbacks(ctx, open, p, back)
|
||||
p.State = stateWas
|
||||
p.Note = noteWas
|
||||
if len(said) > 0 {
|
||||
p.Note = fmt.Sprintf("put back at once: %s (%s); the rest of the send judged to their own verdicts",
|
||||
strings.Join(said, ", "), g.BrokenWhy)
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// passedAloneWhy is the verdict of a module that passed on its own while its send failed, with its own wait.
|
||||
func passedAloneWhy(g *inventory.PlanGate, module string) string {
|
||||
why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[module], g.Why)
|
||||
@@ -617,6 +679,9 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
}
|
||||
switch verdict {
|
||||
case "":
|
||||
if putBackBroken(ctx, open, p, g, "", nil) {
|
||||
return true, nil
|
||||
}
|
||||
note := fmt.Sprintf("judging %s: %s", strings.Join(g.Machines, ", "), gateLine(g))
|
||||
changed := p.Note != note
|
||||
p.Note = note
|
||||
|
||||
@@ -729,6 +729,9 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
}
|
||||
switch verdict {
|
||||
case "":
|
||||
if putBackBroken(ctx, open, p, state.Gate, m, state) {
|
||||
return true, nil
|
||||
}
|
||||
pending = append(pending, fmt.Sprintf("%s judged on %s: %s", m,
|
||||
strings.Join(state.Gate.Machines, ", "), gateLine(state.Gate)))
|
||||
continue
|
||||
@@ -961,7 +964,13 @@ func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module
|
||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||
}()
|
||||
g := state.Gate
|
||||
if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) {
|
||||
if g != nil && slices.Contains(g.Returned, module) {
|
||||
// Put back at once when it broke: its rollback was made then, and is not made again.
|
||||
state.Why = "put back when it broke; its send failed: " + g.Why
|
||||
p.State = inventory.PlanFailed
|
||||
p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was put back when it broke",
|
||||
strings.Join(g.Machines, ", "), p.Tier, g.Why, module)
|
||||
} else if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) {
|
||||
gateFailed(batched, open, p, module, state, machines, why)
|
||||
} else if slices.Contains(g.Passing, module) && state.Build != "" {
|
||||
// **The module the gate is kept on keeps its own pass too** (issue 318 review): healthy for the passes
|
||||
|
||||
@@ -40,13 +40,19 @@ func TestReplay318(t *testing.T) {
|
||||
t.Cleanup(func() { doctorFrom = was })
|
||||
|
||||
build := func(module, repository, commit string, asked time.Time) {
|
||||
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
|
||||
m := catalogue.Manifest{Module: module, Version: "1"}
|
||||
if module == "openrazer" && commit == "c78b5fc9" {
|
||||
// The build that put the operator's account in the group `openrazer` (ADR 0252).
|
||||
m.Resources = []map[string]any{{"id": "operator", "type": "user", "name": "operator",
|
||||
"groups": []any{"openrazer"}}}
|
||||
}
|
||||
manifest, _ := json.Marshal(m)
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
|
||||
Repository: repository, Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
|
||||
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + "-" + commit}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{
|
||||
Repository: repository, Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -159,6 +159,8 @@ type PlanGate struct {
|
||||
// first reason: the send fails, and the modules beside them are judged to their own verdict first.
|
||||
Broken []string `json:"broken,omitempty"`
|
||||
BrokenWhy string `json:"broken_why,omitempty"`
|
||||
// Returned names the broken modules already put back, at once, while the rest of the send is judged.
|
||||
Returned []string `json:"returned,omitempty"`
|
||||
}
|
||||
|
||||
// CarriedMove is one module's build moving on a machine with a gated send.
|
||||
|
||||
Reference in New Issue
Block a user