diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 7bf565be..be469d09 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -332,6 +332,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, Commit: m.Head, Trunk: m.Trunk}) } + // A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686). + if request.Check.Try { + if line, err := json.Marshal(triedOf(v)); err == nil { + say(link.StepTried, string(line)) + } + } } } else if err == nil { // The package-registry credential is a build input, so it is resolved before the clone: a @@ -457,7 +463,8 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref, Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{}, Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base, - Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)} + Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held), + Tests: c.Tests, StoreOnly: c.StoreOnly} for dir, b := range c.Beside { spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref} } @@ -472,6 +479,23 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { return spec } +// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict, +// or the check's where that did not run — a head that does not merge — and the lines naming what failed. +func triedOf(v builder.CheckVerdict) link.Tried { + t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(), + Head: v.Head, Trunk: v.Trunk} + if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") { + t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary + } + if v.Repo != nil && v.Repo.Failed != "" { + t.Failed = strings.Split(v.Repo.Failed, "\n") + } + if t.Verdict == "" { + t.Verdict = "error" + } + return t +} + // checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1). func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt { var out []link.CheckBuilt diff --git a/cmd/mesh-builder/tried_test.go b/cmd/mesh-builder/tried_test.go new file mode 100644 index 00000000..06aaf706 --- /dev/null +++ b/cmd/mesh-builder/tried_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "testing" + "time" + + "github.com/novox/mesh-controller/internal/builder" +) + +// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own +// check's verdict, how long it took, and each line naming what failed. +func TestATrysOutcomeNamesWhatFailed(t *testing.T) { + tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second, + Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"}, + Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", + Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}}) + if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" || + tried.Summary != "its merge-check.sh failed: TestX" { + t.Fatalf("a failed try: %+v", tried) + } + // A head that does not merge fails both layers: the check's own word. + conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"}, + Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}}) + if conflict.Verdict != "fail" || conflict.Summary != "does not merge" { + t.Fatalf("a try that does not merge: %+v", conflict) + } + // No verdict is an error, never a pass. + if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" { + t.Fatalf("a try with no verdict: %+v", none) + } +} diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go index 304186a2..8a1e1a1c 100644 --- a/cmd/mesh-controller/gate.go +++ b/cmd/mesh-controller/gate.go @@ -1443,6 +1443,12 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error } // A release held after a failed one, while builds still wait for a gate (ADR 0236). out = append(out, backlogObservation()...) + // And a rest that went silent (novox/hq ADR 0353 §4). + silent, err := silentObservation(ctx, d.open.inventory) + if err != nil { + return nil, err + } + out = append(out, silent...) return sortedFound(dedupeObservations(out)), nil } @@ -1532,6 +1538,12 @@ func gateLine(g *inventory.PlanGate) string { case inventory.GateFailed: what = "FAILED" } + switch { + case g.Verdict == "" && g.Pending: + what = "kept, not yet sent" // novox/hq ADR 0353 §8 + case g.Verdict == "" && g.Silent: + what = "silent on its send, left" // ADR 0353 §4 + } if g.Verdict == "" && g.Waiting() { what = "waits for a busy container on " + g.Busy.Machine } diff --git a/cmd/mesh-controller/held_back.go b/cmd/mesh-controller/held_back.go index 0d90cc19..a0b4c6ba 100644 --- a/cmd/mesh-controller/held_back.go +++ b/cmd/mesh-controller/held_back.go @@ -151,7 +151,7 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string for _, mv := range f.moves(node, modules, same, known, false) { if planStillToSend(plans, mv.Module, node) == "" { why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+ - "plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To))) + "plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To))) } } if len(why) > 0 { diff --git a/cmd/mesh-controller/issue648_test.go b/cmd/mesh-controller/issue648_test.go new file mode 100644 index 00000000..45261cc0 --- /dev/null +++ b/cmd/mesh-controller/issue648_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A tier whose module is no move — made from the source every machine running it runs, so nothing is +// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the +// send that last carried that build before the walk got there, and nothing will make them report again: +// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever. +func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + + // app is the build machine, so a module of the next tier is built by it: the tier gates the next on app + // running on its machines. + for _, m := range []catalogue.Manifest{ + {Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}, + {Module: "web", Version: "w1"}, + } { + if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil { + t.Fatal(err) + } + } + // What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is. + for _, b := range []inventory.Build{ + {ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app", + Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)}, + {ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()}, + } { + b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + // Each machine answered that send, applied, before the walk's tier was built. + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n, + Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1, + At: time.Now().Add(-10 * time.Second)}); err != nil { + t.Fatal(err) + } + } + built := time.Now().UTC() + p := g.plan(t) + p.Tiers = [][]string{{"app"}, {"web"}} + p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + + advancePlans(ctx, g.open) + + if len(g.sent) != 0 { + t.Fatalf("a no-move build was sent: %v", g.sent) + } + p = g.plan(t) + if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") { + t.Fatalf("the build was not marked no move: %+v", s) + } + if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") { + t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note) + } + + // A machine that has not answered its last send is still waited on: no move is not "never sent". + p.Tier, p.State, p.Note = 0, inventory.PlanRolling, "" + if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + p = g.plan(t) + if p.Tier != 0 || !strings.Contains(p.Note, "waiting for app on laptop") { + t.Fatalf("a machine that has not answered its last send was not waited on: tier %d, %q", p.Tier, p.Note) + } +} diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 1e656622..41d7319d 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -106,6 +106,8 @@ func run() error { return killCommand(ctx, args[1:]) case "forget-checks": return forgetChecksCommand(ctx, args[1:]) + case "try": + return tryCommand(ctx, args[1:]) case "pause", "resume": return pauseCommand(ctx, args[0], args[1:]) case "collection": @@ -381,6 +383,13 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error { // **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is // recorded or registered. if result.Check != nil || result.Checked != nil { + if isTry(result) { + // A branch tried before its pull request (novox/hq issue 686): its asker reads the outcome from the + // build's log. Never said as `checked`, so the forge's holder sets no status. + fmt.Printf("%s: a try of %s at %.8s, read with `try --id`: said to nobody, nothing taken in\n", result.ID, + result.Repository, result.Ref) + return nil + } checked(ctx, result) return nil } diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index a5f705b1..7d60b16a 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -456,6 +456,9 @@ var plainWordings = map[string]func(conditions.Observation) words{ "release-held": worded(func(o conditions.Observation) words { return releaseHeldWords(nil, o.Also) }), + "release-silent": worded(func(o conditions.Observation) words { + return releaseSilentWords(o.Also) + }), "facts-stale": worded(func(o conditions.Observation) words { return words{Headline: "Merge checks use outdated facts", Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " + @@ -841,6 +844,19 @@ func releaseHeldWords(modules, machines []string) words { Resolved: "Resolved: the held updates are released"} } +// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4). +func releaseSilentWords(machines []string) words { + where := "Some machines" + if len(machines) > 0 { + where = conditions.Capital(namesWords(machines, 4)) + } + return words{Headline: "Machines did not answer an update", + Needs: "check that they are awake and on the mesh; the next walk sends them what waits.", + Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " + + "they were left as they were. What they run is not known to be the update.", + Resolved: "Resolved: the next walk reached them"} +} + // reloginNeeds is what an account waiting for its groups needs (ADR 0252). func reloginNeeds(node string) string { return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node) diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go index ca9c0bd4..37d68311 100644 --- a/cmd/mesh-controller/release.go +++ b/cmd/mesh-controller/release.go @@ -39,8 +39,8 @@ import ( // from the same manifest is no move. // // **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no -// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last, -// each sent and judged before the next. A release plan that fails stops the next one opening on its own: +// started plan is walking, the mesh opens one — every such machine, the first sent and judged, then the rest +// at once, each judged by its own gate (novox/hq ADR 0353, in place of one at a time). A release plan that fails stops the next one opening on its own: // a person releases it again (`upgrade release-backlog --why`), after the condition says why. // sendScope is what a send may carry that no gate has seen. @@ -81,6 +81,15 @@ type moveFacts struct { srcs map[string]map[string]string passed map[string]map[string]bool plans []inventory.Plan + // self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353). + self string +} + +type walkingKey struct{} + +// withWalking says a release plan is the one sending: its own gates never hold its own sends back. +func withWalking(ctx context.Context, id string) context.Context { + return context.WithValue(ctx, walkingKey{}, id) } func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) { @@ -99,6 +108,7 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er return f, err } f.plans, err = inv.OpenPlans(ctx) + f.self, _ = ctx.Value(walkingKey{}).(string) return f, err } @@ -193,17 +203,24 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string, // feet. A release keeps no module records: its open gate's carried moves are its walk. func (f moveFacts) walkedBy(module, node, to string) string { for _, p := range f.plans { - if !p.Open() { + if !p.Open() || (p.Release != nil && p.ID == f.self) { continue } if p.Release != nil { - g := p.Release.Gate - if g == nil || g.Verdict != "" { - continue + // The first machine's gate, and every gate of the rest still judging (novox/hq ADR 0353). + gates := []*inventory.PlanGate{p.Release.Gate} + for _, g := range p.Release.Gates { + gates = append(gates, g) } - for _, c := range g.Carried { - if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) { - return p.ID + for _, g := range gates { + // A send kept and not yet made walks nothing yet (ADR 0353 §8). + if g == nil || g.Verdict != "" || g.Silent || g.Pending { + continue + } + for _, c := range g.Carried { + if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) { + return p.ID + } } } continue @@ -303,8 +320,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st } } if len(refused) > 0 { - return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+ - "0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; ")) + return nil, fmt.Errorf("%w: %s — a walk sends them, one machine first and then the rest at once, each judged (novox/hq ADR "+ + "0236, ADR 0353); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; ")) } return kept, nil } @@ -320,19 +337,35 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st // And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490): // answered for the gate to keep, nil when there was none. func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) { - inv := open.inventory - f, err := readMoveFacts(ctx, inv) + moves, err := gatedMoves(ctx, open, node, owns) + if err != nil || len(moves) == 0 { + return nil, nil, nil, err + } + sayRecreations(ctx, open, moves) + sent, grants, err := sendJudged(ctx, open, node) if err != nil { return nil, nil, nil, err } + return moves, sent, grants, nil +} + +// gatedMoves is what a gated send to one machine would carry, with the build each moves to, made of nothing: +// gatedSend's reading, which a walk also keeps before it sends (novox/hq ADR 0353 §8). Refused the same way +// while a plan that has started walks one of those builds elsewhere. +func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, error) { + inv := open.inventory + f, err := readMoveFacts(ctx, inv) + if err != nil { + return nil, err + } moves, err := machineMoves(ctx, open, f, node, true) if err != nil { - return nil, nil, nil, err + return nil, err } // A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module // kept over an older record judged as moving back to it would read the gate as superseded by its own send. if moves, err = withoutKept(ctx, open, node, moves); err != nil { - return nil, nil, nil, err + return nil, err } own := func(module string) bool { return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module }) @@ -342,7 +375,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory. continue } if id := f.walkedBy(mv.Module, node, mv.To); id != "" { - return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, + return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, mv.Module, short(mv.To), node, id) } } @@ -350,7 +383,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory. // plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate. for _, o := range owns { if id := f.walkedBy(o.Module, node, o.To); id != "" { - return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, + return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, o.Module, short(o.To), node, id) } } @@ -363,30 +396,26 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory. moves[i].Build = o.Build } } - if len(moves) == 0 && len(owns) == 0 { - return nil, nil, nil, nil + if len(moves) == 0 { + return nil, nil } for i := range moves { if moves[i].Build == "" { if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil { - return nil, nil, nil, err + return nil, err } } } sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module }) - sayRecreations(ctx, open, moves) - sent, grants, err := sendJudged(ctx, open, node) - if err != nil { - return nil, nil, nil, err - } - return moves, sent, grants, nil + return moves, nil } // passCarried keeps a pass as the verdict of every build the gate judged beside its own module. func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) { seen := map[string]bool{} for _, c := range g.Carried { - if c.Module == except || c.Build == "" || seen[c.Build] { + // A build put back with another machine's failure is not this gate's to pass (novox/hq ADR 0353 §3). + if c.Module == except || c.Build == "" || seen[c.Build] || slices.Contains(g.Returned, c.Module) { continue } seen[c.Build] = true @@ -695,7 +724,7 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository, Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{}, Release: &inventory.PlanRelease{Order: order, By: by}, - Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names), + Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine first, then the rest at once, each judged", len(names), strings.Join(order, ", "))} if err := inv.SavePlan(ctx, &p); err != nil { return nil, err @@ -704,14 +733,23 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl return &p, nil } -// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a -// gate, or the machine being judged judged once more; the plan done when every machine is. +// advanceRelease takes one step of a release plan: the first machine sent everything waiting there under a +// gate and judged; once it passed, every other machine sent at once, each under its own gate (novox/hq ADR +// 0353 §1), and judged; the plan done when every machine is. +// +// **A send is kept before it is made** (ADR 0353 §8): the step that chooses a machine keeps its gate pending with +// what it will carry, and the next step makes the send. A controller replaced between the send and keeping it +// finds the gate pending, reads the machine's last send, and neither sends again nor leaves the machine unjudged. func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) { r := p.Release if r.Settings != nil { return advanceSettingsWalk(ctx, open, p) } + ctx = withWalking(ctx, p.ID) now := time.Now().UTC() + if r.Rest { + return advanceRest(ctx, open, p, now) + } if r.Gate == nil { heard, err := releaseHeard(ctx, open) if err != nil { @@ -724,7 +762,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, r.Next++ continue } - moves, sent, grants, err := gatedSend(ctx, open, node, nil) + moves, err := gatedMoves(ctx, open, node, nil) if errors.Is(err, errWalkedElsewhere) { note := fmt.Sprintf("waiting before %s: %v", node, err) changed := p.Note != note @@ -732,23 +770,15 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, return changed, nil } if err != nil { - return false, fmt.Errorf("sending %s what waits there: %w", node, err) + return false, fmt.Errorf("reading what waits on %s: %w", node, err) } if len(moves) == 0 { r.Done = append(r.Done, node) r.Next++ continue } - r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent), - Grants: grants} - p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves)) - if said := grantsSaid(grants); said != "" { - p.Note += "; " + said - } - if said := recreationsSaid(moves); said != "" { - p.Note += "; " + said - } - fmt.Printf("%s: %s\n", p.ID, p.Note) + r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true} + p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves)) return true, nil } p.State, p.Tier = inventory.PlanDone, len(p.Tiers) @@ -759,6 +789,19 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, return true, nil } g := r.Gate + if g.Pending { + node := firstOf(g.Machines) + out, err := sendKept(ctx, open, p, node, g, now) + if err != nil || out == keptWaiting { + return out == keptWaiting, err + } + if out == keptNothing { + r.Gate = nil + r.Done = append(r.Done, node) + r.Next++ + } + return true, nil + } verdict, err := judgeMoves(ctx, open, g, nil, now) if err != nil { return false, err @@ -777,6 +820,11 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, r.Done = append(r.Done, firstOf(g.Machines)) r.Next++ r.Gate = nil + // **Then the rest at once** (novox/hq ADR 0353 §1): every other machine heard from, each judged by its own + // gate, in place of one machine at a time (ADR 0236 §22). + r.Rest = true + p.Note = fmt.Sprintf("passed on %s; the rest are sent at once, each judged", firstOf(g.Machines)) + fmt.Printf("%s: %s\n", p.ID, p.Note) return true, nil case inventory.GateSuperseded: // Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what @@ -797,6 +845,453 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, return true, nil } +// What sendKept did with a kept send. +const ( + keptSent = iota // sent now + keptAlready // the machine was already sent it: judged from that send, nothing sent + keptNothing // nothing waits there any more: no gate + keptWaiting // another walk judges a build it would carry: it waits +) + +// sendKept makes a send a gate kept before it was made (novox/hq ADR 0353 §8) — unless the machine's last send +// already carries what the gate is for, made before a controller was replaced and before it was kept: then +// nothing is sent, and the gate judges from that send (§6). +func sendKept(ctx context.Context, open *stores, p *inventory.Plan, node string, g *inventory.PlanGate, + now time.Time) (int, error) { + inv := open.inventory + moves, sent, grants, err := gatedSend(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + p.Note = fmt.Sprintf("waiting before %s: %v", node, err) + return keptWaiting, nil + } + if err != nil { + return keptSent, fmt.Errorf("sending %s what waits there: %w", node, err) + } + if len(moves) > 0 { + g.Machines, g.Carried, g.Since, g.Grants, g.Pending = sent, moves, &now, grants, false + g.Sent = sentNow(ctx, inv, sent) + p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves)) + if said := grantsSaid(grants); said != "" { + p.Note += "; " + said + } + if said := recreationsSaid(moves); said != "" { + p.Note += "; " + said + } + fmt.Printf("%s: %s\n", p.ID, p.Note) + return keptSent, nil + } + if !alreadySent(ctx, inv, node, g.Carried) { + return keptNothing, nil + } + since := now + if s, found, err := inv.SentTo(ctx, node); err == nil && found { + if !s.At.IsZero() { + since = s.At + } + g.Sent = map[string]inventory.SentDeclaration{node: s} + } + g.Since, g.Pending = &since, false + p.Note = fmt.Sprintf("%s was already sent the %d build(s) kept for it, at %s: judged from that send, nothing sent again", + node, len(g.Carried), since.Local().Format("15:04:05")) + fmt.Printf("%s: %s\n", p.ID, p.Note) + return keptAlready, nil +} + +// alreadySent says a machine's last send carries every build of the moves: a send made and not kept. +func alreadySent(ctx context.Context, inv *inventory.Inventory, node string, moves []inventory.CarriedMove) bool { + if len(moves) == 0 { + return false + } + sent, known, err := inv.SentBuilds(ctx, node) + if err != nil || !known { + return false + } + for _, mv := range moves { + if !sameCommit(sent[mv.Module], mv.To) { + return false + } + } + return true +} + +// advanceRest takes one step of a release plan's rest (novox/hq ADR 0353): the sends kept for a step made, every +// machine of the step judged by its own gate, one that fails put back by itself while the others go on to their +// verdicts, and the next step — or the end — once every gate of the step has its verdict or is silent. +func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.Time) (bool, error) { + r := p.Release + // The sends kept for this step, in the plan's order: the control node, last in it, last (§1). + var pending []string + for _, n := range r.Order { + if g := r.Gates[n]; g != nil && g.Pending { + pending = append(pending, n) + } + } + if len(pending) > 0 { + // **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on + // a machine since its step was chosen could reach a second machine in it; the step is chosen again. + if grew, err := keptStepGrew(ctx, open, r, pending); err != nil { + return false, err + } else if grew != "" { + for _, n := range pending { + delete(r.Gates, n) + } + opened, waiting, err := openRestStep(ctx, open, p) + if err != nil { + return true, err + } + p.Note = "chosen again: " + grew + if !opened && waiting != "" { + p.Note += "; " + waiting + } + return true, nil + } + var sent, already, waiting []string + for _, node := range pending { + out, err := sendKept(ctx, open, p, node, r.Gates[node], now) + if err != nil { + return true, err + } + switch out { + case keptSent: + sent = append(sent, node) + case keptAlready: + already = append(already, node) + case keptNothing: + delete(r.Gates, node) + r.Done = append(r.Done, node) + case keptWaiting: + waiting = append(waiting, node) + } + } + var said []string + if len(sent) > 0 { + said = append(said, "sent "+strings.Join(sent, ", ")+" at once, each judged there") + } + if len(already) > 0 { + said = append(said, strings.Join(already, ", ")+" already sent it: judged from that send") + } + if len(waiting) > 0 { + said = append(said, "waiting before "+strings.Join(waiting, ", ")+": another walk judges a build it carries") + } + if len(said) > 0 { + p.Note = strings.Join(said, "; ") + } + return len(sent)+len(already) > 0 || len(waiting) < len(pending), nil + } + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return false, err + } + byNode := map[string]inventory.Reported{} + for _, rep := range reports { + byNode[rep.Node] = rep + } + changed, noteWas := false, p.Note + var notes, judging []string + batched, back := batchingRollbacks(ctx) + for _, node := range r.Order { + g := r.Gates[node] + if g == nil || g.Verdict != "" || g.Silent { + continue + } + // What another machine's failure already put back is not this gate's to judge (ADR 0353 §3). + if len(g.Carried) > 0 && len(g.Returned) >= len(g.Carried) { + decide(g, inventory.GateFailed, "what it was sent was put back with another machine's failure", now) + r.Failed = append(r.Failed, node) + changed = true + continue + } + // **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged + // until it reports, and past ApplySilentAfter it is listed and left. + if !reportedOnSend(byNode[node], node, g) { + if now.Sub(*g.Since) > inventory.ApplySilentAfter { + g.Silent = true + r.Silent = append(r.Silent, node) + changed = true + notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node, + inventory.ApplySilentAfter)) + continue + } + judging = append(judging, node+": not yet reported on its send") + continue + } + verdict, err := judgeMoves(ctx, open, g, nil, now) + if err != nil { + return changed, err + } + switch verdict { + case "": + if putBackBroken(ctx, open, p, g, "", nil) { + changed = true + notes = append(notes, p.Note) + } + judging = append(judging, node+": "+gateLine(g)) + case inventory.GatePassed: + passCarried(ctx, open, p, g, "") + r.Done = append(r.Done, node) + changed = true + case inventory.GateSuperseded: + // Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on + // it; what still waits is released again. + r.Done = append(r.Done, node) + changed = true + notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why)) + default: + // **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other + // machine of the step goes on to its own verdict. + stateWas := p.State + p.Note = "" + failCarried(batched, open, p, g, "") + p.State = stateWas + r.Failed = append(r.Failed, node) + changed = true + notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note)) + returnedElsewhere(ctx, open, r, node, g) + } + } + p.Note = "" + sendRollbacks(ctx, open, p, back) + if p.Note != "" { + notes = append(notes, strings.TrimPrefix(p.Note, "; ")) + } + if len(judging) > 0 { + note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ") + p.Note = note + return changed || noteWas != note, nil + } + // Every gate of the step has its verdict or is silent: the next step, unless a machine failed (§3). + if len(r.Failed) == 0 { + opened, waiting, err := openRestStep(ctx, open, p) + if err != nil { + return changed, err + } + if opened { + return true, nil + } + if waiting != "" { + note := strings.Join(append(notes, waiting), "; ") + p.Note = note + return changed || noteWas != note, nil + } + } + p.Tier = len(p.Tiers) + end := fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", "))) + if len(r.Silent) > 0 { + end += "; silent on their send, left: " + strings.Join(r.Silent, ", ") + } + if len(r.Skipped) > 0 { + end += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") + } + p.State = inventory.PlanDone + if len(r.Failed) > 0 { + p.State = inventory.PlanFailed + end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end + } else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) { + // **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it. + p.State = inventory.PlanFailed + end = "no machine of the rest reported on its send; " + end + } + p.Note = strings.Join(append(notes, end), "; ") + fmt.Printf("%s: %s\n", p.ID, p.Note) + return true, nil +} + +// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with; +// empty when none would. +func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) { + f, err := readMoveFacts(ctx, open.inventory) + if err != nil { + return "", err + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return "", err + } + for _, node := range pending { + moves, err := gatedMoves(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + continue // sendKept says it waits + } + if err != nil { + return "", err + } + kept := r.Gates[node].Carried + for _, mv := range moves { + // The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9). + if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool { + return c.Module == mv.Module && sameCommit(c.To, mv.To) + }) { + continue + } + return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil + } + } + return "", nil +} + +// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the +// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last. +func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool { + if r.At == nil { + return false + } + if s, kept := g.Sent[node]; kept { + return s.ReportsOn(r) + } + return r.Current +} + +// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's +// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3). +func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) { + for _, c := range g.Carried { + if c.Build == "" { + continue + } + if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked { + continue + } + for node, other := range r.Gates { + if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) { + continue + } + for _, oc := range other.Carried { + if oc.Module == c.Module && sameCommit(oc.To, c.To) { + other.Returned = append(other.Returned, c.Module) + } + } + } + } +} + +// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2): +// every machine left that is heard from and has something waiting, except that a build no gate has passed goes +// to one machine of the step only. Answers whether a step was kept, or why the walk waits. +func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, string, error) { + r := p.Release + heard, err := releaseHeard(ctx, open) + if err != nil { + return false, "", err + } + f, err := readMoveFacts(ctx, open.inventory) + if err != nil { + return false, "", err + } + waiting := map[string][]inventory.CarriedMove{} + var nodes, walked []string + for _, node := range r.Order[min(r.Next, len(r.Order)):] { + if r.Gates[node] != nil || slices.Contains(r.Done, node) || slices.Contains(r.Skipped, node) || + slices.Contains(r.Failed, node) || slices.Contains(r.Silent, node) { + continue + } + if !heard[node] { + r.Skipped = append(r.Skipped, node) + continue + } + moves, err := gatedMoves(ctx, open, node, nil) + if errors.Is(err, errWalkedElsewhere) { + walked = append(walked, fmt.Sprintf("%s (%v)", node, err)) + continue + } + if err != nil { + return false, "", fmt.Errorf("reading what waits on %s: %w", node, err) + } + if len(moves) == 0 { + r.Done = append(r.Done, node) + continue + } + waiting[node] = moves + nodes = append(nodes, node) + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + return false, "", err + } + step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) }) + if len(step) == 0 { + if len(walked) > 0 { + return false, "waiting before " + strings.Join(walked, "; "), nil + } + return false, "", nil + } + if r.Gates == nil { + r.Gates = map[string]*inventory.PlanGate{} + } + for _, node := range step { + r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true} + } + p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", ")) + if left := len(nodes) - len(step); left > 0 { + p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left) + } + return true, "", nil +} + +// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the +// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus. +func oneAtATime(module string, m catalogue.Manifest) bool { + if module == hostModule { + return true + } + return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat }) +} + +// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless +// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's +// first machine. Pure. +// +// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a +// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus +// shows only as its silence and no rollback reaches it. +func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool, + alone func(module string) bool) []string { + first := map[string]string{} + var out []string + for _, n := range nodes { + var unpassed []string + for _, mv := range waiting[n] { + if !passed(mv.Module, mv.To) || alone(mv.Module) { + unpassed = append(unpassed, mv.Module+"@"+mv.To) + } + } + if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) { + continue + } + for _, k := range unpassed { + first[k] = n + } + out = append(out, n) + } + return out +} + +// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353 +// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing. +func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) { + recent, err := inv.RecentPlans(ctx, 20) + if err != nil { + return nil, err + } + for _, p := range recent { + if p.Release == nil || p.Release.Settings != nil || p.Open() { + continue + } + if len(p.Release.Silent) == 0 { + return nil, nil + } + machines := append([]string(nil), p.Release.Silent...) + sort.Strings(machines) + o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent", + Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines, + Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+ + "left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)} + w := releaseSilentWords(machines) + o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs + return []conditions.Observation{o}, nil + } + return nil, nil +} + // backlogObservation is what the gate's probe says of a release held after a failure. func backlogObservation() []conditions.Observation { if backlogNow.held == "" || len(backlogNow.waiting) == 0 {