Bring main into release 57's branch, part 1 of 3: the builder's try outcome and the release walk
So the release branch carries what main gained; parts 2 and 3 follow on this branch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -332,6 +332,12 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri
|
|||||||
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo,
|
||||||
Commit: m.Head, Trunk: m.Trunk})
|
Commit: m.Head, Trunk: m.Trunk})
|
||||||
}
|
}
|
||||||
|
// A branch tried before its pull request reads its outcome from this log, whole (novox/hq issue 686).
|
||||||
|
if request.Check.Try {
|
||||||
|
if line, err := json.Marshal(triedOf(v)); err == nil {
|
||||||
|
say(link.StepTried, string(line))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else if err == nil {
|
} else if err == nil {
|
||||||
// The package-registry credential is a build input, so it is resolved before the clone: a
|
// The package-registry credential is a build input, so it is resolved before the clone: a
|
||||||
@@ -457,7 +463,8 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
|||||||
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
spec := builder.CheckSpec{ID: request.ID, Repository: request.Repository, Ref: request.Ref,
|
||||||
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{},
|
||||||
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base,
|
||||||
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held)}
|
Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held),
|
||||||
|
Tests: c.Tests, StoreOnly: c.StoreOnly}
|
||||||
for dir, b := range c.Beside {
|
for dir, b := range c.Beside {
|
||||||
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref}
|
||||||
}
|
}
|
||||||
@@ -472,6 +479,23 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec {
|
|||||||
return spec
|
return spec
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// triedOf is a try's outcome as its log line says it (novox/hq issue 686): the repository's own check's verdict,
|
||||||
|
// or the check's where that did not run — a head that does not merge — and the lines naming what failed.
|
||||||
|
func triedOf(v builder.CheckVerdict) link.Tried {
|
||||||
|
t := link.Tried{Verdict: v.Verdict, Summary: v.Summary, Took: v.Took.Round(time.Second).String(),
|
||||||
|
Head: v.Head, Trunk: v.Trunk}
|
||||||
|
if v.Repo != nil && (v.Gate == nil || v.Gate.Verdict == "pass") {
|
||||||
|
t.Verdict, t.Summary = v.Repo.Verdict, v.Repo.Summary
|
||||||
|
}
|
||||||
|
if v.Repo != nil && v.Repo.Failed != "" {
|
||||||
|
t.Failed = strings.Split(v.Repo.Failed, "\n")
|
||||||
|
}
|
||||||
|
if t.Verdict == "" {
|
||||||
|
t.Verdict = "error"
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
|
// checkBuiltOf is a check's builds for its merge as the outcome carries them (novox/hq ADR 0350 §1).
|
||||||
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
|
func checkBuiltOf(built []builder.CheckBuilt) []link.CheckBuilt {
|
||||||
var out []link.CheckBuilt
|
var out []link.CheckBuilt
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/builder"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A try's outcome, as the build seat says it in the build's log (novox/hq issue 686): the repository's own
|
||||||
|
// check's verdict, how long it took, and each line naming what failed.
|
||||||
|
func TestATrysOutcomeNamesWhatFailed(t *testing.T) {
|
||||||
|
tried := triedOf(builder.CheckVerdict{Verdict: "pass", Summary: "the change touches no module", Took: 303 * time.Second,
|
||||||
|
Head: "c0ffee", Trunk: "ca3f43c", Gate: &builder.Layer{Verdict: "pass"},
|
||||||
|
Repo: &builder.Layer{Verdict: "fail", Summary: "its merge-check.sh failed: TestX",
|
||||||
|
Failed: "--- FAIL: TestX (0.01s)\n x_test.go:12: got 1, want 2"}})
|
||||||
|
if tried.Verdict != "fail" || tried.Took != "5m3s" || len(tried.Failed) != 2 || tried.Trunk != "ca3f43c" ||
|
||||||
|
tried.Summary != "its merge-check.sh failed: TestX" {
|
||||||
|
t.Fatalf("a failed try: %+v", tried)
|
||||||
|
}
|
||||||
|
// A head that does not merge fails both layers: the check's own word.
|
||||||
|
conflict := triedOf(builder.CheckVerdict{Verdict: "fail", Summary: "does not merge", Gate: &builder.Layer{Verdict: "fail"},
|
||||||
|
Repo: &builder.Layer{Verdict: "fail", Summary: "does not merge"}})
|
||||||
|
if conflict.Verdict != "fail" || conflict.Summary != "does not merge" {
|
||||||
|
t.Fatalf("a try that does not merge: %+v", conflict)
|
||||||
|
}
|
||||||
|
// No verdict is an error, never a pass.
|
||||||
|
if none := triedOf(builder.CheckVerdict{}); none.Verdict != "error" {
|
||||||
|
t.Fatalf("a try with no verdict: %+v", none)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1443,6 +1443,12 @@ func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error
|
|||||||
}
|
}
|
||||||
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
|
// A release held after a failed one, while builds still wait for a gate (ADR 0236).
|
||||||
out = append(out, backlogObservation()...)
|
out = append(out, backlogObservation()...)
|
||||||
|
// And a rest that went silent (novox/hq ADR 0353 §4).
|
||||||
|
silent, err := silentObservation(ctx, d.open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
out = append(out, silent...)
|
||||||
return sortedFound(dedupeObservations(out)), nil
|
return sortedFound(dedupeObservations(out)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1532,6 +1538,12 @@ func gateLine(g *inventory.PlanGate) string {
|
|||||||
case inventory.GateFailed:
|
case inventory.GateFailed:
|
||||||
what = "FAILED"
|
what = "FAILED"
|
||||||
}
|
}
|
||||||
|
switch {
|
||||||
|
case g.Verdict == "" && g.Pending:
|
||||||
|
what = "kept, not yet sent" // novox/hq ADR 0353 §8
|
||||||
|
case g.Verdict == "" && g.Silent:
|
||||||
|
what = "silent on its send, left" // ADR 0353 §4
|
||||||
|
}
|
||||||
if g.Verdict == "" && g.Waiting() {
|
if g.Verdict == "" && g.Waiting() {
|
||||||
what = "waits for a busy container on " + g.Busy.Machine
|
what = "waits for a busy container on " + g.Busy.Machine
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -151,7 +151,7 @@ func heldMachines(ctx context.Context, open *stores, names []string) (map[string
|
|||||||
for _, mv := range f.moves(node, modules, same, known, false) {
|
for _, mv := range f.moves(node, modules, same, known, false) {
|
||||||
if planStillToSend(plans, mv.Module, node) == "" {
|
if planStillToSend(plans, mv.Module, node) == "" {
|
||||||
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+
|
||||||
"plan sends it, one machine at a time, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
"plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(why) > 0 {
|
if len(why) > 0 {
|
||||||
|
|||||||
@@ -0,0 +1,87 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/novox/mesh-controller/internal/catalogue"
|
||||||
|
"github.com/novox/mesh-controller/internal/inventory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A tier whose module is no move — made from the source every machine running it runs, so nothing is
|
||||||
|
// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the
|
||||||
|
// send that last carried that build before the walk got there, and nothing will make them report again:
|
||||||
|
// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever.
|
||||||
|
func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
|
||||||
|
judgeLeavesAsTheCore(t)
|
||||||
|
g := aGateMesh(t)
|
||||||
|
ctx := t.Context()
|
||||||
|
inv := g.open.inventory
|
||||||
|
|
||||||
|
// app is the build machine, so a module of the next tier is built by it: the tier gates the next on app
|
||||||
|
// running on its machines.
|
||||||
|
for _, m := range []catalogue.Manifest{
|
||||||
|
{Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}},
|
||||||
|
{Module: "web", Version: "w1"},
|
||||||
|
} {
|
||||||
|
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
||||||
|
Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is.
|
||||||
|
for _, b := range []inventory.Build{
|
||||||
|
{ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app",
|
||||||
|
Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)},
|
||||||
|
{ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()},
|
||||||
|
} {
|
||||||
|
b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
||||||
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Each machine answered that send, applied, before the walk's tier was built.
|
||||||
|
for _, n := range []string{"anchor", "laptop"} {
|
||||||
|
if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n,
|
||||||
|
Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1,
|
||||||
|
At: time.Now().Add(-10 * time.Second)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
built := time.Now().UTC()
|
||||||
|
p := g.plan(t)
|
||||||
|
p.Tiers = [][]string{{"app"}, {"web"}}
|
||||||
|
p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
advancePlans(ctx, g.open)
|
||||||
|
|
||||||
|
if len(g.sent) != 0 {
|
||||||
|
t.Fatalf("a no-move build was sent: %v", g.sent)
|
||||||
|
}
|
||||||
|
p = g.plan(t)
|
||||||
|
if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") {
|
||||||
|
t.Fatalf("the build was not marked no move: %+v", s)
|
||||||
|
}
|
||||||
|
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
|
||||||
|
t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A machine that has not answered its last send is still waited on: no move is not "never sent".
|
||||||
|
p.Tier, p.State, p.Note = 0, inventory.PlanRolling, ""
|
||||||
|
if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
advancePlans(ctx, g.open)
|
||||||
|
p = g.plan(t)
|
||||||
|
if p.Tier != 0 || !strings.Contains(p.Note, "waiting for app on laptop") {
|
||||||
|
t.Fatalf("a machine that has not answered its last send was not waited on: tier %d, %q", p.Tier, p.Note)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -106,6 +106,8 @@ func run() error {
|
|||||||
return killCommand(ctx, args[1:])
|
return killCommand(ctx, args[1:])
|
||||||
case "forget-checks":
|
case "forget-checks":
|
||||||
return forgetChecksCommand(ctx, args[1:])
|
return forgetChecksCommand(ctx, args[1:])
|
||||||
|
case "try":
|
||||||
|
return tryCommand(ctx, args[1:])
|
||||||
case "pause", "resume":
|
case "pause", "resume":
|
||||||
return pauseCommand(ctx, args[0], args[1:])
|
return pauseCommand(ctx, args[0], args[1:])
|
||||||
case "collection":
|
case "collection":
|
||||||
@@ -381,6 +383,13 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error {
|
|||||||
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
|
// **A merge check builds nothing** (novox/hq to-be 45 §9): its verdict is said, and nothing of it is
|
||||||
// recorded or registered.
|
// recorded or registered.
|
||||||
if result.Check != nil || result.Checked != nil {
|
if result.Check != nil || result.Checked != nil {
|
||||||
|
if isTry(result) {
|
||||||
|
// A branch tried before its pull request (novox/hq issue 686): its asker reads the outcome from the
|
||||||
|
// build's log. Never said as `checked`, so the forge's holder sets no status.
|
||||||
|
fmt.Printf("%s: a try of %s at %.8s, read with `try --id`: said to nobody, nothing taken in\n", result.ID,
|
||||||
|
result.Repository, result.Ref)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
checked(ctx, result)
|
checked(ctx, result)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -456,6 +456,9 @@ var plainWordings = map[string]func(conditions.Observation) words{
|
|||||||
"release-held": worded(func(o conditions.Observation) words {
|
"release-held": worded(func(o conditions.Observation) words {
|
||||||
return releaseHeldWords(nil, o.Also)
|
return releaseHeldWords(nil, o.Also)
|
||||||
}),
|
}),
|
||||||
|
"release-silent": worded(func(o conditions.Observation) words {
|
||||||
|
return releaseSilentWords(o.Also)
|
||||||
|
}),
|
||||||
"facts-stale": worded(func(o conditions.Observation) words {
|
"facts-stale": worded(func(o conditions.Observation) words {
|
||||||
return words{Headline: "Merge checks use outdated facts",
|
return words{Headline: "Merge checks use outdated facts",
|
||||||
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
Explanation: "The facts a merge check judges a change against are out of date; they are gathered again " +
|
||||||
@@ -841,6 +844,19 @@ func releaseHeldWords(modules, machines []string) words {
|
|||||||
Resolved: "Resolved: the held updates are released"}
|
Resolved: "Resolved: the held updates are released"}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// releaseSilentWords are the plain words of machines that said nothing on an update they were sent (ADR 0353 §4).
|
||||||
|
func releaseSilentWords(machines []string) words {
|
||||||
|
where := "Some machines"
|
||||||
|
if len(machines) > 0 {
|
||||||
|
where = conditions.Capital(namesWords(machines, 4))
|
||||||
|
}
|
||||||
|
return words{Headline: "Machines did not answer an update",
|
||||||
|
Needs: "check that they are awake and on the mesh; the next walk sends them what waits.",
|
||||||
|
Explanation: where + " were sent module updates and said nothing about them within fifteen minutes, so " +
|
||||||
|
"they were left as they were. What they run is not known to be the update.",
|
||||||
|
Resolved: "Resolved: the next walk reached them"}
|
||||||
|
}
|
||||||
|
|
||||||
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
// reloginNeeds is what an account waiting for its groups needs (ADR 0252).
|
||||||
func reloginNeeds(node string) string {
|
func reloginNeeds(node string) string {
|
||||||
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node)
|
||||||
|
|||||||
+537
-42
@@ -39,8 +39,8 @@ import (
|
|||||||
// from the same manifest is no move.
|
// from the same manifest is no move.
|
||||||
//
|
//
|
||||||
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
|
// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no
|
||||||
// started plan is walking, the mesh opens one — every such machine, one at a time, the control node last,
|
// started plan is walking, the mesh opens one — every such machine, the first sent and judged, then the rest
|
||||||
// each sent and judged before the next. A release plan that fails stops the next one opening on its own:
|
// at once, each judged by its own gate (novox/hq ADR 0353, in place of one at a time). A release plan that fails stops the next one opening on its own:
|
||||||
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
|
// a person releases it again (`upgrade release-backlog --why`), after the condition says why.
|
||||||
|
|
||||||
// sendScope is what a send may carry that no gate has seen.
|
// sendScope is what a send may carry that no gate has seen.
|
||||||
@@ -81,6 +81,15 @@ type moveFacts struct {
|
|||||||
srcs map[string]map[string]string
|
srcs map[string]map[string]string
|
||||||
passed map[string]map[string]bool
|
passed map[string]map[string]bool
|
||||||
plans []inventory.Plan
|
plans []inventory.Plan
|
||||||
|
// self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353).
|
||||||
|
self string
|
||||||
|
}
|
||||||
|
|
||||||
|
type walkingKey struct{}
|
||||||
|
|
||||||
|
// withWalking says a release plan is the one sending: its own gates never hold its own sends back.
|
||||||
|
func withWalking(ctx context.Context, id string) context.Context {
|
||||||
|
return context.WithValue(ctx, walkingKey{}, id)
|
||||||
}
|
}
|
||||||
|
|
||||||
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
|
func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) {
|
||||||
@@ -99,6 +108,7 @@ func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, er
|
|||||||
return f, err
|
return f, err
|
||||||
}
|
}
|
||||||
f.plans, err = inv.OpenPlans(ctx)
|
f.plans, err = inv.OpenPlans(ctx)
|
||||||
|
f.self, _ = ctx.Value(walkingKey{}).(string)
|
||||||
return f, err
|
return f, err
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -193,17 +203,24 @@ func (f moveFacts) moves(node string, modules []string, sent map[string]string,
|
|||||||
// feet. A release keeps no module records: its open gate's carried moves are its walk.
|
// feet. A release keeps no module records: its open gate's carried moves are its walk.
|
||||||
func (f moveFacts) walkedBy(module, node, to string) string {
|
func (f moveFacts) walkedBy(module, node, to string) string {
|
||||||
for _, p := range f.plans {
|
for _, p := range f.plans {
|
||||||
if !p.Open() {
|
if !p.Open() || (p.Release != nil && p.ID == f.self) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if p.Release != nil {
|
if p.Release != nil {
|
||||||
g := p.Release.Gate
|
// The first machine's gate, and every gate of the rest still judging (novox/hq ADR 0353).
|
||||||
if g == nil || g.Verdict != "" {
|
gates := []*inventory.PlanGate{p.Release.Gate}
|
||||||
continue
|
for _, g := range p.Release.Gates {
|
||||||
|
gates = append(gates, g)
|
||||||
}
|
}
|
||||||
for _, c := range g.Carried {
|
for _, g := range gates {
|
||||||
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
|
// A send kept and not yet made walks nothing yet (ADR 0353 §8).
|
||||||
return p.ID
|
if g == nil || g.Verdict != "" || g.Silent || g.Pending {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, c := range g.Carried {
|
||||||
|
if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) {
|
||||||
|
return p.ID
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
@@ -303,8 +320,8 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(refused) > 0 {
|
if len(refused) > 0 {
|
||||||
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine at a time, each judged (novox/hq ADR "+
|
return nil, fmt.Errorf("%w: %s — a walk sends them, one machine first and then the rest at once, each judged (novox/hq ADR "+
|
||||||
"0236); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
"0236, ADR 0353); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; "))
|
||||||
}
|
}
|
||||||
return kept, nil
|
return kept, nil
|
||||||
}
|
}
|
||||||
@@ -320,19 +337,35 @@ func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder st
|
|||||||
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
|
// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490):
|
||||||
// answered for the gate to keep, nil when there was none.
|
// answered for the gate to keep, nil when there was none.
|
||||||
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
|
func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) {
|
||||||
inv := open.inventory
|
moves, err := gatedMoves(ctx, open, node, owns)
|
||||||
f, err := readMoveFacts(ctx, inv)
|
if err != nil || len(moves) == 0 {
|
||||||
|
return nil, nil, nil, err
|
||||||
|
}
|
||||||
|
sayRecreations(ctx, open, moves)
|
||||||
|
sent, grants, err := sendJudged(ctx, open, node)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, nil, nil, err
|
||||||
}
|
}
|
||||||
|
return moves, sent, grants, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// gatedMoves is what a gated send to one machine would carry, with the build each moves to, made of nothing:
|
||||||
|
// gatedSend's reading, which a walk also keeps before it sends (novox/hq ADR 0353 §8). Refused the same way
|
||||||
|
// while a plan that has started walks one of those builds elsewhere.
|
||||||
|
func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
f, err := readMoveFacts(ctx, inv)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
moves, err := machineMoves(ctx, open, f, node, true)
|
moves, err := machineMoves(ctx, open, f, node, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
|
// A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module
|
||||||
// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
|
// kept over an older record judged as moving back to it would read the gate as superseded by its own send.
|
||||||
if moves, err = withoutKept(ctx, open, node, moves); err != nil {
|
if moves, err = withoutKept(ctx, open, node, moves); err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
own := func(module string) bool {
|
own := func(module string) bool {
|
||||||
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module })
|
||||||
@@ -342,7 +375,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
if id := f.walkedBy(mv.Module, node, mv.To); id != "" {
|
||||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||||
mv.Module, short(mv.To), node, id)
|
mv.Module, short(mv.To), node, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -350,7 +383,7 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
// plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate.
|
||||||
for _, o := range owns {
|
for _, o := range owns {
|
||||||
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
if id := f.walkedBy(o.Module, node, o.To); id != "" {
|
||||||
return nil, nil, nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere,
|
||||||
o.Module, short(o.To), node, id)
|
o.Module, short(o.To), node, id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -363,30 +396,26 @@ func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.
|
|||||||
moves[i].Build = o.Build
|
moves[i].Build = o.Build
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(moves) == 0 && len(owns) == 0 {
|
if len(moves) == 0 {
|
||||||
return nil, nil, nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
for i := range moves {
|
for i := range moves {
|
||||||
if moves[i].Build == "" {
|
if moves[i].Build == "" {
|
||||||
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil {
|
||||||
return nil, nil, nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module })
|
||||||
sayRecreations(ctx, open, moves)
|
return moves, nil
|
||||||
sent, grants, err := sendJudged(ctx, open, node)
|
|
||||||
if err != nil {
|
|
||||||
return nil, nil, nil, err
|
|
||||||
}
|
|
||||||
return moves, sent, grants, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
// passCarried keeps a pass as the verdict of every build the gate judged beside its own module.
|
||||||
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
|
func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) {
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
for _, c := range g.Carried {
|
for _, c := range g.Carried {
|
||||||
if c.Module == except || c.Build == "" || seen[c.Build] {
|
// A build put back with another machine's failure is not this gate's to pass (novox/hq ADR 0353 §3).
|
||||||
|
if c.Module == except || c.Build == "" || seen[c.Build] || slices.Contains(g.Returned, c.Module) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seen[c.Build] = true
|
seen[c.Build] = true
|
||||||
@@ -695,7 +724,7 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
|||||||
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
|
p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository,
|
||||||
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
|
Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{},
|
||||||
Release: &inventory.PlanRelease{Order: order, By: by},
|
Release: &inventory.PlanRelease{Order: order, By: by},
|
||||||
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine at a time, each judged", len(names),
|
Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine first, then the rest at once, each judged", len(names),
|
||||||
strings.Join(order, ", "))}
|
strings.Join(order, ", "))}
|
||||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -704,14 +733,23 @@ func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Pl
|
|||||||
return &p, nil
|
return &p, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// advanceRelease takes one step of a release plan: the next machine sent everything waiting there under a
|
// advanceRelease takes one step of a release plan: the first machine sent everything waiting there under a
|
||||||
// gate, or the machine being judged judged once more; the plan done when every machine is.
|
// gate and judged; once it passed, every other machine sent at once, each under its own gate (novox/hq ADR
|
||||||
|
// 0353 §1), and judged; the plan done when every machine is.
|
||||||
|
//
|
||||||
|
// **A send is kept before it is made** (ADR 0353 §8): the step that chooses a machine keeps its gate pending with
|
||||||
|
// what it will carry, and the next step makes the send. A controller replaced between the send and keeping it
|
||||||
|
// finds the gate pending, reads the machine's last send, and neither sends again nor leaves the machine unjudged.
|
||||||
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
|
func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) {
|
||||||
r := p.Release
|
r := p.Release
|
||||||
if r.Settings != nil {
|
if r.Settings != nil {
|
||||||
return advanceSettingsWalk(ctx, open, p)
|
return advanceSettingsWalk(ctx, open, p)
|
||||||
}
|
}
|
||||||
|
ctx = withWalking(ctx, p.ID)
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
|
if r.Rest {
|
||||||
|
return advanceRest(ctx, open, p, now)
|
||||||
|
}
|
||||||
if r.Gate == nil {
|
if r.Gate == nil {
|
||||||
heard, err := releaseHeard(ctx, open)
|
heard, err := releaseHeard(ctx, open)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -724,7 +762,7 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
r.Next++
|
r.Next++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
moves, err := gatedMoves(ctx, open, node, nil)
|
||||||
if errors.Is(err, errWalkedElsewhere) {
|
if errors.Is(err, errWalkedElsewhere) {
|
||||||
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
||||||
changed := p.Note != note
|
changed := p.Note != note
|
||||||
@@ -732,23 +770,15 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
return changed, nil
|
return changed, nil
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, fmt.Errorf("sending %s what waits there: %w", node, err)
|
return false, fmt.Errorf("reading what waits on %s: %w", node, err)
|
||||||
}
|
}
|
||||||
if len(moves) == 0 {
|
if len(moves) == 0 {
|
||||||
r.Done = append(r.Done, node)
|
r.Done = append(r.Done, node)
|
||||||
r.Next++
|
r.Next++
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
r.Gate = &inventory.PlanGate{Machines: sent, Since: &now, Carried: moves, Sent: sentNow(ctx, open.inventory, sent),
|
r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true}
|
||||||
Grants: grants}
|
p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves))
|
||||||
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
|
||||||
if said := grantsSaid(grants); said != "" {
|
|
||||||
p.Note += "; " + said
|
|
||||||
}
|
|
||||||
if said := recreationsSaid(moves); said != "" {
|
|
||||||
p.Note += "; " + said
|
|
||||||
}
|
|
||||||
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
|
||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
|
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
|
||||||
@@ -759,6 +789,19 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
g := r.Gate
|
g := r.Gate
|
||||||
|
if g.Pending {
|
||||||
|
node := firstOf(g.Machines)
|
||||||
|
out, err := sendKept(ctx, open, p, node, g, now)
|
||||||
|
if err != nil || out == keptWaiting {
|
||||||
|
return out == keptWaiting, err
|
||||||
|
}
|
||||||
|
if out == keptNothing {
|
||||||
|
r.Gate = nil
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
r.Next++
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
@@ -777,6 +820,11 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
r.Done = append(r.Done, firstOf(g.Machines))
|
r.Done = append(r.Done, firstOf(g.Machines))
|
||||||
r.Next++
|
r.Next++
|
||||||
r.Gate = nil
|
r.Gate = nil
|
||||||
|
// **Then the rest at once** (novox/hq ADR 0353 §1): every other machine heard from, each judged by its own
|
||||||
|
// gate, in place of one machine at a time (ADR 0236 §22).
|
||||||
|
r.Rest = true
|
||||||
|
p.Note = fmt.Sprintf("passed on %s; the rest are sent at once, each judged", firstOf(g.Machines))
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
return true, nil
|
return true, nil
|
||||||
case inventory.GateSuperseded:
|
case inventory.GateSuperseded:
|
||||||
// Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what
|
// Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what
|
||||||
@@ -797,6 +845,453 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
|||||||
return true, nil
|
return true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// What sendKept did with a kept send.
|
||||||
|
const (
|
||||||
|
keptSent = iota // sent now
|
||||||
|
keptAlready // the machine was already sent it: judged from that send, nothing sent
|
||||||
|
keptNothing // nothing waits there any more: no gate
|
||||||
|
keptWaiting // another walk judges a build it would carry: it waits
|
||||||
|
)
|
||||||
|
|
||||||
|
// sendKept makes a send a gate kept before it was made (novox/hq ADR 0353 §8) — unless the machine's last send
|
||||||
|
// already carries what the gate is for, made before a controller was replaced and before it was kept: then
|
||||||
|
// nothing is sent, and the gate judges from that send (§6).
|
||||||
|
func sendKept(ctx context.Context, open *stores, p *inventory.Plan, node string, g *inventory.PlanGate,
|
||||||
|
now time.Time) (int, error) {
|
||||||
|
inv := open.inventory
|
||||||
|
moves, sent, grants, err := gatedSend(ctx, open, node, nil)
|
||||||
|
if errors.Is(err, errWalkedElsewhere) {
|
||||||
|
p.Note = fmt.Sprintf("waiting before %s: %v", node, err)
|
||||||
|
return keptWaiting, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return keptSent, fmt.Errorf("sending %s what waits there: %w", node, err)
|
||||||
|
}
|
||||||
|
if len(moves) > 0 {
|
||||||
|
g.Machines, g.Carried, g.Since, g.Grants, g.Pending = sent, moves, &now, grants, false
|
||||||
|
g.Sent = sentNow(ctx, inv, sent)
|
||||||
|
p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves))
|
||||||
|
if said := grantsSaid(grants); said != "" {
|
||||||
|
p.Note += "; " + said
|
||||||
|
}
|
||||||
|
if said := recreationsSaid(moves); said != "" {
|
||||||
|
p.Note += "; " + said
|
||||||
|
}
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return keptSent, nil
|
||||||
|
}
|
||||||
|
if !alreadySent(ctx, inv, node, g.Carried) {
|
||||||
|
return keptNothing, nil
|
||||||
|
}
|
||||||
|
since := now
|
||||||
|
if s, found, err := inv.SentTo(ctx, node); err == nil && found {
|
||||||
|
if !s.At.IsZero() {
|
||||||
|
since = s.At
|
||||||
|
}
|
||||||
|
g.Sent = map[string]inventory.SentDeclaration{node: s}
|
||||||
|
}
|
||||||
|
g.Since, g.Pending = &since, false
|
||||||
|
p.Note = fmt.Sprintf("%s was already sent the %d build(s) kept for it, at %s: judged from that send, nothing sent again",
|
||||||
|
node, len(g.Carried), since.Local().Format("15:04:05"))
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return keptAlready, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// alreadySent says a machine's last send carries every build of the moves: a send made and not kept.
|
||||||
|
func alreadySent(ctx context.Context, inv *inventory.Inventory, node string, moves []inventory.CarriedMove) bool {
|
||||||
|
if len(moves) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
sent, known, err := inv.SentBuilds(ctx, node)
|
||||||
|
if err != nil || !known {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, mv := range moves {
|
||||||
|
if !sameCommit(sent[mv.Module], mv.To) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// advanceRest takes one step of a release plan's rest (novox/hq ADR 0353): the sends kept for a step made, every
|
||||||
|
// machine of the step judged by its own gate, one that fails put back by itself while the others go on to their
|
||||||
|
// verdicts, and the next step — or the end — once every gate of the step has its verdict or is silent.
|
||||||
|
func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.Time) (bool, error) {
|
||||||
|
r := p.Release
|
||||||
|
// The sends kept for this step, in the plan's order: the control node, last in it, last (§1).
|
||||||
|
var pending []string
|
||||||
|
for _, n := range r.Order {
|
||||||
|
if g := r.Gates[n]; g != nil && g.Pending {
|
||||||
|
pending = append(pending, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(pending) > 0 {
|
||||||
|
// **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on
|
||||||
|
// a machine since its step was chosen could reach a second machine in it; the step is chosen again.
|
||||||
|
if grew, err := keptStepGrew(ctx, open, r, pending); err != nil {
|
||||||
|
return false, err
|
||||||
|
} else if grew != "" {
|
||||||
|
for _, n := range pending {
|
||||||
|
delete(r.Gates, n)
|
||||||
|
}
|
||||||
|
opened, waiting, err := openRestStep(ctx, open, p)
|
||||||
|
if err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
p.Note = "chosen again: " + grew
|
||||||
|
if !opened && waiting != "" {
|
||||||
|
p.Note += "; " + waiting
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
var sent, already, waiting []string
|
||||||
|
for _, node := range pending {
|
||||||
|
out, err := sendKept(ctx, open, p, node, r.Gates[node], now)
|
||||||
|
if err != nil {
|
||||||
|
return true, err
|
||||||
|
}
|
||||||
|
switch out {
|
||||||
|
case keptSent:
|
||||||
|
sent = append(sent, node)
|
||||||
|
case keptAlready:
|
||||||
|
already = append(already, node)
|
||||||
|
case keptNothing:
|
||||||
|
delete(r.Gates, node)
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
case keptWaiting:
|
||||||
|
waiting = append(waiting, node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var said []string
|
||||||
|
if len(sent) > 0 {
|
||||||
|
said = append(said, "sent "+strings.Join(sent, ", ")+" at once, each judged there")
|
||||||
|
}
|
||||||
|
if len(already) > 0 {
|
||||||
|
said = append(said, strings.Join(already, ", ")+" already sent it: judged from that send")
|
||||||
|
}
|
||||||
|
if len(waiting) > 0 {
|
||||||
|
said = append(said, "waiting before "+strings.Join(waiting, ", ")+": another walk judges a build it carries")
|
||||||
|
}
|
||||||
|
if len(said) > 0 {
|
||||||
|
p.Note = strings.Join(said, "; ")
|
||||||
|
}
|
||||||
|
return len(sent)+len(already) > 0 || len(waiting) < len(pending), nil
|
||||||
|
}
|
||||||
|
reports, err := open.inventory.LastReports(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
byNode := map[string]inventory.Reported{}
|
||||||
|
for _, rep := range reports {
|
||||||
|
byNode[rep.Node] = rep
|
||||||
|
}
|
||||||
|
changed, noteWas := false, p.Note
|
||||||
|
var notes, judging []string
|
||||||
|
batched, back := batchingRollbacks(ctx)
|
||||||
|
for _, node := range r.Order {
|
||||||
|
g := r.Gates[node]
|
||||||
|
if g == nil || g.Verdict != "" || g.Silent {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// What another machine's failure already put back is not this gate's to judge (ADR 0353 §3).
|
||||||
|
if len(g.Carried) > 0 && len(g.Returned) >= len(g.Carried) {
|
||||||
|
decide(g, inventory.GateFailed, "what it was sent was put back with another machine's failure", now)
|
||||||
|
r.Failed = append(r.Failed, node)
|
||||||
|
changed = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged
|
||||||
|
// until it reports, and past ApplySilentAfter it is listed and left.
|
||||||
|
if !reportedOnSend(byNode[node], node, g) {
|
||||||
|
if now.Sub(*g.Since) > inventory.ApplySilentAfter {
|
||||||
|
g.Silent = true
|
||||||
|
r.Silent = append(r.Silent, node)
|
||||||
|
changed = true
|
||||||
|
notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node,
|
||||||
|
inventory.ApplySilentAfter))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
judging = append(judging, node+": not yet reported on its send")
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||||
|
if err != nil {
|
||||||
|
return changed, err
|
||||||
|
}
|
||||||
|
switch verdict {
|
||||||
|
case "":
|
||||||
|
if putBackBroken(ctx, open, p, g, "", nil) {
|
||||||
|
changed = true
|
||||||
|
notes = append(notes, p.Note)
|
||||||
|
}
|
||||||
|
judging = append(judging, node+": "+gateLine(g))
|
||||||
|
case inventory.GatePassed:
|
||||||
|
passCarried(ctx, open, p, g, "")
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
changed = true
|
||||||
|
case inventory.GateSuperseded:
|
||||||
|
// Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on
|
||||||
|
// it; what still waits is released again.
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
changed = true
|
||||||
|
notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why))
|
||||||
|
default:
|
||||||
|
// **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other
|
||||||
|
// machine of the step goes on to its own verdict.
|
||||||
|
stateWas := p.State
|
||||||
|
p.Note = ""
|
||||||
|
failCarried(batched, open, p, g, "")
|
||||||
|
p.State = stateWas
|
||||||
|
r.Failed = append(r.Failed, node)
|
||||||
|
changed = true
|
||||||
|
notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note))
|
||||||
|
returnedElsewhere(ctx, open, r, node, g)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.Note = ""
|
||||||
|
sendRollbacks(ctx, open, p, back)
|
||||||
|
if p.Note != "" {
|
||||||
|
notes = append(notes, strings.TrimPrefix(p.Note, "; "))
|
||||||
|
}
|
||||||
|
if len(judging) > 0 {
|
||||||
|
note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ")
|
||||||
|
p.Note = note
|
||||||
|
return changed || noteWas != note, nil
|
||||||
|
}
|
||||||
|
// Every gate of the step has its verdict or is silent: the next step, unless a machine failed (§3).
|
||||||
|
if len(r.Failed) == 0 {
|
||||||
|
opened, waiting, err := openRestStep(ctx, open, p)
|
||||||
|
if err != nil {
|
||||||
|
return changed, err
|
||||||
|
}
|
||||||
|
if opened {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
if waiting != "" {
|
||||||
|
note := strings.Join(append(notes, waiting), "; ")
|
||||||
|
p.Note = note
|
||||||
|
return changed || noteWas != note, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.Tier = len(p.Tiers)
|
||||||
|
end := fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
|
||||||
|
if len(r.Silent) > 0 {
|
||||||
|
end += "; silent on their send, left: " + strings.Join(r.Silent, ", ")
|
||||||
|
}
|
||||||
|
if len(r.Skipped) > 0 {
|
||||||
|
end += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
|
||||||
|
}
|
||||||
|
p.State = inventory.PlanDone
|
||||||
|
if len(r.Failed) > 0 {
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end
|
||||||
|
} else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) {
|
||||||
|
// **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it.
|
||||||
|
p.State = inventory.PlanFailed
|
||||||
|
end = "no machine of the rest reported on its send; " + end
|
||||||
|
}
|
||||||
|
p.Note = strings.Join(append(notes, end), "; ")
|
||||||
|
fmt.Printf("%s: %s\n", p.ID, p.Note)
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with;
|
||||||
|
// empty when none would.
|
||||||
|
func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) {
|
||||||
|
f, err := readMoveFacts(ctx, open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
for _, node := range pending {
|
||||||
|
moves, err := gatedMoves(ctx, open, node, nil)
|
||||||
|
if errors.Is(err, errWalkedElsewhere) {
|
||||||
|
continue // sendKept says it waits
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
kept := r.Gates[node].Carried
|
||||||
|
for _, mv := range moves {
|
||||||
|
// The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9).
|
||||||
|
if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
|
||||||
|
return c.Module == mv.Module && sameCommit(c.To, mv.To)
|
||||||
|
}) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the
|
||||||
|
// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last.
|
||||||
|
func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool {
|
||||||
|
if r.At == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if s, kept := g.Sent[node]; kept {
|
||||||
|
return s.ReportsOn(r)
|
||||||
|
}
|
||||||
|
return r.Current
|
||||||
|
}
|
||||||
|
|
||||||
|
// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's
|
||||||
|
// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3).
|
||||||
|
func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) {
|
||||||
|
for _, c := range g.Carried {
|
||||||
|
if c.Build == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for node, other := range r.Gates {
|
||||||
|
if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, oc := range other.Carried {
|
||||||
|
if oc.Module == c.Module && sameCommit(oc.To, c.To) {
|
||||||
|
other.Returned = append(other.Returned, c.Module)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2):
|
||||||
|
// every machine left that is heard from and has something waiting, except that a build no gate has passed goes
|
||||||
|
// to one machine of the step only. Answers whether a step was kept, or why the walk waits.
|
||||||
|
func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, string, error) {
|
||||||
|
r := p.Release
|
||||||
|
heard, err := releaseHeard(ctx, open)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
f, err := readMoveFacts(ctx, open.inventory)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
waiting := map[string][]inventory.CarriedMove{}
|
||||||
|
var nodes, walked []string
|
||||||
|
for _, node := range r.Order[min(r.Next, len(r.Order)):] {
|
||||||
|
if r.Gates[node] != nil || slices.Contains(r.Done, node) || slices.Contains(r.Skipped, node) ||
|
||||||
|
slices.Contains(r.Failed, node) || slices.Contains(r.Silent, node) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !heard[node] {
|
||||||
|
r.Skipped = append(r.Skipped, node)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
moves, err := gatedMoves(ctx, open, node, nil)
|
||||||
|
if errors.Is(err, errWalkedElsewhere) {
|
||||||
|
walked = append(walked, fmt.Sprintf("%s (%v)", node, err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, "", fmt.Errorf("reading what waits on %s: %w", node, err)
|
||||||
|
}
|
||||||
|
if len(moves) == 0 {
|
||||||
|
r.Done = append(r.Done, node)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
waiting[node] = moves
|
||||||
|
nodes = append(nodes, node)
|
||||||
|
}
|
||||||
|
shelf, err := open.inventory.Catalogue(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, "", err
|
||||||
|
}
|
||||||
|
step := restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) })
|
||||||
|
if len(step) == 0 {
|
||||||
|
if len(walked) > 0 {
|
||||||
|
return false, "waiting before " + strings.Join(walked, "; "), nil
|
||||||
|
}
|
||||||
|
return false, "", nil
|
||||||
|
}
|
||||||
|
if r.Gates == nil {
|
||||||
|
r.Gates = map[string]*inventory.PlanGate{}
|
||||||
|
}
|
||||||
|
for _, node := range step {
|
||||||
|
r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true}
|
||||||
|
}
|
||||||
|
p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", "))
|
||||||
|
if left := len(nodes) - len(step); left > 0 {
|
||||||
|
p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left)
|
||||||
|
}
|
||||||
|
return true, "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the
|
||||||
|
// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus.
|
||||||
|
func oneAtATime(module string, m catalogue.Manifest) bool {
|
||||||
|
if module == hostModule {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat })
|
||||||
|
}
|
||||||
|
|
||||||
|
// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless
|
||||||
|
// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's
|
||||||
|
// first machine. Pure.
|
||||||
|
//
|
||||||
|
// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a
|
||||||
|
// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus
|
||||||
|
// shows only as its silence and no rollback reaches it.
|
||||||
|
func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool,
|
||||||
|
alone func(module string) bool) []string {
|
||||||
|
first := map[string]string{}
|
||||||
|
var out []string
|
||||||
|
for _, n := range nodes {
|
||||||
|
var unpassed []string
|
||||||
|
for _, mv := range waiting[n] {
|
||||||
|
if !passed(mv.Module, mv.To) || alone(mv.Module) {
|
||||||
|
unpassed = append(unpassed, mv.Module+"@"+mv.To)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, k := range unpassed {
|
||||||
|
first[k] = n
|
||||||
|
}
|
||||||
|
out = append(out, n)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
|
||||||
|
// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
|
||||||
|
func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) {
|
||||||
|
recent, err := inv.RecentPlans(ctx, 20)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for _, p := range recent {
|
||||||
|
if p.Release == nil || p.Release.Settings != nil || p.Open() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(p.Release.Silent) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
machines := append([]string(nil), p.Release.Silent...)
|
||||||
|
sort.Strings(machines)
|
||||||
|
o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent",
|
||||||
|
Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines,
|
||||||
|
Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+
|
||||||
|
"left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)}
|
||||||
|
w := releaseSilentWords(machines)
|
||||||
|
o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs
|
||||||
|
return []conditions.Observation{o}, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
// backlogObservation is what the gate's probe says of a release held after a failure.
|
// backlogObservation is what the gate's probe says of a release held after a failure.
|
||||||
func backlogObservation() []conditions.Observation {
|
func backlogObservation() []conditions.Observation {
|
||||||
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
|
if backlogNow.held == "" || len(backlogNow.waiting) == 0 {
|
||||||
|
|||||||
Reference in New Issue
Block a user