Keep the first-machine gate behind a switch for walks begun before hq ADR 0360, and test both paths
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate head 5b455b8a2438 merged into trunk 76304d9db817 — pass: builds mesh-controller → novox; no bus step; every machine composes with the chang…
mesh/repo-check head 5b455b8a2438 merged into trunk 76304d9db817 — fail: its merge-check.sh failed: --- FAIL: TestARenewalWrittenThoughItsAnswerWasLostIsSt…
mesh/delivery superseded: a newer head of the same pull request
mesh/merge-gate head 5b455b8a2438 merged into trunk 76304d9db817 — pass: builds mesh-controller → novox; no bus step; every machine composes with the chang…
mesh/repo-check head 5b455b8a2438 merged into trunk 76304d9db817 — fail: its merge-check.sh failed: --- FAIL: TestARenewalWrittenThoughItsAnswerWasLostIsSt…
This commit is contained in:
@@ -125,6 +125,7 @@ func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) {
|
||||
// build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and
|
||||
// is named in testdata/store-tests.
|
||||
func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
@@ -22,6 +22,7 @@ func foundDirectory(module string, since time.Time) inventory.ResourceHealth {
|
||||
}
|
||||
|
||||
func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
now := time.Now()
|
||||
sent := now.Add(-time.Minute)
|
||||
f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now,
|
||||
@@ -48,6 +49,7 @@ func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) {
|
||||
// The whole walk: a module whose directory was used as found long before still gets its fix to every machine,
|
||||
// its pass kept and the wait said; a directory this very send found holds it and puts it back.
|
||||
func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
found time.Duration // when the directory was found, against now
|
||||
@@ -116,6 +118,7 @@ func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) {
|
||||
|
||||
// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over.
|
||||
func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
k, _ := withConditionsInMemory(t)
|
||||
ctx := t.Context()
|
||||
rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}}
|
||||
|
||||
@@ -57,6 +57,7 @@ func healthyContainer(module string) inventory.ResourceHealth {
|
||||
// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass,
|
||||
// counted over the gate's spacing — before it says its own. The broken one alone is put back.
|
||||
func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -99,6 +100,7 @@ const broker_runtime = "node-tools"
|
||||
// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick
|
||||
// while a module beside it is not yet healthy counts the healthy one once.
|
||||
func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
withGateBounds(t, time.Hour, time.Hour, time.Hour)
|
||||
@@ -123,6 +125,7 @@ func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) {
|
||||
|
||||
// **A carried build's pass carries its own wait, never another's** (review (e)).
|
||||
func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
@@ -147,6 +150,7 @@ func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) {
|
||||
// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the
|
||||
// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it.
|
||||
func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
@@ -227,6 +231,7 @@ func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that
|
||||
// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent.
|
||||
func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
@@ -289,6 +294,7 @@ func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) {
|
||||
// tools are marked and put back in the very judging that found them broken — their registered build is the
|
||||
// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass.
|
||||
func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -431,6 +437,7 @@ func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testi
|
||||
// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous
|
||||
// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group.
|
||||
func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -36,6 +36,10 @@ func aGateMesh(t *testing.T) *gateMesh {
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
g := &gateMesh{open: open, health: map[string]health{}}
|
||||
// The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are.
|
||||
wasGated := gateFirstMachine
|
||||
gateFirstMachine = true
|
||||
t.Cleanup(func() { gateFirstMachine = wasGated })
|
||||
g.keeper, _ = withConditionsInMemory(t)
|
||||
g.told = &conditions.Told{}
|
||||
was := doctorFrom
|
||||
@@ -271,6 +275,34 @@ func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health
|
||||
// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent.
|
||||
func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) {
|
||||
g := aGateMesh(t)
|
||||
gateFirstMachine = false
|
||||
ctx := t.Context()
|
||||
judged := false
|
||||
wasGather := gatherGateFacts
|
||||
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
||||
judged = true
|
||||
return wasGather(ctx, open, component)
|
||||
}
|
||||
t.Cleanup(func() { gatherGateFacts = wasGather })
|
||||
for i := 0; i < 6; i++ {
|
||||
advancePlans(ctx, g.open)
|
||||
}
|
||||
p := g.plan(t)
|
||||
if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) {
|
||||
t.Fatalf("sent %v, not both machines in one send", g.sent)
|
||||
}
|
||||
if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged {
|
||||
t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged)
|
||||
}
|
||||
if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 {
|
||||
t.Fatalf("a rolled build left a condition: %+v %v", obs, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the
|
||||
// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524).
|
||||
func TestALeafModulePassesOnOneHealthyReading(t *testing.T) {
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module
|
||||
// to another build supersedes the judging: no verdict, nothing put back.
|
||||
func TestReplay352(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
||||
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
||||
@@ -67,6 +68,7 @@ func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
||||
|
||||
// A plan's own first send waits while a release judges the same module on that machine with another build.
|
||||
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
@@ -82,6 +84,7 @@ func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
||||
|
||||
// A merge plan's judging is superseded the same way: another send moved its module on the first machine.
|
||||
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -140,6 +143,7 @@ func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) {
|
||||
// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept
|
||||
// without its send reads the report against the send made last, as before. Pure.
|
||||
func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490}
|
||||
for _, c := range []struct {
|
||||
r inventory.Reported
|
||||
@@ -185,6 +189,7 @@ func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) {
|
||||
// A gate judges only the build the machine was last sent: last sent another build of the module, the
|
||||
// judging is superseded, whatever the machine reports. Pure.
|
||||
func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
at := time.Now()
|
||||
f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied,
|
||||
At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{},
|
||||
@@ -212,6 +217,7 @@ func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) {
|
||||
// A move of another build of a module to a machine where a release or a plan is judging that module
|
||||
// waits for that judging; the same build to that machine is already there. Pure.
|
||||
func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
at := time.Now()
|
||||
release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{
|
||||
Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{
|
||||
@@ -249,6 +255,7 @@ func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) {
|
||||
// The controller is never put back to a build that reaches less of the store's schema than the store
|
||||
// has, or to one that never said what it reaches: the current build is kept, and the condition says so.
|
||||
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
@@ -345,6 +352,7 @@ func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T)
|
||||
|
||||
// The lease's holder names the build the declaration told it it is, and the version stamp only without one.
|
||||
func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
t.Setenv(RunningBuildVar, " ad62528c47c7 ")
|
||||
if h := holderOf("x"); h.Build != "ad62528c47c7" {
|
||||
t.Fatalf("the holder's build is %q", h.Build)
|
||||
|
||||
@@ -20,8 +20,8 @@ func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
|
||||
ctx := t.Context()
|
||||
inv := g.open.inventory
|
||||
|
||||
// app is the build machine, so a module of the next tier is built by it: the tier gates the next on app
|
||||
// running on its machines.
|
||||
// app holds the build seat, so a module of the next tier is built by it; since novox/hq ADR 0360 §3 that gates
|
||||
// nothing.
|
||||
for _, m := range []catalogue.Manifest{
|
||||
{Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}},
|
||||
{Module: "web", Version: "w1"},
|
||||
@@ -71,7 +71,8 @@ func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
|
||||
t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note)
|
||||
}
|
||||
|
||||
// A machine that has not answered its last send is still waited on: no move is not "never sent".
|
||||
// A machine that has not answered its last send is not waited on either: a built-by edge gates no tier
|
||||
// (novox/hq ADR 0360 §3). Before, the walk waited for app on laptop.
|
||||
p.Tier, p.State, p.Note = 0, inventory.PlanRolling, ""
|
||||
if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -81,7 +82,7 @@ func TestANoMoveTierWaitsForNoNewReport(t *testing.T) {
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
p = g.plan(t)
|
||||
if p.Tier != 0 || !strings.Contains(p.Note, "waiting for app on laptop") {
|
||||
t.Fatalf("a machine that has not answered its last send was not waited on: tier %d, %q", p.Tier, p.Note)
|
||||
if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") {
|
||||
t.Fatalf("the walk waits on a built-by edge: tier %d, %q", p.Tier, p.Note)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -39,3 +40,149 @@ func TestAWalkEndsOnItsMachinesReports(t *testing.T) {
|
||||
t.Fatalf("the walk still waits once every machine reported: %v", waiting)
|
||||
}
|
||||
}
|
||||
|
||||
// withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before
|
||||
// novox/hq ADR 0360 started.
|
||||
func withTheGate(t *testing.T) {
|
||||
t.Helper()
|
||||
was := gateFirstMachine
|
||||
gateFirstMachine = true
|
||||
t.Cleanup(func() { gateFirstMachine = was })
|
||||
}
|
||||
|
||||
// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7);
|
||||
// before, the first machine was judged before the rest were sent (ADR 0353 §1).
|
||||
func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
said := events(t)
|
||||
for i := 0; i < 12; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
for _, n := range []string{"desk", "laptop", "anchor"} {
|
||||
if !slices.Contains(*said, "send "+n) {
|
||||
t.Fatalf("%s was not sent: %v", n, *said)
|
||||
}
|
||||
}
|
||||
if slices.Contains(*said, "judge") {
|
||||
t.Fatalf("a machine was judged by its health: %v", *said)
|
||||
}
|
||||
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other
|
||||
// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3).
|
||||
func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
b.broken["desk"] = true // desk fails what it is sent
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") {
|
||||
t.Fatalf("the others have no verdict of their own: %+v", p.Release)
|
||||
}
|
||||
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" {
|
||||
t.Fatalf("desk was put back: %v", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
||||
// made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7).
|
||||
func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
p, err := releaseBacklog(ctx, b.open, "")
|
||||
if err != nil || p == nil {
|
||||
t.Fatalf("no release opened: %v", err)
|
||||
}
|
||||
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending {
|
||||
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The send is made, and the controller is replaced before it keeps it.
|
||||
if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 {
|
||||
t.Fatalf("anchor was sent again: %v", b.sent)
|
||||
}
|
||||
g := b.release(t).Release.Gates["anchor"]
|
||||
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
||||
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
||||
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7).
|
||||
func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) {
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
reporting := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing
|
||||
b.sent = append(b.sent, names)
|
||||
current, _ := open.inventory.CurrentBuilds(ctx)
|
||||
return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep",
|
||||
map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit})
|
||||
}
|
||||
return reporting(ctx, open, names)
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = reporting })
|
||||
for i := 0; i < 8; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
p := b.release(t)
|
||||
g := p.Release.Gates["laptop"]
|
||||
if !p.Open() || g == nil || g.Verdict != "" || g.Silent {
|
||||
t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g)
|
||||
}
|
||||
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
||||
g.Since = &since
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
p = b.release(t)
|
||||
// anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7).
|
||||
if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
||||
t.Fatal("a silent machine's build was marked failed")
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
||||
t.Fatalf("laptop was put back: %v", sent)
|
||||
}
|
||||
obs, err := silentObservation(ctx, inv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) {
|
||||
t.Fatalf("the silence is not said: %+v", obs)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,7 @@ func userUnit(module, account string) inventory.ResourceHealth {
|
||||
// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
|
||||
// before, so that a fault is never excused.
|
||||
func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
notInGroup := relogin("lights", "operator")
|
||||
notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
|
||||
systemUnit := userUnit("lights", "operator")
|
||||
@@ -74,6 +75,7 @@ func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
|
||||
// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module
|
||||
// with a container down beside it is not yet healthy, as before.
|
||||
func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
now := time.Now()
|
||||
since := now.Add(-time.Minute)
|
||||
account := relogin("lights", "operator")
|
||||
@@ -102,6 +104,7 @@ func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) {
|
||||
// The module's condition says the wait in one sentence for a person, and clears on the first statement that
|
||||
// no longer says it — said as the module's own fault when its unit still fails after the new login.
|
||||
func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
k, _ := withConditionsInMemory(t)
|
||||
@@ -158,6 +161,7 @@ func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) {
|
||||
// A module that is healthy on its own keeps its pass when another module of the same send fails at the
|
||||
// bound: it is neither put back nor left without a verdict for every other walk to wait on.
|
||||
func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
@@ -207,6 +211,7 @@ func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) {
|
||||
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
|
||||
// no button, since nothing but their own new login can do it.
|
||||
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
|
||||
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
|
||||
plainExample(t, o, "openrazer waits for a new login on g14",
|
||||
@@ -238,6 +243,7 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
|
||||
// When a module not working turns into one waiting for a new login, and back, the clearing line says what it
|
||||
// became, never that it works again (issue 318 review).
|
||||
func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
store := conditions.NewInMemory()
|
||||
@@ -286,6 +292,7 @@ func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) {
|
||||
|
||||
// Which moves add an account group, read from the builds' manifests (issue 318 review).
|
||||
func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
user := func(groups ...any) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user",
|
||||
"name": "operator", "groups": groups}}}
|
||||
|
||||
@@ -133,7 +133,7 @@ func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) {
|
||||
{"the route proxy's program: the route proxy alone", "mesh-controller",
|
||||
[]string{"examples/route-proxy/main.go"}, "route-proxy", true},
|
||||
{"a package all three build from: all three", "mesh-controller",
|
||||
[]string{"internal/broker/broker.go"}, "mesh-controller | build-agent | route-proxy", false},
|
||||
[]string{"internal/broker/broker.go"}, "mesh-controller,route-proxy | build-agent", false},
|
||||
{"the route proxy's directory in the catalogue: it alone", "mesh-catalog",
|
||||
[]string{"modules/route-proxy/module.json"}, "route-proxy", false},
|
||||
{"the build agent's directory: it alone, nothing it builds", "mesh-catalog",
|
||||
|
||||
@@ -616,6 +616,7 @@ func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) {
|
||||
// A plan that stopped at its first machine is retried: the module sent to that machine again, the
|
||||
// send recorded as the first anew, and the plan goes on — unless a newer plan holds the module.
|
||||
func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
asked := asksRecorded(t)
|
||||
|
||||
+220
-11
@@ -750,11 +750,49 @@ func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool,
|
||||
if r.Rest {
|
||||
return advanceRest(ctx, open, p, now)
|
||||
}
|
||||
if r.Gate == nil {
|
||||
if r.Gate == nil && !gateFirstMachine {
|
||||
// **No first machine** (novox/hq ADR 0360 §7): every machine heard from is sent at once, in one step.
|
||||
r.Rest = true
|
||||
return advanceRest(ctx, open, p, now)
|
||||
}
|
||||
if r.Gate == nil {
|
||||
heard, err := releaseHeard(ctx, open)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for r.Next < len(r.Order) {
|
||||
node := r.Order[r.Next]
|
||||
if !heard[node] {
|
||||
r.Skipped = append(r.Skipped, node)
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
moves, err := gatedMoves(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
note := fmt.Sprintf("waiting before %s: %v", node, err)
|
||||
changed := p.Note != note
|
||||
p.Note = note
|
||||
return changed, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("reading what waits on %s: %w", node, err)
|
||||
}
|
||||
if len(moves) == 0 {
|
||||
r.Done = append(r.Done, node)
|
||||
r.Next++
|
||||
continue
|
||||
}
|
||||
r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true}
|
||||
p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves))
|
||||
return true, nil
|
||||
}
|
||||
p.State, p.Tier = inventory.PlanDone, len(p.Tiers)
|
||||
p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", ")))
|
||||
if len(r.Skipped) > 0 {
|
||||
p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ")
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
g := r.Gate
|
||||
if g.Pending {
|
||||
node := firstOf(g.Machines)
|
||||
@@ -894,6 +932,24 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
|
||||
}
|
||||
}
|
||||
if len(pending) > 0 {
|
||||
// **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on
|
||||
// a machine since its step was chosen could reach a second machine in it; the step is chosen again.
|
||||
if grew, err := keptStepGrew(ctx, open, r, pending); err != nil {
|
||||
return false, err
|
||||
} else if grew != "" {
|
||||
for _, n := range pending {
|
||||
delete(r.Gates, n)
|
||||
}
|
||||
opened, waiting, err := openRestStep(ctx, open, p)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
p.Note = "chosen again: " + grew
|
||||
if !opened && waiting != "" {
|
||||
p.Note += "; " + waiting
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
var sent, already, waiting []string
|
||||
for _, node := range pending {
|
||||
out, err := sendKept(ctx, open, p, node, r.Gates[node], now)
|
||||
@@ -937,6 +993,7 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
|
||||
}
|
||||
changed, noteWas := false, p.Note
|
||||
var notes, judging []string
|
||||
batched, back := batchingRollbacks(ctx)
|
||||
for _, node := range r.Order {
|
||||
g := r.Gates[node]
|
||||
if g == nil || g.Verdict != "" || g.Silent {
|
||||
@@ -963,23 +1020,64 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
|
||||
judging = append(judging, node+": not yet reported on its send")
|
||||
continue
|
||||
}
|
||||
// **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and
|
||||
// fails the walk, with nothing put back — apply-failing says it.
|
||||
switch rep := byNode[node]; rep.Outcome {
|
||||
case inventory.OutcomeApplied:
|
||||
decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now)
|
||||
if !gateFirstMachine {
|
||||
// **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and
|
||||
// fails the walk, with nothing put back — apply-failing says it.
|
||||
switch rep := byNode[node]; rep.Outcome {
|
||||
case inventory.OutcomeApplied:
|
||||
decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now)
|
||||
passCarried(ctx, open, p, g, "")
|
||||
r.Done = append(r.Done, node)
|
||||
changed = true
|
||||
case inventory.OutcomeFailed, inventory.OutcomeRefused:
|
||||
decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now)
|
||||
r.Failed = append(r.Failed, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome))
|
||||
default:
|
||||
judging = append(judging, node+": not yet reported on its send")
|
||||
}
|
||||
continue
|
||||
}
|
||||
verdict, err := judgeMoves(ctx, open, g, nil, now)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
switch verdict {
|
||||
case "":
|
||||
if putBackBroken(ctx, open, p, g, "", nil) {
|
||||
changed = true
|
||||
notes = append(notes, p.Note)
|
||||
}
|
||||
judging = append(judging, node+": "+gateLine(g))
|
||||
case inventory.GatePassed:
|
||||
passCarried(ctx, open, p, g, "")
|
||||
r.Done = append(r.Done, node)
|
||||
changed = true
|
||||
case inventory.OutcomeFailed, inventory.OutcomeRefused:
|
||||
decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, rep.Outcome), now)
|
||||
case inventory.GateSuperseded:
|
||||
// Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on
|
||||
// it; what still waits is released again.
|
||||
r.Done = append(r.Done, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why))
|
||||
default:
|
||||
// **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other
|
||||
// machine of the step goes on to its own verdict.
|
||||
stateWas := p.State
|
||||
p.Note = ""
|
||||
failCarried(batched, open, p, g, "")
|
||||
p.State = stateWas
|
||||
r.Failed = append(r.Failed, node)
|
||||
changed = true
|
||||
notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, rep.Outcome))
|
||||
default:
|
||||
judging = append(judging, node+": not yet reported on its send")
|
||||
notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note))
|
||||
returnedElsewhere(ctx, open, r, node, g)
|
||||
}
|
||||
}
|
||||
p.Note = ""
|
||||
sendRollbacks(ctx, open, p, back)
|
||||
if p.Note != "" {
|
||||
notes = append(notes, strings.TrimPrefix(p.Note, "; "))
|
||||
}
|
||||
if len(judging) > 0 {
|
||||
note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ")
|
||||
p.Note = note
|
||||
@@ -1022,6 +1120,42 @@ func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with;
|
||||
// empty when none would.
|
||||
func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) {
|
||||
if !gateFirstMachine {
|
||||
return "", nil // every machine goes at once (novox/hq ADR 0360 §7): a send carries what waits when it is made
|
||||
}
|
||||
f, err := readMoveFacts(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, node := range pending {
|
||||
moves, err := gatedMoves(ctx, open, node, nil)
|
||||
if errors.Is(err, errWalkedElsewhere) {
|
||||
continue // sendKept says it waits
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
kept := r.Gates[node].Carried
|
||||
for _, mv := range moves {
|
||||
// The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9).
|
||||
if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool {
|
||||
return c.Module == mv.Module && sameCommit(c.To, mv.To)
|
||||
}) {
|
||||
continue
|
||||
}
|
||||
return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil
|
||||
}
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the
|
||||
// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last.
|
||||
func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool {
|
||||
@@ -1034,6 +1168,29 @@ func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bo
|
||||
return r.Current
|
||||
}
|
||||
|
||||
// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's
|
||||
// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3).
|
||||
func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) {
|
||||
for _, c := range g.Carried {
|
||||
if c.Build == "" {
|
||||
continue
|
||||
}
|
||||
if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked {
|
||||
continue
|
||||
}
|
||||
for node, other := range r.Gates {
|
||||
if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) {
|
||||
continue
|
||||
}
|
||||
for _, oc := range other.Carried {
|
||||
if oc.Module == c.Module && sameCommit(oc.To, c.To) {
|
||||
other.Returned = append(other.Returned, c.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2):
|
||||
// every machine left that is heard from and has something waiting, except that a build no gate has passed goes
|
||||
// to one machine of the step only. Answers whether a step was kept, or why the walk waits.
|
||||
@@ -1043,6 +1200,10 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
f, err := readMoveFacts(ctx, open.inventory)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
waiting := map[string][]inventory.CarriedMove{}
|
||||
var nodes, walked []string
|
||||
for _, node := range r.Order[min(r.Next, len(r.Order)):] {
|
||||
@@ -1069,8 +1230,15 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s
|
||||
waiting[node] = moves
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
shelf, err := open.inventory.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return false, "", err
|
||||
}
|
||||
// **Every machine at once** (novox/hq ADR 0360 §2, §7): no build waits for one machine to pass first.
|
||||
step := nodes
|
||||
if gateFirstMachine {
|
||||
step = restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) })
|
||||
}
|
||||
if len(step) == 0 {
|
||||
if len(walked) > 0 {
|
||||
return false, "waiting before " + strings.Join(walked, "; "), nil
|
||||
@@ -1084,9 +1252,50 @@ func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, s
|
||||
r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true}
|
||||
}
|
||||
p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", "))
|
||||
if left := len(nodes) - len(step); left > 0 {
|
||||
p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left)
|
||||
}
|
||||
return true, "", nil
|
||||
}
|
||||
|
||||
// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the
|
||||
// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus.
|
||||
func oneAtATime(module string, m catalogue.Manifest) bool {
|
||||
if module == hostModule {
|
||||
return true
|
||||
}
|
||||
return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat })
|
||||
}
|
||||
|
||||
// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless
|
||||
// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's
|
||||
// first machine. Pure.
|
||||
//
|
||||
// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a
|
||||
// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus
|
||||
// shows only as its silence and no rollback reaches it.
|
||||
func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool,
|
||||
alone func(module string) bool) []string {
|
||||
first := map[string]string{}
|
||||
var out []string
|
||||
for _, n := range nodes {
|
||||
var unpassed []string
|
||||
for _, mv := range waiting[n] {
|
||||
if !passed(mv.Module, mv.To) || alone(mv.Module) {
|
||||
unpassed = append(unpassed, mv.Module+"@"+mv.To)
|
||||
}
|
||||
}
|
||||
if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) {
|
||||
continue
|
||||
}
|
||||
for _, k := range unpassed {
|
||||
first[k] = n
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353
|
||||
// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing.
|
||||
func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) {
|
||||
|
||||
@@ -27,6 +27,10 @@ import (
|
||||
// alone cannot (a tier waiting for machines to report); a controller replaced mid-plan finds the
|
||||
// plan where it left it.
|
||||
|
||||
// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since
|
||||
// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on.
|
||||
var gateFirstMachine = false
|
||||
|
||||
// planWaitBound is how long a plan may wait on one thing before `status` names it red.
|
||||
const planWaitBound = 30 * time.Minute
|
||||
|
||||
@@ -846,8 +850,11 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan,
|
||||
return false, err
|
||||
}
|
||||
// **Every machine at once** (novox/hq ADR 0360 §1, §2): a module that rolls out is sent to every machine
|
||||
// running it in one step, with no first machine and no gate, whatever module it is.
|
||||
policy.Together = true
|
||||
// running it in one step, with no first machine and no gate, whatever module it is. A module a controller
|
||||
// before ADR 0360 already sent to its first machine finishes under its gate.
|
||||
if !gateFirstMachine && state.FirstAt == nil {
|
||||
policy.Together = true
|
||||
}
|
||||
var reports []inventory.Reported
|
||||
if !policy.Together {
|
||||
// Read for the choice of the first machine as well as for its report.
|
||||
|
||||
@@ -169,6 +169,7 @@ func (b *backlogMesh) release(t *testing.T) inventory.Plan {
|
||||
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
||||
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
||||
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
@@ -224,6 +225,7 @@ func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
||||
// A release that fails on its first machine puts back what it carried there, goes no further, and the
|
||||
// next one waits for a person, said as a condition; a person releases it with why.
|
||||
func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence
|
||||
// for the operator, that a new login is needed on the laptop.
|
||||
func TestReplay318(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
// at its gate with the pass's own words, and put it back. Written with only what the controller had before
|
||||
// its fix, so it is laid over the commit before.
|
||||
func TestReplay335(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
t.Run("the first machine's later reports", testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports)
|
||||
t.Run("a walk stopped after the pass", testAWalkStoppedAfterItsGatePassedKeepsThePass)
|
||||
}
|
||||
@@ -77,6 +78,7 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) {
|
||||
// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the
|
||||
// carried module's step said the first machine never applied it, and the passed build was put back.
|
||||
func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
tm := aTierMesh(t, "m01", "m02")
|
||||
ctx := t.Context()
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
// had before its fix — the stores, register, assign, a plan saved and advanced, the watchdogs' gathering and
|
||||
// seeing — so it is laid over the commit before.
|
||||
func TestReplay336(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -139,6 +139,7 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) {
|
||||
// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is
|
||||
// written only with what the controller had before the judging, for the prover to lay over that commit.
|
||||
func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq ADR 0353: after its first machine passes, a build goes to every other machine at once, each judged by
|
||||
// novox/hq ADR 0353, kept behind gateFirstMachine for walks a controller before ADR 0360 started: after its first machine passes, a build goes to every other machine at once, each judged by
|
||||
// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them —
|
||||
// a machine already sent what a send is for is not sent again, and its judging goes on from what was read.
|
||||
//
|
||||
@@ -59,13 +59,15 @@ func events(t *testing.T) *[]string {
|
||||
return &said
|
||||
}
|
||||
|
||||
// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7);
|
||||
// before, the first machine was judged before the rest were sent (ADR 0353 §1).
|
||||
func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) {
|
||||
// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second
|
||||
// was judged to its verdict before the third was sent.
|
||||
func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
said := events(t)
|
||||
gateEvery = 0
|
||||
for i := 0; i < 12; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
@@ -73,27 +75,28 @@ func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) {
|
||||
if p.State != inventory.PlanDone || len(p.Release.Done) != 3 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
for _, n := range []string{"desk", "laptop", "anchor"} {
|
||||
if !slices.Contains(*said, "send "+n) {
|
||||
t.Fatalf("%s was not sent: %v", n, *said)
|
||||
}
|
||||
desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop")
|
||||
if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) {
|
||||
t.Fatalf("sends and judgings: %v", *said)
|
||||
}
|
||||
if slices.Contains(*said, "judge") {
|
||||
t.Fatalf("a machine was judged by its health: %v", *said)
|
||||
if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") {
|
||||
t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said)
|
||||
}
|
||||
if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other
|
||||
// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3).
|
||||
func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) {
|
||||
// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own
|
||||
// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent.
|
||||
func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
b.broken["desk"] = true // desk fails what it is sent
|
||||
gateEvery = 0
|
||||
b.broken["desk"] = true // desk fails what it is sent in the rest's step
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 12 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
@@ -102,20 +105,106 @@ func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) {
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") {
|
||||
t.Fatalf("the others have no verdict of their own: %+v", p.Release)
|
||||
if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) {
|
||||
t.Fatalf("laptop was never sent: %v", b.sent)
|
||||
}
|
||||
if !slices.Contains(p.Release.Done, "laptop") {
|
||||
t.Fatalf("laptop has no verdict of its own: %+v", p.Release)
|
||||
}
|
||||
// late waited on laptop alone and was healthy there: its pass is kept, whatever desk did.
|
||||
if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
||||
t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err)
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" {
|
||||
t.Fatalf("desk was put back: %v", sent)
|
||||
// app failed on desk: marked, and put back wherever it was sent (ADR 0236).
|
||||
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
|
||||
t.Fatal("app's build is not marked failed")
|
||||
}
|
||||
if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" {
|
||||
t.Fatalf("desk was not put back: %v", sent)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that
|
||||
// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading.
|
||||
func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
inv := b.open.inventory
|
||||
advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once
|
||||
release := b.release(t)
|
||||
judging := release.Release.Gate
|
||||
if judging == nil || judging.Pending || judging.Passes == 0 {
|
||||
t.Fatalf("the release is not judging anchor: %+v", judging)
|
||||
}
|
||||
built := time.Now().UTC()
|
||||
plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2",
|
||||
Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2",
|
||||
Build: "build-app-c2"}}}
|
||||
if err := inv.SavePlan(ctx, &plan); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if len(b.sent) != 1 {
|
||||
t.Fatalf("anchor was sent again what it runs: %v", b.sent)
|
||||
}
|
||||
p, err := inv.PlanByID(ctx, "plan-app")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
g := p.Modules["app"].Gate
|
||||
if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID {
|
||||
t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging)
|
||||
}
|
||||
if p.Modules["app"].Previous != "c1" {
|
||||
t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous)
|
||||
}
|
||||
}
|
||||
|
||||
// A walk's first send that was made and not kept — the controller replaced between the two — is not made again
|
||||
// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6).
|
||||
func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
// The old controller sends anchor app c2 and is replaced before it keeps the send.
|
||||
made := sendRollout
|
||||
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
if _, err := made(ctx, open, names); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, context.Canceled
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
sendRollout = made
|
||||
if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil {
|
||||
t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent)
|
||||
}
|
||||
last, _, err := g.open.inventory.SentTo(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, g.open)
|
||||
if len(g.sent) != 1 {
|
||||
t.Fatalf("the send was made again: %v", g.sent)
|
||||
}
|
||||
s := g.plan(t).Modules["app"]
|
||||
if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) {
|
||||
t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate)
|
||||
}
|
||||
// What anchor ran before was kept before the send, so a failure still puts it back (§5).
|
||||
if s.Previous != "c1" || s.Gate.From != "c1" {
|
||||
t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From)
|
||||
}
|
||||
}
|
||||
|
||||
// A release's send kept before it was made, and made, is judged by the controller that takes over without being
|
||||
// made again (ADR 0353 §8).
|
||||
func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
@@ -124,7 +213,7 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
|
||||
if err != nil || p == nil {
|
||||
t.Fatalf("no release opened: %v", err)
|
||||
}
|
||||
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending {
|
||||
if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending {
|
||||
t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, p); err != nil {
|
||||
@@ -135,18 +224,89 @@ func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 {
|
||||
if len(b.sent) != 1 {
|
||||
t.Fatalf("anchor was sent again: %v", b.sent)
|
||||
}
|
||||
g := b.release(t).Release.Gates["anchor"]
|
||||
g := b.release(t).Release.Gate
|
||||
if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 {
|
||||
t.Fatalf("anchor is not judged on the send made: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits
|
||||
// for the next step, and one whose every build passed goes now (ADR 0353 §2).
|
||||
func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) {
|
||||
withTheGate(t)
|
||||
waiting := map[string][]inventory.CarriedMove{
|
||||
"desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
||||
"laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}},
|
||||
"shanks": {{Module: "app", To: "c2"}},
|
||||
"novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}},
|
||||
}
|
||||
passed := func(module, commit string) bool { return module == "app" && commit == "c2" }
|
||||
never := func(string) bool { return false }
|
||||
step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never)
|
||||
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
||||
t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step)
|
||||
}
|
||||
if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) {
|
||||
t.Fatalf("a machine whose every build passed waits: %v", step)
|
||||
}
|
||||
}
|
||||
|
||||
// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step
|
||||
// carries either to one machine only, while what else passed goes everywhere at once.
|
||||
func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) {
|
||||
withTheGate(t)
|
||||
waiting := map[string][]inventory.CarriedMove{
|
||||
"desk": {{Module: hostModule, To: "h2"}},
|
||||
"laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}},
|
||||
"shanks": {{Module: "app", To: "c2"}},
|
||||
"novox": {{Module: "dns", To: "d2"}},
|
||||
"ace": {{Module: "dns", To: "d2"}},
|
||||
}
|
||||
resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}}
|
||||
alone := func(m string) bool {
|
||||
if m == "dns" {
|
||||
return oneAtATime(m, resolver)
|
||||
}
|
||||
return oneAtATime(m, catalogue.Manifest{Module: m})
|
||||
}
|
||||
step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting,
|
||||
func(string, string) bool { return true }, alone)
|
||||
if !slices.Equal(step, []string{"desk", "shanks", "novox"}) {
|
||||
t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step)
|
||||
}
|
||||
if oneAtATime("app", catalogue.Manifest{Module: "app"}) {
|
||||
t.Fatal("a leaf module is held to one machine at a time")
|
||||
}
|
||||
}
|
||||
|
||||
// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent.
|
||||
func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) {
|
||||
withTheGate(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
gateEvery = 0
|
||||
b.broken["anchor"] = true
|
||||
advancePlans(ctx, b.open)
|
||||
for i := 0; i < 6 && b.release(t).Open(); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
for _, s := range b.sent {
|
||||
if !slices.Equal(s, []string{"anchor"}) {
|
||||
t.Fatalf("sent %v: the rest after a failed first machine", b.sent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is
|
||||
// ordered after every other.
|
||||
func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklogOnThree(t)
|
||||
ctx := t.Context()
|
||||
@@ -155,6 +315,7 @@ func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
said := events(t)
|
||||
gateEvery = 0
|
||||
for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
@@ -170,6 +331,7 @@ func TestTheControlNodeIsSentLastInAStep(t *testing.T) {
|
||||
// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put
|
||||
// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4).
|
||||
func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
|
||||
withTheGate(t)
|
||||
judgeLeavesAsTheCore(t)
|
||||
b := aBacklog(t)
|
||||
ctx := t.Context()
|
||||
@@ -185,6 +347,7 @@ func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
|
||||
return reporting(ctx, open, names)
|
||||
}
|
||||
t.Cleanup(func() { sendRollout = reporting })
|
||||
gateEvery = 0
|
||||
for i := 0; i < 8; i++ {
|
||||
advancePlans(ctx, b.open)
|
||||
}
|
||||
@@ -200,8 +363,7 @@ func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) {
|
||||
}
|
||||
advancePlans(ctx, b.open)
|
||||
p = b.release(t)
|
||||
// anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7).
|
||||
if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
||||
if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 {
|
||||
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
||||
}
|
||||
if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed {
|
||||
|
||||
@@ -171,6 +171,7 @@ func (tm *tierMesh) raise(machine, kind, summary string) {
|
||||
// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one
|
||||
// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen.
|
||||
func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
var modules []string
|
||||
for i := 1; i <= 12; i++ {
|
||||
@@ -223,6 +224,7 @@ func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) {
|
||||
// settling, is the node tools' — inside their settle window, the gate's bound — and no failure of the
|
||||
// modules sent with them: at the bound only the node tools are put back.
|
||||
func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule)
|
||||
ctx := t.Context()
|
||||
inv := tm.open.inventory
|
||||
@@ -276,6 +278,7 @@ func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) {
|
||||
// A machine-level condition about anything else holds the machine back as a whole: everything the send
|
||||
// moved there fails together, once, with that reason — not one module for it.
|
||||
func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
tm := aTierMesh(t, "app1", "app2", "app3")
|
||||
ctx := t.Context()
|
||||
@@ -307,6 +310,7 @@ func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) {
|
||||
// The machine-level conditions, sorted: one naming a module is that module's; the core behind is the
|
||||
// core's when the send moved it and nobody's when it did not; anything else is the machine's as a whole.
|
||||
func TestTheConditionsAboutAMachineItself(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
since := time.Now().Add(-time.Minute)
|
||||
about := func(kind, id string) conditions.Condition {
|
||||
return conditions.Condition{Key: "machine." + id + "." + kind, Kind: kind, Raised: time.Now(),
|
||||
@@ -339,6 +343,7 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) {
|
||||
// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never
|
||||
// judged (the incident's "NOT put back: no build kept").
|
||||
func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
g := aGateMesh(t)
|
||||
ctx := t.Context()
|
||||
@@ -394,6 +399,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) {
|
||||
// them, they are registered back and marked in the judging that finds them broken, the plan goes on judging
|
||||
// the other module to its own pass, and only then fails.
|
||||
func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
judgeLeavesAsTheCore(t)
|
||||
for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} {
|
||||
t.Run("gate kept on "+order[0], func(t *testing.T) {
|
||||
@@ -454,6 +460,7 @@ func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) {
|
||||
// decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool
|
||||
// runner's rule: one reading, not before the node-engine's witness window for it has ended.
|
||||
func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) {
|
||||
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
||||
tm := aTierMesh(t, broker.RuntimeModule, "app1")
|
||||
ctx := t.Context()
|
||||
// One reading counted per advance, as every test with a gate left open does: a gate judged with no time
|
||||
|
||||
Reference in New Issue
Block a user