jochen
3c19e9ee93
Rebased on #305: Approve now sends the layer by a walk, so the summary's "takes it at its next push" was false. The summary says it is set and sent at once by a walk to where it reaches, one machine at a time, each judged; the rest of what Approve does (onApprove) leads the Details, beside every key whole.
569 lines
25 KiB
Go
569 lines
25 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/lease"
|
|
)
|
|
|
|
// novox/hq ADR 0344, issue 641: the operator's one Approve on a settings proposal is the act that makes it take
|
|
// effect. The warrant sets the layer and opens a settings walk, which sends it to the machines it reaches one at a
|
|
// time, judging the module on each; the ask says so before the Approve, and nothing names a push.
|
|
|
|
// The order: every machine running the module, or the one the layer names, the control node last.
|
|
func TestASettingsWalkGoesToTheMachinesTheLayerReachesTheControlNodeLast(t *testing.T) {
|
|
running := []string{"shanks", "novox", "g14", "ace"}
|
|
for _, c := range []struct {
|
|
node string
|
|
want []string
|
|
}{
|
|
{"", []string{"ace", "g14", "shanks", "novox"}},
|
|
{"g14", []string{"g14"}},
|
|
{"novox", []string{"novox"}},
|
|
{"elsewhere", nil},
|
|
} {
|
|
if got := settingsWalkOrder(c.node, running, []string{"novox"}); !reflect.DeepEqual(got, c.want) {
|
|
t.Errorf("a layer on %q goes to %v, want %v", c.node, got, c.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A settings walk sends a machine only when no other walk judges there.
|
|
func TestASettingsWalkWaitsWhileAnotherWalkJudgesOnTheMachine(t *testing.T) {
|
|
judging := &inventory.PlanGate{Machines: []string{"g14"}}
|
|
passed := &inventory.PlanGate{Machines: []string{"g14"}, Verdict: inventory.GatePassed}
|
|
plans := []inventory.Plan{
|
|
{ID: "settings-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: judging}},
|
|
{ID: "release-2", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: passed}},
|
|
{ID: "plan-3", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}},
|
|
{ID: "plan-4", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}},
|
|
}
|
|
if id := judgingIn(plans, "settings-1", "g14"); id != "plan-3" {
|
|
t.Errorf("on g14 %q judges, want plan-3 (itself, a passed gate and an ended walk are not judging)", id)
|
|
}
|
|
if id := judgingIn(plans[1:2], "x", "g14"); id != "" {
|
|
t.Errorf("a passed gate judges: %q", id)
|
|
}
|
|
if id := judgingIn(plans, "x", "shanks"); id != "" {
|
|
t.Errorf("on shanks %q judges", id)
|
|
}
|
|
}
|
|
|
|
// The ask says, before the Approve, that the layer is sent at once and judged, and that its end is told as every
|
|
// walk's is; neither the ask nor the proposer's answer names a push any more.
|
|
func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) {
|
|
for _, c := range []struct {
|
|
in proposeInput
|
|
where string
|
|
had bool
|
|
}{
|
|
{proposeInput{module: "mounts", node: "shanks", values: mountsSources}, "shanks", false},
|
|
{proposeInput{module: "notes", values: map[string]any{"x": "1"}}, "every machine running notes", true},
|
|
{proposeInput{module: "notes", node: "laptop", clear: true}, "laptop", true},
|
|
} {
|
|
r := newProposerRig(t)
|
|
if c.had {
|
|
r.before, r.had = map[string]any{"x": "0"}, true
|
|
}
|
|
words, err := r.pr.propose(context.Background(), c.in)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
q := r.askSent(t)
|
|
if err := q.Check(r.now); err != nil {
|
|
t.Fatalf("the ask is refused: %v", err)
|
|
}
|
|
// At a glance (ADR 0345): the summary says it is sent at once by a walk, one machine at a time; the rest of
|
|
// what Approve does is the Details', which the router notes after the answers.
|
|
if !strings.Contains(q.Explanation, "sent at once by a walk to "+c.where+", one machine at a time, each judged") ||
|
|
!strings.Contains(q.Details, "and sent at once to "+c.where+", one machine at a time, each judged, "+
|
|
"every build there kept; its end is told as every walk's is") {
|
|
t.Errorf("the ask does not say the layer is sent at once to %s:\n%s", c.where, q.Explanation)
|
|
}
|
|
approve := q.Options[0]
|
|
if approve.Label != "Approve" || !strings.Contains(approve.Does, "sent to "+c.where+" at once") {
|
|
t.Errorf("Approve does %q", approve.Does)
|
|
}
|
|
if !strings.Contains(words, "a walk sends it to "+c.where+" at once") {
|
|
t.Errorf("the proposer's answer:\n%s", words)
|
|
}
|
|
for _, said := range []string{q.Explanation, q.Whole, approve.Does, words} {
|
|
if strings.Contains(said, "push") {
|
|
t.Errorf("a push is still named, as if a second act were needed:\n%s", said)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- on the real stores -----------------------------------------------------------------------------
|
|
|
|
// The warrant sets the layer and opens the walk that sends it, naming the machines it reaches; a layer for a
|
|
// machine that does not run the module opens none and says so.
|
|
func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
register(t, open, catalogue.Manifest{Module: "notes", Version: "1",
|
|
Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644",
|
|
"content": "x = ${setting:x}\n"}}})
|
|
if _, err := assign(ctx, open, "laptop", "notes"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// A layer set by hand opens no walk: it still waits for a push (ADR 0344 §7).
|
|
if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"0"}`, "--node", "laptop"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 0 {
|
|
t.Fatalf("a layer set at the terminal opened %d walk(s)", len(plans))
|
|
}
|
|
set := setLayerIn(open)
|
|
now := time.Now()
|
|
values := map[string]any{"x": "1"}
|
|
p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now,
|
|
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
|
words, err := set(ctx, p, "s9", "approved by the operator (ask s9)")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") {
|
|
t.Errorf("the warrant's words: %q", words)
|
|
}
|
|
plans, err := open.inventory.OpenPlans(ctx)
|
|
if err != nil || len(plans) != 1 {
|
|
t.Fatalf("open walks: %+v %v", plans, err)
|
|
}
|
|
w := plans[0]
|
|
if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" ||
|
|
!reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" ||
|
|
!reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) {
|
|
t.Fatalf("the settings walk: %+v %+v", w, w.Release)
|
|
}
|
|
|
|
// A machine's layer where the module does not run: set, and nothing to send.
|
|
q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now,
|
|
Digest: layerDigest(values), BeforeDigest: layerDigest(nil)}
|
|
words, err = set(ctx, q, "s10", "approved by the operator (ask s10)")
|
|
if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") {
|
|
t.Fatalf("a layer nobody runs: %q %v", words, err)
|
|
}
|
|
if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 {
|
|
t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans))
|
|
}
|
|
|
|
// A walk that cannot be opened is the act's failure, said with the push that sends it by hand.
|
|
was := openSettingsWalk
|
|
openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) {
|
|
return "", fmt.Errorf("the plans are not to be had")
|
|
}
|
|
t.Cleanup(func() { openSettingsWalk = was })
|
|
again := map[string]any{"x": "2"}
|
|
byHand := map[string]any{"x": "0"}
|
|
r := settingsProposal{Module: "notes", Node: "laptop", Values: again, Before: byHand, HadLayer: true,
|
|
From: "g14/claude-code", At: now, Digest: layerDigest(again), BeforeDigest: layerDigest(byHand)}
|
|
if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+
|
|
"sends it could not be opened: the plans are not to be had — `push laptop` sends it") {
|
|
t.Errorf("an unopened walk: %v", err)
|
|
}
|
|
}
|
|
|
|
// walkMesh is a mesh where `app` runs c1 on anchor and laptop and nothing waits for a gate: every send applied at
|
|
// once, recorded, and judged from the reports, with a machine named broken failing what it was sent.
|
|
type walkMesh struct {
|
|
open *stores
|
|
sent [][]string
|
|
broken map[string]bool
|
|
// kept says, per send, whether it kept every build of its machine; moves is the data a machine's send would move.
|
|
kept []bool
|
|
moves map[string][]dataMove
|
|
heard map[string]bool
|
|
}
|
|
|
|
func aSettledMesh(t *testing.T) *walkMesh {
|
|
t.Helper()
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
w := &walkMesh{open: open, broken: map[string]bool{}, moves: map[string][]dataMove{},
|
|
heard: map[string]bool{"anchor": true, "laptop": true}}
|
|
withConditionsInMemory(t)
|
|
was := doctorFrom
|
|
doctorFrom = nil
|
|
t.Cleanup(func() { doctorFrom = was })
|
|
old := time.Now().Add(-2 * time.Hour)
|
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1",
|
|
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old,
|
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
|
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
if _, err := inv.Assign(ctx, n, "app"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
wasMoves, wasHeard, wasSend, wasGather, wasData := machineMoves, releaseHeard, sendRollout, gatherGateFacts, settingsDataMoves
|
|
settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { return w.moves[node], nil }
|
|
machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
|
modules, err := open.inventory.Assigned(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return f.moves(node, modules, sent, known, all), nil
|
|
}
|
|
releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return w.heard, nil }
|
|
n := 0
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
w.sent = append(w.sent, append([]string(nil), names...))
|
|
w.kept = append(w.kept, len(names) == 1 && settingsKeptOn(ctx, names[0]))
|
|
current, err := open.inventory.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, node := range names {
|
|
modules, _ := open.inventory.Assigned(ctx, node)
|
|
carried := map[string]string{}
|
|
for _, m := range modules {
|
|
carried[m] = current[m].Commit
|
|
}
|
|
n++
|
|
digest := fmt.Sprintf("d-%s-%d", node, n)
|
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
|
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return names, nil
|
|
}
|
|
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
|
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
|
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
|
reports, err := open.inventory.LastReports(ctx)
|
|
if err != nil {
|
|
return f, err
|
|
}
|
|
for _, r := range reports {
|
|
if w.broken[r.Node] {
|
|
r.Outcome = inventory.OutcomeFailed
|
|
}
|
|
f.reports[r.Node] = r
|
|
}
|
|
return f, nil
|
|
}
|
|
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
|
gateSettle, gateEvery = 0, time.Hour
|
|
t.Cleanup(func() {
|
|
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
|
settingsDataMoves = wasData
|
|
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
|
|
})
|
|
return w
|
|
}
|
|
|
|
func (w *walkMesh) walk(t *testing.T) inventory.Plan {
|
|
t.Helper()
|
|
plans, err := w.open.inventory.RecentPlans(t.Context(), 10)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, p := range plans {
|
|
if p.Release != nil && p.Release.Settings != nil {
|
|
return p
|
|
}
|
|
}
|
|
t.Fatal("no settings walk")
|
|
return inventory.Plan{}
|
|
}
|
|
|
|
// The walk sends the first machine alone and judges the module there before it sends the next, and ends
|
|
// delivered on both; nothing waited for a gate, so each send carried the layer alone.
|
|
func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
w := aSettledMesh(t)
|
|
ctx := t.Context()
|
|
words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"})
|
|
if err != nil || !strings.Contains(words, "sends it to anchor, laptop") {
|
|
t.Fatalf("%q %v", words, err)
|
|
}
|
|
advancePlans(ctx, w.open)
|
|
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
|
|
t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent)
|
|
}
|
|
p := w.walk(t)
|
|
if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") {
|
|
t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release)
|
|
}
|
|
gateEvery = 0
|
|
for i := 0; i < 6; i++ {
|
|
advancePlans(ctx, w.open)
|
|
}
|
|
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
|
t.Fatalf("sent %v", w.sent)
|
|
}
|
|
p = w.walk(t)
|
|
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) ||
|
|
p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") {
|
|
t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
// It said its start and its end, so the operator is told once (ADR 0308).
|
|
if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil {
|
|
t.Fatalf("the walk's start and end are not marked: %+v", p.Times)
|
|
}
|
|
// Nothing waits, and no walk of waiting builds opened beside it.
|
|
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil {
|
|
t.Fatalf("%+v %v", r, err)
|
|
}
|
|
}
|
|
|
|
// A machine that fails what it was sent stops the walk there: the next is not sent, the layer stays, and the
|
|
// failure holds back no walk of waiting builds.
|
|
func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) {
|
|
w := aSettledMesh(t)
|
|
ctx := t.Context()
|
|
inv := w.open.inventory
|
|
w.broken["anchor"] = true
|
|
if err := inv.KeepSettings(ctx, "", "app", map[string]any{"x": "1"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 4; i++ {
|
|
advancePlans(ctx, w.open)
|
|
}
|
|
p := w.walk(t)
|
|
if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed ||
|
|
!strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") ||
|
|
!strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") {
|
|
t.Fatalf("the walk is %s: %s", p.State, p.Note)
|
|
}
|
|
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
|
|
t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent)
|
|
}
|
|
if layer, has, err := inv.Layer(ctx, "", "app"); err != nil || !has || layer["x"] != "1" {
|
|
t.Fatalf("the layer after the failure: %v %v %v", layer, has, err)
|
|
}
|
|
// A build that waits for a gate is still walked: the failed settings walk holds nothing back.
|
|
w.broken["anchor"] = false
|
|
now := time.Now().Add(-time.Minute)
|
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
|
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
|
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil {
|
|
t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err)
|
|
}
|
|
}
|
|
|
|
// Each send of a settings walk keeps every build of its machine: the layer goes, and a build held for a person or
|
|
// waiting for a gate stays where it is (ADR 0344 §3).
|
|
func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) {
|
|
w := aSettledMesh(t)
|
|
ctx := t.Context()
|
|
inv := w.open.inventory
|
|
gateEvery = 0
|
|
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s3"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 4; i++ {
|
|
advancePlans(ctx, w.open)
|
|
}
|
|
if !reflect.DeepEqual(w.kept, []bool{true, true}) || w.walk(t).State != inventory.PlanDone {
|
|
t.Fatalf("the sends kept every build: %v; the walk: %s", w.kept, w.walk(t).Note)
|
|
}
|
|
// What such a send keeps, on the store: a rolled-out build waiting there is kept at the one last sent.
|
|
now := time.Now().Add(-time.Minute)
|
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
|
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
|
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
kept, err := recordedKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor")
|
|
if err != nil || kept["app"] != "c1" {
|
|
t.Fatalf("a settings send to anchor keeps %v (%v), want app at c1", kept, err)
|
|
}
|
|
if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" {
|
|
t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept)
|
|
}
|
|
// And so the ungated build waiting on anchor is no move of the settings send's: nothing for a gate to see.
|
|
f, err := readMoveFacts(ctx, inv)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waiting, _ := machineMoves(ctx, w.open, f, "anchor", false)
|
|
if len(waiting) != 1 || waiting[0].Module != "app" {
|
|
t.Fatalf("anchor waits for %+v, want app's c2", waiting)
|
|
}
|
|
if moved, err := withoutKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor", waiting); err != nil || len(moved) != 0 {
|
|
t.Fatalf("a settings send to anchor would move %+v (%v)", moved, err)
|
|
}
|
|
}
|
|
|
|
// A machine whose send would move a running module's data stops the walk there, failed and said with the push that
|
|
// sends it on a person's word (ADR 0217 §3); one not heard from is passed over and named; and a walk that reached
|
|
// no machine is not said delivered.
|
|
func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T) {
|
|
w := aSettledMesh(t)
|
|
ctx := t.Context()
|
|
gateEvery = 0
|
|
w.moves["laptop"] = []dataMove{{Container: "app-server", Inside: "/data", From: "/srv/a", To: "/srv/b"}}
|
|
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s4"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 4; i++ {
|
|
advancePlans(ctx, w.open)
|
|
}
|
|
p := w.walk(t)
|
|
if p.State != inventory.PlanFailed || !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) ||
|
|
!strings.Contains(p.Note, "not sent to laptop: a running module's data would move there") ||
|
|
!strings.Contains(p.Note, "push laptop --move app") {
|
|
t.Fatalf("sent %v; the walk is %s: %s", w.sent, p.State, p.Note)
|
|
}
|
|
|
|
v := aSettledMesh(t)
|
|
v.heard = map[string]bool{"laptop": true}
|
|
gateEvery = 0
|
|
if _, err := openSettingsWalk(ctx, v.open, inventory.PlanSettings{Module: "app", Ask: "s5"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 4; i++ {
|
|
advancePlans(ctx, v.open)
|
|
}
|
|
p = v.walk(t)
|
|
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Skipped, []string{"anchor"}) ||
|
|
!strings.Contains(p.Note, "delivered on laptop; not heard from, left as they were, their next send carries it: anchor") {
|
|
t.Fatalf("the walk with anchor away is %s: %s", p.State, p.Note)
|
|
}
|
|
|
|
u := aSettledMesh(t)
|
|
u.heard = map[string]bool{}
|
|
if _, err := openSettingsWalk(ctx, u.open, inventory.PlanSettings{Module: "app", Ask: "s6"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advancePlans(ctx, u.open)
|
|
if p := u.walk(t); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed ||
|
|
!strings.Contains(p.Note, "was sent to no machine") || len(u.sent) != 0 {
|
|
t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note)
|
|
}
|
|
}
|
|
|
|
// The gate keeps the module's build its send kept, so a crossing send — a merge's walk sending the machine a new
|
|
// build of the module while the settings gate judges there — supersedes it, and its report is never read as the
|
|
// settings send's (issue 641 review, finding 1).
|
|
func TestACrossingSendSupersedesASettingsGate(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
w := aSettledMesh(t)
|
|
ctx := t.Context()
|
|
inv := w.open.inventory
|
|
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s7"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advancePlans(ctx, w.open)
|
|
p := w.walk(t)
|
|
if g := p.Release.Gate; g == nil || g.To != "c1" || g.From != "c1" {
|
|
t.Fatalf("the settings gate judges no build: %+v", p.Release.Gate)
|
|
}
|
|
// Another walk sends anchor app's c2 while the settings gate judges there.
|
|
now := time.Now().Add(-time.Minute)
|
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
|
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, w.open, "anchor"), "d-merge", map[string]string{"app": "c2"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := inv.RecordDoing(ctx, nodeID(t, w.open, "anchor"), inventory.Doing{Node: "anchor",
|
|
Outcome: inventory.OutcomeApplied, Declared: "d-merge", Applied: 1, At: time.Now()}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gateEvery = 0
|
|
advancePlans(ctx, w.open)
|
|
p = w.walk(t)
|
|
if p.State != inventory.PlanSuperseded || p.Release.Gate == nil || p.Release.Gate.Verdict != inventory.GateSuperseded ||
|
|
!strings.Contains(p.Note, "superseded on anchor") || len(p.Release.Done) != 0 {
|
|
t.Fatalf("the settings walk after a crossing send is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) {
|
|
t.Fatalf("sent %v: laptop was sent after a superseded judging", w.sent)
|
|
}
|
|
}
|
|
|
|
// A send refusal that cannot clear by itself — what the machine was last sent is not known, or a build it runs is
|
|
// gone — fails the walk with why, rather than keeping it open and holding the walk of waiting builds; and a
|
|
// failure names the machines passed over as away (issue 641 review, findings 2 and 3).
|
|
func TestASettingsWalkFailsOnARefusalThatCannotClear(t *testing.T) {
|
|
w := aSettledMesh(t)
|
|
ctx := t.Context()
|
|
inv := w.open.inventory
|
|
// What a send to a machine never sent can keep: nothing, and said as the refusal the walk fails on.
|
|
if _, err := inv.AddNode(ctx, "fresh"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := recordedKept(keepingBuildsForSettings(ctx, "fresh"), w.open, "fresh"); !errors.Is(err, errSettingsCannotKeep) {
|
|
t.Fatalf("a settings send to a machine never sent: %v", err)
|
|
}
|
|
w.heard = map[string]bool{"laptop": true}
|
|
settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) {
|
|
return nil, fmt.Errorf("%w: what %s was last sent is not known", errSettingsCannotKeep, node)
|
|
}
|
|
if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s8"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advancePlans(ctx, w.open)
|
|
p := w.walk(t)
|
|
if p.State != inventory.PlanFailed || len(w.sent) != 0 ||
|
|
!strings.Contains(p.Note, "not sent to laptop: the settings cannot be sent with every build kept: what laptop was last sent is not known") ||
|
|
!strings.Contains(p.Note, "not heard from, left as they were: anchor") {
|
|
t.Fatalf("the walk is %s: %s", p.State, p.Note)
|
|
}
|
|
// It is not open, so a build waiting for a gate is walked.
|
|
now := time.Now().Add(-time.Minute)
|
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"})
|
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2",
|
|
Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now,
|
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{
|
|
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil {
|
|
t.Fatalf("the waiting build is held by the failed settings walk: %+v %v", r, err)
|
|
}
|
|
}
|