jschoubben
313efa826c
A declaration newer in sequence than every other named four fewer modules than the assignments held, a machine applied it, and nothing the mesh kept said who sent it or from what view. The store now raises an assignment generation in the same transaction as every assignment change (a trigger, so a cascade counts too); a send reads it before composing, carries it to engines that said they read it, marks a gate's put-back, and is recorded with its sequence, sender, generation and modules. The would-send is stamped with what was last sent, so nothing reads behind for it; a refusal is kept on the send it refused and raised as S20 naming the sender; plan says what the machine was last told.
354 lines
15 KiB
Go
354 lines
15 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"reflect"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/overlay"
|
|
)
|
|
|
|
// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its
|
|
// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything
|
|
// else that moved is still a consequence the push sends (ADR 0083).
|
|
func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) {
|
|
current := map[string]inventory.CurrentBuild{
|
|
"resolver": {Commit: "c2c2c2c2c2"},
|
|
"agent": {Commit: "a2", RollOut: true},
|
|
"network": {},
|
|
}
|
|
modules := []string{"network", "resolver", "agent"}
|
|
sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"}
|
|
|
|
// A module whose policy records moved: held, naming it, both builds and why.
|
|
why := heldBack("laptop", modules, sent, true, current, nil)
|
|
if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") ||
|
|
!strings.Contains(why[0], "upgrade policy records") {
|
|
t.Fatalf("a recorded upgrade did not hold the machine: %v", why)
|
|
}
|
|
|
|
// Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057).
|
|
sent["resolver"] = "c2c2c2c2c2"
|
|
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
|
t.Fatalf("a machine whose builds are all current was held: %v", why)
|
|
}
|
|
|
|
// A module whose policy rolls out moved, and no plan holds it: sent, as before.
|
|
sent["agent"] = "a1"
|
|
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 {
|
|
t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why)
|
|
}
|
|
|
|
// The last send's builds are not known: held whole.
|
|
if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 ||
|
|
!strings.Contains(why[0], "not known") {
|
|
t.Fatalf("a machine whose last send was not recorded was not held: %v", why)
|
|
}
|
|
|
|
// A module the machine was never sent, under a recording policy: held, and said so.
|
|
delete(sent, "resolver")
|
|
sent["agent"] = "a2"
|
|
if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 ||
|
|
!strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") {
|
|
t.Fatalf("a module never sent under a recording policy: %v", why)
|
|
}
|
|
}
|
|
|
|
// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push
|
|
// naming some other machine must not send them for it.
|
|
func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) {
|
|
at := time.Now()
|
|
current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}}
|
|
sent := map[string]string{"agent": "a1"}
|
|
waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
|
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}}
|
|
|
|
why := heldBack("g14", []string{"agent"}, sent, true, current, waiting)
|
|
if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") {
|
|
t.Fatalf("a machine the plan has not reached was not held: %v", why)
|
|
}
|
|
// The first machine itself was sent by the plan: not held by it.
|
|
if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 {
|
|
t.Fatalf("the plan's first machine was held: %v", why)
|
|
}
|
|
// Built but not yet sent anywhere, or not yet built: the plan has it still to send.
|
|
for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} {
|
|
plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding,
|
|
Modules: map[string]*inventory.PlanModule{"agent": s}}}
|
|
if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 {
|
|
t.Errorf("%s: not held: %v", what, why)
|
|
}
|
|
}
|
|
// Sent everywhere, failed, or a plan no longer open: the plan holds nothing back.
|
|
for what, plans := range map[string][]inventory.Plan{
|
|
"sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
|
"agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}},
|
|
"failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{
|
|
"agent": {State: "failed"}}}},
|
|
"closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{
|
|
"agent": {State: "built"}}}},
|
|
} {
|
|
if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 {
|
|
t.Errorf("%s: held: %v", what, why)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A send records the build of each module it carried; a module left out of it keeps the build it
|
|
// was last sent, since the machine keeps that one.
|
|
func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) {
|
|
current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}}
|
|
got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"},
|
|
current, map[string]string{"a": "a1", "b": "b1"})
|
|
if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("carried %v, wanted %v", got, want)
|
|
}
|
|
// Not known before: the left-out module is not recorded at all, so it reads as never sent.
|
|
got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil)
|
|
if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("carried %v, wanted %v", got, want)
|
|
}
|
|
}
|
|
|
|
// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison
|
|
// reads the machine as current — and writes down which machines it declared.
|
|
type recordedDelivery struct {
|
|
inv *inventory.Inventory
|
|
declared []string
|
|
}
|
|
|
|
func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil }
|
|
|
|
func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
|
r.declared = append(r.declared, s.node)
|
|
return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared))
|
|
}
|
|
|
|
// aResolver is a module built from a repository, at a commit, with something on the machine that
|
|
// says which build it is.
|
|
func aResolver(t *testing.T, open *stores, commit string, asked time.Time) {
|
|
t.Helper()
|
|
m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{
|
|
{"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit},
|
|
}}
|
|
if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{
|
|
Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// A third machine on the private network: once it is sent, every other machine's peers change with
|
|
// it, which is a consequence a push must still send — no build moved.
|
|
func aThirdMachine(t *testing.T, open *stores) {
|
|
t.Helper()
|
|
ctx := t.Context()
|
|
record, err := open.inventory.AddNode(ctx, "spare")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{
|
|
{"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true},
|
|
{"name": "systemd", "present": true}}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var profile map[string]any
|
|
if err := json.Unmarshal(reported, &profile); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// The issue as it happened, against the real stores: a change merged with the policy `record`, a push
|
|
// naming the anchor, and the laptop — running the same module — left with what it had, by name.
|
|
func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
asked := time.Now().Add(-time.Hour)
|
|
aResolver(t, open, "c1c1c1c1c1", asked)
|
|
for _, node := range []string{"anchor", "laptop"} {
|
|
if _, err := inv.Assign(ctx, node, "resolver"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
// Recorded by a person's choice: the default rolls out since novox/hq ADR 0236.
|
|
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gens, err := generators(ctx, open)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
compose := composeForPush(open, gens)
|
|
d := &recordedDelivery{inv: inv}
|
|
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" {
|
|
t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err)
|
|
}
|
|
digestOfLaptop := func() string {
|
|
sent, err := inv.Outstanding(ctx, "laptop")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return sent
|
|
}
|
|
before := digestOfLaptop()
|
|
|
|
// The change merges; the policy is record. `push anchor` sends the anchor...
|
|
aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute))
|
|
d.declared = nil
|
|
if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// ...and its cascade leaves the laptop, saying so.
|
|
var said bytes.Buffer
|
|
d.declared = nil
|
|
refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said)
|
|
if err != nil || len(refused) != 0 {
|
|
t.Fatalf("the cascade failed: %v %v", refused, err)
|
|
}
|
|
if len(d.declared) != 0 {
|
|
t.Fatalf("the cascade sent %v a build its policy records", d.declared)
|
|
}
|
|
if digestOfLaptop() != before {
|
|
t.Fatal("the laptop's last send moved: it was sent the held build")
|
|
}
|
|
for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2",
|
|
"upgrade policy records", "`push laptop` sends it"} {
|
|
if !strings.Contains(said.String(), want) {
|
|
t.Errorf("the push did not say %q:\n%s", want, said.String())
|
|
}
|
|
}
|
|
|
|
// Held and owed something else at once — a peer joined: still not sent, and both said: why it
|
|
// is held, and that what else it is owed waits with it.
|
|
aThirdMachine(t, open)
|
|
d.declared = nil
|
|
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d.declared = nil
|
|
said.Reset()
|
|
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
|
compose, d, "", nil, &said); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(d.declared) != 0 || digestOfLaptop() != before {
|
|
t.Fatalf("a held machine owed a consequence was sent: %v", d.declared)
|
|
}
|
|
if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") {
|
|
t.Fatalf("the push did not say both:\n%s", said.String())
|
|
}
|
|
|
|
// A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it
|
|
// (novox/hq ADR 0236).
|
|
if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
said.Reset()
|
|
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
|
compose, d, "", nil, &said); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") {
|
|
t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String())
|
|
}
|
|
// Once it passed a gate on some machine, the laptop is a consequence like any other, and sent.
|
|
if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2",
|
|
Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
said.Reset()
|
|
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true},
|
|
compose, d, "", nil, &said); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before {
|
|
t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String())
|
|
}
|
|
if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" {
|
|
t.Fatalf("the new send did not record the new build: %v", builds)
|
|
}
|
|
}
|
|
|
|
// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved
|
|
// for another reason is sent by a push that names someone else.
|
|
func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour))
|
|
if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gens, err := generators(ctx, open)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
compose := composeForPush(open, gens)
|
|
d := &recordedDelivery{inv: inv}
|
|
if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
// `push spare`, the machine just placed: the others' peers change with it.
|
|
aThirdMachine(t, open)
|
|
if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d.declared = nil
|
|
var said bytes.Buffer
|
|
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) {
|
|
t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String())
|
|
}
|
|
if strings.Contains(said.String(), "was not sent") {
|
|
t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String())
|
|
}
|
|
|
|
// A machine whose last send was not recorded — a declaration sent by hand — is held until named.
|
|
record, err := inv.NodeByName(ctx, "laptop")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
d.declared = nil
|
|
said.Reset()
|
|
if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// The anchor, the hub, may still be settling from the machine placed above; the laptop is the
|
|
// question.
|
|
if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") {
|
|
t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String())
|
|
}
|
|
}
|