package main import ( "bytes" "context" "encoding/json" "reflect" "slices" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/overlay" ) // novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its // upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything // else that moved is still a consequence the push sends (ADR 0083). func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) { current := map[string]inventory.CurrentBuild{ "resolver": {Commit: "c2c2c2c2c2"}, "agent": {Commit: "a2", RollOut: true}, "network": {}, } modules := []string{"network", "resolver", "agent"} sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"} // A module whose policy records moved: held, naming it, both builds and why. why := heldBack("laptop", modules, sent, true, current, nil) if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") || !strings.Contains(why[0], "upgrade policy records") { t.Fatalf("a recorded upgrade did not hold the machine: %v", why) } // Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057). sent["resolver"] = "c2c2c2c2c2" if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 { t.Fatalf("a machine whose builds are all current was held: %v", why) } // A module whose policy rolls out moved, and no plan holds it: sent, as before. sent["agent"] = "a1" if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 { t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why) } // The last send's builds are not known: held whole. if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 || !strings.Contains(why[0], "not known") { t.Fatalf("a machine whose last send was not recorded was not held: %v", why) } // A module the machine was never sent, under a recording policy: held, and said so. delete(sent, "resolver") sent["agent"] = "a2" if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 || !strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") { t.Fatalf("a module never sent under a recording policy: %v", why) } } // ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push // naming some other machine must not send them for it. func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) { at := time.Now() current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}} sent := map[string]string{"agent": "a1"} waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ "agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}} why := heldBack("g14", []string{"agent"}, sent, true, current, waiting) if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") { t.Fatalf("a machine the plan has not reached was not held: %v", why) } // The first machine itself was sent by the plan: not held by it. if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 { t.Fatalf("the plan's first machine was held: %v", why) } // Built but not yet sent anywhere, or not yet built: the plan has it still to send. for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} { plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding, Modules: map[string]*inventory.PlanModule{"agent": s}}} if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 { t.Errorf("%s: not held: %v", what, why) } } // Sent everywhere, failed, or a plan no longer open: the plan holds nothing back. for what, plans := range map[string][]inventory.Plan{ "sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ "agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}}, "failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ "agent": {State: "failed"}}}}, "closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{ "agent": {State: "built"}}}}, } { if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 { t.Errorf("%s: held: %v", what, why) } } } // A send records the build of each module it carried; a module left out of it keeps the build it // was last sent, since the machine keeps that one. func TestASendCarriesTheCurrentBuildsAndALeftOutModuleKeepsItsOwn(t *testing.T) { current := map[string]inventory.CurrentBuild{"a": {Commit: "a2"}, "b": {Commit: "b2"}, "c": {}} got := carriedBuilds([]string{"a", "b", "c"}, map[string]string{"b": "a setting does not compose"}, current, map[string]string{"a": "a1", "b": "b1"}) if want := map[string]string{"a": "a2", "b": "b1", "c": ""}; !reflect.DeepEqual(got, want) { t.Fatalf("carried %v, wanted %v", got, want) } // Not known before: the left-out module is not recorded at all, so it reads as never sent. got = carriedBuilds([]string{"a", "b"}, map[string]string{"b": "x"}, current, nil) if want := map[string]string{"a": "a2"}; !reflect.DeepEqual(got, want) { t.Fatalf("carried %v, wanted %v", got, want) } } // recordedDelivery sends nothing and records each send as the mesh does, so the next comparison // reads the machine as current — and writes down which machines it declared. type recordedDelivery struct { inv *inventory.Inventory declared []string } func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil } func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) { r.declared = append(r.declared, s.node) return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch) } // aResolver is a module built from a repository, at a commit, with something on the machine that // says which build it is. func aResolver(t *testing.T, open *stores, commit string, asked time.Time) { t.Helper() m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{ {"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit}, }} if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{ Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil { t.Fatal(err) } } // A third machine on the private network: once it is sent, every other machine's peers change with // it, which is a consequence a push must still send — no build moved. func aThirdMachine(t *testing.T, open *stores) { t.Helper() ctx := t.Context() record, err := open.inventory.AddNode(ctx, "spare") if err != nil { t.Fatal(err) } if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil { t.Fatal(err) } reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{ {"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true}, {"name": "systemd", "present": true}}}) if err != nil { t.Fatal(err) } var profile map[string]any if err := json.Unmarshal(reported, &profile); err != nil { t.Fatal(err) } if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil { t.Fatal(err) } if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { t.Fatal(err) } if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil { t.Fatal(err) } if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil { t.Fatal(err) } } // The issue as it happened, against the real stores: a change merged with the policy `record`, a push // naming the anchor, and the laptop — running the same module — left with what it had, by name. func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory asked := time.Now().Add(-time.Hour) aResolver(t, open, "c1c1c1c1c1", asked) for _, node := range []string{"anchor", "laptop"} { if _, err := inv.Assign(ctx, node, "resolver"); err != nil { t.Fatal(err) } } // Recorded by a person's choice: the default rolls out since novox/hq ADR 0236. if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil { t.Fatal(err) } gens, err := generators(ctx, open) if err != nil { t.Fatal(err) } compose := composeForPush(open, gens) d := &recordedDelivery{inv: inv} if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { t.Fatal(err) } if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" { t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err) } digestOfLaptop := func() string { sent, err := inv.Outstanding(ctx, "laptop") if err != nil { t.Fatal(err) } return sent } before := digestOfLaptop() // The change merges; the policy is record. `push anchor` sends the anchor... aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute)) d.declared = nil if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil { t.Fatal(err) } // ...and its cascade leaves the laptop, saying so. var said bytes.Buffer d.declared = nil refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said) if err != nil || len(refused) != 0 { t.Fatalf("the cascade failed: %v %v", refused, err) } if len(d.declared) != 0 { t.Fatalf("the cascade sent %v a build its policy records", d.declared) } if digestOfLaptop() != before { t.Fatal("the laptop's last send moved: it was sent the held build") } for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2", "upgrade policy records", "`push laptop` sends it"} { if !strings.Contains(said.String(), want) { t.Errorf("the push did not say %q:\n%s", want, said.String()) } } // Held and owed something else at once — a peer joined: still not sent, and both said: why it // is held, and that what else it is owed waits with it. aThirdMachine(t, open) d.declared = nil if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { t.Fatal(err) } d.declared = nil said.Reset() if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, compose, d, "", nil, &said); err != nil { t.Fatal(err) } if len(d.declared) != 0 || digestOfLaptop() != before { t.Fatalf("a held machine owed a consequence was sent: %v", d.declared) } if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") { t.Fatalf("the push did not say both:\n%s", said.String()) } // A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it // (novox/hq ADR 0236). if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil { t.Fatal(err) } said.Reset() if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, compose, d, "", nil, &said); err != nil { t.Fatal(err) } if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") { t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String()) } // Once it passed a gate on some machine, the laptop is a consequence like any other, and sent. if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2", Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { t.Fatal(err) } said.Reset() if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, compose, d, "", nil, &said); err != nil { t.Fatal(err) } if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before { t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String()) } if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" { t.Fatalf("the new send did not record the new build: %v", builds) } } // Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved // for another reason is sent by a push that names someone else. func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour)) if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil { t.Fatal(err) } gens, err := generators(ctx, open) if err != nil { t.Fatal(err) } compose := composeForPush(open, gens) d := &recordedDelivery{inv: inv} if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { t.Fatal(err) } // `push spare`, the machine just placed: the others' peers change with it. aThirdMachine(t, open) if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { t.Fatal(err) } d.declared = nil var said bytes.Buffer if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { t.Fatal(err) } if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) { t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String()) } if strings.Contains(said.String(), "was not sent") { t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String()) } // A machine whose last send was not recorded — a declaration sent by hand — is held until named. record, err := inv.NodeByName(ctx, "laptop") if err != nil { t.Fatal(err) } if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil { t.Fatal(err) } d.declared = nil said.Reset() if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { t.Fatal(err) } // The anchor, the hub, may still be settling from the machine placed above; the laptop is the // question. if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") { t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String()) } }