173 lines
7.3 KiB
Go
173 lines
7.3 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
// handingAnchor makes every send to anchor hand the provisioner the given modules (module -> templates digest), and
|
|
// the clients' words the given reports (key `node/module`), as the stream would give them (issue 840, ADR 0362).
|
|
func handingAnchor(t *testing.T, b *backlogMesh, handed map[string]string) map[string]inventory.ProvReport {
|
|
t.Helper()
|
|
says := map[string]inventory.ProvReport{}
|
|
wasSays := provAppliedOn
|
|
provAppliedOn = func(_ context.Context, node, module string) (inventory.ProvReport, bool, error) {
|
|
r, ok := says[node+"/"+module]
|
|
return r, ok, nil
|
|
}
|
|
inner := sendRollout
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
got, err := inner(ctx, open, names)
|
|
if err != nil {
|
|
return got, err
|
|
}
|
|
for _, n := range names {
|
|
if n != "anchor" {
|
|
continue
|
|
}
|
|
inv := open.inventory
|
|
s, _, err := inv.SentTo(ctx, n)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
builds, _, err := inv.SentBuilds(ctx, n)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
plans := map[string]inventory.Provisioning{}
|
|
for m, d := range handed {
|
|
plans[m] = inventory.Provisioning{Templates: inventory.TemplatesArchive{Digest: d}}
|
|
}
|
|
raw, err := json.Marshal(plans)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
id := nodeID(t, open, n)
|
|
if err := inv.RecordSentWith(ctx, id, s.Digest, builds, 0, 0, inventory.Send{Node: id, Provisioning: raw}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return got, nil
|
|
}
|
|
t.Cleanup(func() { sendRollout, provAppliedOn = inner, wasSays })
|
|
return says
|
|
}
|
|
|
|
func stepRelease(t *testing.T, b *backlogMesh, n int) inventory.Plan {
|
|
t.Helper()
|
|
for i := 0; i < n; i++ {
|
|
advancePlans(t.Context(), b.open)
|
|
}
|
|
return b.release(t)
|
|
}
|
|
|
|
// Issue 840 (novox/hq ADR 0362 §1-5): the release's rest gate, which every delivery passes since ADR 0360, counts a
|
|
// machine's send of a provisioned module applied only once the engine and the machine's provisioner client both said
|
|
// so; a failed client is a failed apply named with its reason; none past ApplySilentAfter is silent, never done.
|
|
func TestTheRestGateWaitsForTheProvisioner(t *testing.T) {
|
|
judgeLeavesAsTheCore(t)
|
|
applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: time.Now()}
|
|
|
|
t.Run("a engine applied, the client has not said: no verdict, the release stays open", func(t *testing.T) {
|
|
b := aBacklog(t)
|
|
handingAnchor(t, b, map[string]string{"app": "t1"})
|
|
p := stepRelease(t, b, 8)
|
|
g := p.Release.Gates["anchor"]
|
|
if !p.Open() || g == nil || g.Verdict != "" || g.Silent || slices.Contains(p.Release.Done, "anchor") {
|
|
t.Fatalf("anchor is judged before its provisioner reported: %s %s %+v", p.State, p.Note, g)
|
|
}
|
|
if !slices.Contains(p.Release.Done, "laptop") {
|
|
t.Fatalf("laptop hands nothing and passes on the engine's report: %+v", p.Release)
|
|
}
|
|
})
|
|
t.Run("b the client failed: a failed apply named with the reason", func(t *testing.T) {
|
|
b := aBacklog(t)
|
|
says := handingAnchor(t, b, map[string]string{"app": "t1"})
|
|
says["anchor/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: time.Now()}
|
|
p := stepRelease(t, b, 8)
|
|
g := p.Release.Gates["anchor"]
|
|
if g == nil || g.Verdict != inventory.GateFailed || !slices.Contains(p.Release.Failed, "anchor") ||
|
|
!strings.Contains(p.Note, "provisioning failed (app: no value)") {
|
|
t.Fatalf("the failure is not named: %s %q %+v %+v", p.State, p.Note, g, p.Release)
|
|
}
|
|
})
|
|
t.Run("c no client report past ApplySilentAfter: silent, never done", func(t *testing.T) {
|
|
b := aBacklog(t)
|
|
handingAnchor(t, b, map[string]string{"app": "t1"})
|
|
p := stepRelease(t, b, 8)
|
|
g := p.Release.Gates["anchor"]
|
|
if g == nil {
|
|
t.Fatalf("no gate: %+v", p.Release)
|
|
}
|
|
since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute)
|
|
g.Since = &since
|
|
if err := b.open.inventory.SavePlan(t.Context(), &p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p = stepRelease(t, b, 1)
|
|
if !slices.Equal(p.Release.Silent, []string{"anchor"}) || slices.Contains(p.Release.Done, "anchor") {
|
|
t.Fatalf("anchor is not silent: %s %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
})
|
|
t.Run("d both applied: the gate passes", func(t *testing.T) {
|
|
b := aBacklog(t)
|
|
says := handingAnchor(t, b, map[string]string{"app": "t1"})
|
|
says["anchor/app"] = applied
|
|
p := stepRelease(t, b, 8)
|
|
if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 {
|
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
})
|
|
t.Run("e a send handing nothing passes on the engine's report", func(t *testing.T) {
|
|
b := aBacklog(t)
|
|
p := stepRelease(t, b, 8)
|
|
if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") {
|
|
t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
})
|
|
t.Run("f a module the delivery does not carry does not touch its end", func(t *testing.T) {
|
|
// anchor's declaration also hands `same`, rebuilt with the very bytes and so not carried: its failed report
|
|
// is none of this delivery's.
|
|
b := aBacklog(t)
|
|
says := handingAnchor(t, b, map[string]string{"app": "t1", "same": "t9"})
|
|
says["anchor/app"] = applied
|
|
says["anchor/same"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t9", Reason: "stuck", At: time.Now()}
|
|
p := stepRelease(t, b, 8)
|
|
if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 {
|
|
t.Fatalf("another module's failure ended this delivery: %s %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
})
|
|
}
|
|
|
|
// Issue 840: a walk's end waits on the modules it sent to a machine, not on every module that machine's declaration
|
|
// hands to the provisioner.
|
|
func TestAWalkEndWaitsOnlyOnTheModulesItSent(t *testing.T) {
|
|
now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC)
|
|
at := now.Add(-10 * time.Second)
|
|
handed := map[string]string{"app": "t1", "other": "t2"}
|
|
p := inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{
|
|
"ace": {Digest: "d-ace", At: now.Add(-time.Minute), Provisioned: handed}}}}}
|
|
reports := []inventory.Reported{{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}}
|
|
says := map[string]inventory.ProvReport{"ace/app": {Status: inventory.ProvApplied, Templates: "t1", At: at}}
|
|
prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok }
|
|
// other has said nothing: not this walk's to wait for.
|
|
if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 {
|
|
t.Fatalf("missing: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
|
}
|
|
says["ace/other"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t2", Reason: "stuck", At: at}
|
|
if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 {
|
|
t.Fatalf("failed: waiting %v, failed %v, silent %v", waiting, failed, silent)
|
|
}
|
|
// once the walk carries it too, it does.
|
|
p.Modules["other"] = &inventory.PlanModule{Rest: p.Modules["app"].Rest}
|
|
if _, failed, _ := walkReports(p, reports, now, prov); len(failed) != 1 || !strings.Contains(failed[0], "other: stuck") {
|
|
t.Fatalf("a carried module's failure is not named: %v", failed)
|
|
}
|
|
}
|