package main import ( "context" "encoding/json" "errors" "fmt" "reflect" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" ) // The backlog the old default left โ€” builds registered and sent nowhere โ€” is released by a plan, one // machine at a time, each judged, never by the next send of something else (novox/hq ADR 0236). type backlogMesh struct { open *stores sent [][]string broken map[string]bool } // aBacklog is a mesh where `app` moved c1 โ†’ c2 on anchor and laptop, `late` moved on laptop only, and // `same` was rebuilt with the very artifacts it had: two machines heard, every send applied at once. func aBacklog(t *testing.T) *backlogMesh { t.Helper() open := aMesh(t) ctx := t.Context() inv := open.inventory b := &backlogMesh{open: open, broken: map[string]bool{}} withConditionsInMemory(t) was := doctorFrom doctorFrom = nil t.Cleanup(func() { doctorFrom = was }) build := func(module, commit, made string, asked time.Time) { manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"}) if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit, Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: made}}}); err != nil { t.Fatal(err) } if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{ Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit, Asked: asked}); err != nil { t.Fatal(err) } } old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) on := map[string][]string{"app": {"anchor", "laptop"}, "late": {"laptop"}, "same": {"anchor", "laptop"}} for _, m := range []string{"app", "late", "same"} { build(m, "c1", "sha256:"+m+"-1", old) for _, n := range on[m] { if _, err := inv.Assign(ctx, n, m); err != nil { t.Fatal(err) } } } for _, n := range []string{"anchor", "laptop"} { sent := map[string]string{} for m, nodes := range on { for _, x := range nodes { if x == n { sent[m] = "c1" } } } if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil { t.Fatal(err) } } build("app", "c2", "sha256:app-2", now) build("late", "c2", "sha256:late-2", now) build("same", "c2", "sha256:same-1", now) // rebuilt, the same bytes assigned := func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { modules, err := open.inventory.Assigned(ctx, node) if err != nil { return nil, err } sent, known, err := open.inventory.SentBuilds(ctx, node) if err != nil { return nil, err } return f.moves(node, modules, sent, known, all), nil } wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts machineMoves = assigned releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"anchor": true, "laptop": true}, nil } n := 0 sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { b.sent = append(b.sent, append([]string(nil), names...)) current, err := open.inventory.CurrentBuilds(ctx) if err != nil { return nil, err } for _, node := range names { modules, _ := open.inventory.Assigned(ctx, node) carried := map[string]string{} for _, m := range modules { carried[m] = current[m].Commit } n++ digest := fmt.Sprintf("d-%s-%d", node, n) if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { return nil, err } // A broken machine reports its apply failed (novox/hq ADR 0360 ยง7: the report is the verdict). outcome := inventory.OutcomeApplied if b.broken[node] { outcome = inventory.OutcomeFailed } if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, Outcome: outcome, Declared: digest, Applied: 1, At: time.Now()}); err != nil { return nil, err } } return names, nil } gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} reports, err := open.inventory.LastReports(ctx) if err != nil { return f, err } for _, r := range reports { if b.broken[r.Node] { r.Outcome = inventory.OutcomeFailed } f.reports[r.Node] = r } return f, nil } wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). wasWindow := gateWitnessWindow gateWitnessWindow = 0 t.Cleanup(func() { gateWitnessWindow = wasWindow }) gateWithin, gateEvery = time.Hour, time.Hour t.Cleanup(func() { machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) return b } func (b *backlogMesh) release(t *testing.T) inventory.Plan { t.Helper() plans, err := b.open.inventory.RecentPlans(t.Context(), 10) if err != nil { t.Fatal(err) } for _, p := range plans { if p.Release != nil { return p } } t.Fatal("no release plan") return inventory.Plan{} } // The backlog goes out one machine at a time: the first judged before the second is sent, each build's // pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused. func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) { withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) judgeLeavesAsTheCore(t) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory f, err := readMoveFacts(ctx, inv) if err != nil { t.Fatal(err) } moves, _ := machineMoves(ctx, b.open, f, "laptop", false) var names []string for _, mv := range moves { names = append(names, mv.Module) } if !reflect.DeepEqual(names, []string{"app", "late"}) { t.Fatalf("laptop waits for %v; a rebuild with the same bytes is no move", names) } // Nothing else may carry them: a send that judges nothing is refused. if _, err := ungatedIn(ctx, b.open, []string{"laptop"}, ""); !errors.Is(err, errUngated) { t.Fatalf("a send that judges nothing would carry them: %v", err) } advancePlans(ctx, b.open) if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) { t.Fatalf("sent %v: the first machine alone, before it is judged", b.sent) } p := b.release(t) if !reflect.DeepEqual(p.Release.Order, []string{"anchor", "laptop"}) || p.Release.Gate == nil { t.Fatalf("the release plan is %+v", p.Release) } gateEvery = 0 for i := 0; i < 4; i++ { advancePlans(ctx, b.open) } if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"laptop"}}) { t.Fatalf("sent %v", b.sent) } p = b.release(t) if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) { t.Fatalf("the release plan is %s: %s %+v", p.State, p.Note, p.Release) } for _, build := range []string{"build-app-c2", "build-late-c2"} { if v, found, err := inv.GateOf(ctx, build); err != nil || !found || v.Verdict != inventory.GatePassed { t.Fatalf("%s's pass was not kept: %+v %v %v", build, v, found, err) } } // Nothing waits now, and nothing opens again. if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil { t.Fatalf("a release opened with nothing waiting: %+v %v", p, err) } } // A release that fails on its first machine puts back what it carried there, goes no further, and the // next one waits for a person, said as a condition; a person releases it with why. func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) { withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) b := aBacklog(t) ctx := t.Context() inv := b.open.inventory gateEvery = 0 b.broken["anchor"] = true advancePlans(ctx, b.open) p := b.release(t) if p.State != inventory.PlanFailed { t.Fatalf("the release is %s: %s", p.State, p.Note) } if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"anchor"}}) { t.Fatalf("sent %v: the first machine, then the put-back to it, and nothing to laptop", b.sent) } if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { t.Fatalf("app is at %s, not put back", current["app"].Commit) } if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { t.Fatal("app's build is not marked failed at its gate") } // late still waits on laptop, and is not released on its own after a failure. if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil { t.Fatalf("a release opened after a failed one: %+v %v", p, err) } if obs := backlogObservation(); len(obs) != 1 || !strings.Contains(obs[0].Summary, "release-backlog") { t.Fatalf("the held release is not said: %+v", obs) } b.broken["anchor"] = false if err := backlogCommand(ctx, "release-backlog", []string{"--why", "anchor is fixed"}); err != nil { t.Fatal(err) } for i := 0; i < 4; i++ { advancePlans(ctx, b.open) } if p := b.release(t); p.State != inventory.PlanDone || p.Release.By == "" { t.Fatalf("the person's release is %s (%+v)", p.State, p.Release) } if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { t.Fatalf("late was not released to laptop: %v", sent) } }