//go:build integration package main import ( "context" "encoding/json" "fmt" "strings" "sync" "testing" "time" "github.com/nats-io/nats.go" "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/testbus" ) // 1160/AC5, live, with the store and a real bus and no ticker: a plan of two tiers whose first tier has a machine that // never reports, and whose second tier waits on the first (the plan's own order), passes its first tier after the bound, // the silent key's expiry the only trigger, and sends its second tier to the machine that reported alone (issue 1160). func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToTheOthers(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory // A quarter of an hour of silence is two seconds here. wasSilent := applySilentAfter applySilentAfter = 2 * time.Second t.Cleanup(func() { applySilentAfter = wasSilent }) js, err := broker.Dial(testbus.URL(t)) if err != nil { t.Fatal(err) } t.Cleanup(js.Close) if err := broker.AssertMeshStreams(js); err != nil { t.Fatal(err) } if err := js.EnsureControllerBuckets(); err != nil { t.Fatal(err) } legacy, err := js.Conn().JetStream() if err != nil { t.Fatal(err) } checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} t.Cleanup(func() { checkEvents = nil }) sentHeard := make(chan *nats.Msg, 32) sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) wasSaved := inventory.PlanSaved inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open) t.Cleanup(func() { inventory.PlanSaved = wasSaved }) // The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept. var mu sync.Mutex var sends [][]string asksRecorded(t) // each module is taken from a done key as the walk reaches it digests := 0 wasSend := sendRollout sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { mu.Lock() defer mu.Unlock() sends = append(sends, append([]string(nil), names...)) current, err := open.inventory.CurrentBuilds(ctx) if err != nil { return nil, err } builds := map[string]string{} for _, m := range []string{"a", "b"} { builds[m] = current[m].Commit } for _, node := range names { digests++ if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil { return nil, err } } return names, nil } t.Cleanup(func() { sendRollout = wasSend }) register := func(m, commit string, asked time.Time) { t.Helper() if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit, Asked: asked}); err != nil { t.Fatal(err) } } record := func(id, m, commit string, at time.Time) { t.Helper() b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at} b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) if err := inv.RecordBuild(ctx, b); err != nil { t.Fatal(err) } } old := time.Now().Add(-2 * time.Hour) for _, m := range []string{"a", "b"} { record("build-"+m+"-1", m, "c1", old) register(m, "c1", old) for _, n := range []string{"anchor", "laptop"} { if _, err := inv.Assign(ctx, n, m); err != nil { t.Fatal(err) } } } for _, n := range []string{"anchor", "laptop"} { if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil { t.Fatal(err) } } record("build-a-2", "a", "c2", time.Now().Add(-time.Minute)) register("a", "c2", time.Now().Add(-time.Minute)) // A machine's report on what it was sent, heard as the controller hears it. listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} seq := int64(0) reports := func(node string) { t.Helper() last, found, err := inv.SentTo(ctx, node) if err != nil || !found { t.Fatalf("%s was sent nothing: %v", node, err) } seq++ body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, "report_sequence": seq}) var report link.Report if err := json.Unmarshal(body, &report); err != nil { t.Fatal(err) } if _, err := listener.Heard(ctx, report); err != nil { t.Fatal(err) } } plan := func() inventory.Plan { t.Helper() p, err := inv.PlanByID(ctx, "plan-1160") if err != nil { t.Fatal(err) } return p } built := time.Now().UTC() p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}}, Modules: map[string]*inventory.PlanModule{ "a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"}, "b": {}, }} first, stopFirst := context.WithCancel(ctx) defer stopFirst() waitFirst := startRolloutEvents(first, open, js) // Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338). t.Cleanup(func() { stopFirst(); waitFirst() }) if err := inv.SavePlan(ctx, &p); err != nil { t.Fatal(err) } eventually(t, "tier 0 sent to both machines", func() bool { mu.Lock() defer mu.Unlock() return len(sends) == 1 }) reports("anchor") time.Sleep(time.Second) if got := plan().Tier; got != 0 { t.Fatalf("1160/AC5: the plan went on to tier %d before the bound, with laptop not heard from", got) } // The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own. var moved string eventually(t, "tier 1 passed once laptop's key expired", func() bool { if q := plan(); q.Tier == 1 { moved = q.Note return true } return false }) if !strings.Contains(moved, "laptop") || strings.Contains(moved, "anchor") { t.Errorf("1160/AC5: the plan says %q; want laptop named silent and not anchor", moved) } // Tier 1's module is taken from its key, which the walk's look does by itself. eventually(t, "tier 1 sent to anchor alone", func() bool { mu.Lock() defer mu.Unlock() return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor" }) }