package main import ( "context" "encoding/json" "errors" "fmt" "slices" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) // failedBuild settles the merge's build of modules/sensors as failed. func failedBuild(t *testing.T, open *stores, id string) { t.Helper() if err := (builds{open.inventory, open}).Built(t.Context(), outcome(id, "modules/sensors", "3da80a4b00aa", "", "boom")); err != nil { t.Fatal(err) } } func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool { t.Helper() _, found, err := conditionsFrom.Get(t.Context(), pendingObservation(p).Key()) if err != nil { t.Fatal(err) } return found } func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment { t.Helper() rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0)) if err != nil { t.Fatal(err) } for _, r := range rows { if r.ID == id { return r } } t.Fatalf("pending assignment %d is not on record", id) return inventory.PendingAssignment{} } // novox/hq issue 325: an assignment of a module the catalogue does not hold says which case it is in — a // build in flight (kept pending), known and not built (said, with build), or unknown (refused, with the // closest names) — and a pending assignment is made when its build registers the module, or ends with why. // aCatalogueMesh is aMesh with one module held from the catalogue, so a merge of it is acted on. func aCatalogueMesh(t *testing.T) *stores { t.Helper() open := aMesh(t) if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "networkmanager", Version: "1"}, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { t.Fatal(err) } return open } // mergeAdding is the catalogue's merge adding one module's directory, acted on as the bus hands it over. func mergeAdding(t *testing.T, open *stores, dir, commit string) { t.Helper() m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: []string{dir + "/module.json"}, ModuleDirs: []string{dir}, ModuleDirsSaid: true} if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil { t.Fatal(err) } } // outcome is a build's outcome as the build seat announces it: registered as module, or failed. func outcome(id, dir, commit, module, failed string) link.BuildResult { r := link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: dir, Ref: "main", Commit: commit, On: "anchor", Failed: failed, Source: &link.SourceOnSeat{Repository: "novox/mesh-catalog", Seat: "git"}} if failed == "" { r.Module = module r.Manifest, _ = json.Marshal(catalogue.Manifest{Module: module, Version: "1"}) } return r } func assignedTo(t *testing.T, open *stores, node string) []string { t.Helper() got, err := open.inventory.Assigned(t.Context(), node) if err != nil { t.Fatal(err) } return got } func pendingOf(t *testing.T, open *stores) []inventory.PendingAssignment { t.Helper() got, err := open.inventory.Pending(t.Context(), time.Now().Add(-time.Hour)) if err != nil { t.Fatal(err) } return got } // A build in flight: the merge asked for it, the request is kept as the merge's, the assignment is kept // pending and said so with the build, status reads it without settling anything, and the build's outcome // makes it — saying that a push sends it, and nothing more. func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() asksWithPaths(t) mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") requests, err := open.inventory.RequestsNamed(ctx, "sensors") if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" { t.Fatalf("the merge's build request: %+v %v", requests, err) } said, err := assign(ctx, open, "laptop", "sensors") if err != nil { t.Fatalf("an assignment while its build runs was refused: %v", err) } t.Logf("assign laptop sensors, while its build runs:\n%s", said) for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)", "build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} { if !strings.Contains(said, want) { t.Fatalf("the answer does not say %q:\n%s", want, said) } } if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { t.Fatal("a module not registered was assigned") } pending := pendingOf(t, open) if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" { t.Fatalf("pending: %+v", pending) } if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") { t.Fatalf("a second assignment: %q %v", again, err) } asked, err := theThreeQuestions(ctx, open) if err != nil { t.Fatal(err) } if asked.well() { t.Fatal("status called the mesh well while an assignment waits") } body, err := statusAsJSON(asked) if err != nil { t.Fatal(err) } var doc meshStatus if err := json.Unmarshal(body, &doc); err != nil { t.Fatal(err) } if len(doc.Pending) != 1 || doc.Pending[0].State != "waiting" || doc.Pending[0].Module != "sensors" { t.Fatalf("status says pending %+v", doc.Pending) } if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil { t.Fatal(err) } if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { t.Fatal("the build registered sensors and the pending assignment was not made") } pending = pendingOf(t, open) if len(pending) != 1 || pending[0].State != inventory.PendingApplied || !strings.HasSuffix(pending[0].Note, "`push laptop` sends it") { t.Fatalf("pending after the build: %+v", pending) } } // **A read settles nothing** (review of #150, point 6): status — and so the board, the summary and the // probe, which compose from the same reading — leaves a pending assignment whose module is registered as it // is; the controller's tick makes it. func TestAStatusReadSettlesNothing(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() asksWithPaths(t) mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { t.Fatal(err) } register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) for range 2 { if _, err := theThreeQuestions(ctx, open); err != nil { t.Fatal(err) } } if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting { t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open)) } said := settlePending(ctx, open, time.Now()) if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied { t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open)) } } // The build of a pending assignment fails: it expires with the build's words and nothing is assigned; the // tick raises it as a condition once; it then reads as known and not built; and `unassign` takes it back, // after which the tick clears the condition. Status is not called well while the condition is open, and is // again once it clears — no fixed day of "not well" (review point 6). func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() asksWithPaths(t) mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { t.Fatal(err) } if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "", "go build: undefined: sensorsRead")); err != nil { t.Fatal(err) } pending := pendingOf(t, open) if len(pending) != 1 || pending[0].State != inventory.PendingExpired { t.Fatalf("pending after a failed build: %+v", pending) } for _, want := range []string{"expired", "build b-modules/sensors of modules/sensors failed", "undefined: sensorsRead", "nothing was assigned", "build \"true\""} { if !strings.Contains(pending[0].Note, want) { t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note) } } if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { t.Fatal("a failed build's module was assigned") } settlePending(ctx, open, time.Now()) key := pendingObservation(pending[0]).Key() c, found, err := conditionsFrom.Get(ctx, key) if err != nil || !found || c.Kind != kindPendingEnded { t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err) } if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 { t.Fatalf("status does not hold the open condition: %v", err) } _, err = assign(ctx, open, "laptop", "sensors") if !errors.Is(err, inventory.ErrNoSuchModule) { t.Fatalf("a module whose build failed: %v", err) } for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors", "failed: go build: undefined: sensorsRead", "build \"true\"", "repository novox/mesh-catalog, path modules/sensors"} { if !strings.Contains(err.Error(), want) { t.Fatalf("the refusal does not say %q:\n%v", want, err) } } said, err := unassign(ctx, open, "laptop", "sensors") if err != nil || !strings.Contains(said, "taken back") { t.Fatalf("unassign of an expired pending assignment: %q %v", said, err) } settlePending(ctx, open, time.Now()) if _, found, _ := conditionsFrom.Get(ctx, key); found { t.Fatal("the condition stayed open after the assignment was taken back") } // Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it. asked, err := theThreeQuestions(ctx, open) if err != nil || pendingOpen(asked.pending) { t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err) } for _, c := range asked.conditions { if c.Kind == kindPendingEnded { t.Fatalf("status still holds the condition: %+v", c) } } } // The condition also clears when the module is later assigned to the machine. func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() asksWithPaths(t) mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { t.Fatal(err) } if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "", "boom")); err != nil { t.Fatal(err) } settlePending(ctx, open, time.Now()) key := pendingObservation(pendingOf(t, open)[0]).Key() register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { t.Fatal(err) } settlePending(ctx, open, time.Now()) if _, found, _ := conditionsFrom.Get(ctx, key); found { t.Fatal("the condition stayed open after the module was assigned") } } // Known and not built, asked with build: the build is asked from where the last one was, the request kept as // assign's, and the assignment kept pending on the new build. func TestAssignWithBuildAsksForAKnownModule(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sensors", Ref: "main", For: "build", At: time.Now().Add(-time.Hour)}); err != nil { t.Fatal(err) } if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", Path: "modules/sensors", On: "anchor", Failed: "no space left on device"}); err != nil { t.Fatal(err) } asked := asksWithPaths(t) said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") if err != nil { t.Fatal(err) } if len(*asked) != 1 || (*asked)[0] != [3]string{"novox/mesh-catalog", "modules/sensors", "main"} { t.Fatalf("asked %v", *asked) } for _, want := range []string{"known and not registered", "no space left on device", "build b-modules/sensors of novox/mesh-catalog@main (modules/sensors) is asked for now", "kept as pending"} { if !strings.Contains(said, want) { t.Fatalf("the answer does not say %q:\n%s", want, said) } } pending := pendingOf(t, open) if len(pending) != 1 || pending[0].Build != "b-modules/sensors" { t.Fatalf("pending: %+v", pending) } requests, _ := inv.RequestsNamed(ctx, "sensors") if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" { t.Fatalf("the request is not kept as assign's: %+v", requests) } } // **build "true" with a pending assignment already waiting** (review point 5): the waiting one is made to // wait for the new build, and no build is asked that nothing waits on. func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil { t.Fatal(err) } if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil { t.Fatal(err) } // Waiting on the failed build, its expiry not yet settled. if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors", Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil { t.Fatal(err) } _, err := assign(ctx, open, "laptop", "sensors") if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") { t.Fatalf("known and not built with a pending assignment waiting: %v", err) } asked := asksWithPaths(t) said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") { t.Fatalf("assign with build: %q %v", said, err) } if len(*asked) != 1 { t.Fatalf("asked %v", *asked) } pending := pendingOf(t, open) if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting { t.Fatalf("pending: %+v", pending) } // The new build in flight now: build "true" again asks nothing. if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil || !strings.Contains(said, "no second build was asked") || len(*asked) != 1 { t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked) } } // **A failed ask never reads as in flight** (review point 2): a merge whose ask could not be made keeps the // request as not asked, and assign says the merge could not ask; a request whose asker could not hand it over // or stopped waiting reads the same, unless its outcome was heard. func TestAFailedAskIsNotABuildInFlight(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { return "", errors.New("cannot submit a build: nats: timeout") } t.Cleanup(func() { askABuild = was }) mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") _, err := assign(ctx, open, "laptop", "sensors") if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") { t.Fatalf("a merge that could not ask: %v", err) } if len(pendingOf(t, open)) != 0 { t.Fatal("a pending assignment waits on a build never asked") } if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil { t.Fatal(err) } if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil { t.Fatal(err) } _, err = assign(ctx, open, "laptop", "gauges") if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") { t.Fatalf("a build not handed over: %v", err) } // Heard first, the outcome stands: marking it afterwards changes nothing. if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main", Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil { t.Fatal(err) } _, err = assign(ctx, open, "laptop", "gauges") if err == nil || !strings.Contains(err.Error(), "failed: the real failure") { t.Fatalf("an outcome heard after a failed wait: %v", err) } } // A plan's tier keeps its requests as the plan's (review point 3). func TestAPlansBuildRequestIsThePlans(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() asksWithPaths(t) if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { t.Fatal(err) } m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} if err := (following{open: open}).SourceMoved(ctx, m); err != nil { t.Fatal(err) } requests, err := open.inventory.RequestsNamed(ctx, "app") if err != nil || len(requests) != 1 || requests[0].For != "plan" { t.Fatalf("the plan's request: %+v %v", requests, err) } } // **A build heard as built and not registered yet is in flight** (review point 4): the outcome is recorded // before the module is registered; for that moment assign keeps the assignment pending. Past the grace it is // recorded and not registered, and the tick ends the pending assignment with that. func TestABuildBeingRegisteredIsInFlight(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil { t.Fatal(err) } if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main", Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil { t.Fatal(err) } said, err := assign(ctx, open, "laptop", "sensors") if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") { t.Fatalf("a build being registered: %q %v", said, err) } settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute)) p := pendingOf(t, open) if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") { t.Fatalf("past the grace: %+v", p) } } // Unknown: refused as no module of that name, claiming only what was looked at, with the closest names; build // asks for nothing. func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() register(t, open, catalogue.Manifest{Module: "sensord", Version: "1"}) asked := asksWithPaths(t) _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") if !errors.Is(err, inventory.ErrNoSuchModule) { t.Fatalf("an unknown module: %v", err) } for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name", "no build request the controller kept (the last 30 days)", "the closest names it holds: sensord", "asks for nothing here"} { if !strings.Contains(err.Error(), want) { t.Fatalf("the refusal does not say %q:\n%v", want, err) } } if strings.Contains(err.Error(), "merge") { t.Fatalf("the refusal claims a merge it never looked at: %v", err) } if len(*asked) != 0 || len(pendingOf(t, open)) != 0 { t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open)) } } // Several modules in one act, one of them not registered: nothing is assigned and nothing kept pending, and // each missing one is said. func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() asksWithPaths(t) register(t, open, catalogue.Manifest{Module: "known", Version: "1"}) mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") _, err := assign(ctx, open, "laptop", "known", "sensors") if err == nil || !strings.Contains(err.Error(), "nothing was assigned") || !strings.Contains(err.Error(), "being built") { t.Fatalf("an act naming a module in flight: %v", err) } if slices.Contains(assignedTo(t, open, "laptop"), "known") || len(pendingOf(t, open)) != 0 { t.Fatal("an act refused whole was made in part") } } // unassign withdraws a waiting pending assignment; the build goes on and registers the module assigned // nowhere: a withdrawn row is never claimed. func TestUnassignWithdrawsAPendingAssignment(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() asksWithPaths(t) mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { t.Fatal(err) } said, err := unassign(ctx, open, "laptop", "sensors") if err != nil || !strings.Contains(said, "withdrawn") { t.Fatalf("unassign of a pending assignment: %q %v", said, err) } if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil { t.Fatal(err) } settlePending(ctx, open, time.Now()) if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { t.Fatal("a withdrawn assignment was made") } } // **A claim is never overridden** (review point 1): a pending assignment being made is claimed // (waiting → applying); an unassign meanwhile is refused and says so, and leaves the claim; making a row // already withdrawn makes nothing. A claim left by a controller that stopped is settled by the tick from what // the mesh holds: back to waiting when the module is not assigned, applied when it is. func TestAWithdrawalNeverOverridesAClaim(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory asksWithPaths(t) mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { t.Fatal(err) } p := pendingOf(t, open)[0] if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { t.Fatalf("claim: %v %v", ok, err) } _, err := unassign(ctx, open, "laptop", "sensors") if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") { t.Fatalf("unassign of a claimed pending assignment: %v", err) } if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying { t.Fatalf("the claim was overridden: %+v", got) } // Pending(zero) is the waiting ones alone (review point 7). if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 { t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err) } // Left by a controller that stopped: back to waiting, since sensors is not assigned. settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting { t.Fatalf("a stale claim was not released: %+v", got) } // Withdrawn, then the build registers it: applyPending finds nothing to claim. if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { t.Fatal(err) } register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" { t.Fatalf("a withdrawn pending assignment was made: %s", line) } if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { t.Fatal("a withdrawn pending assignment was assigned") } // A stale claim whose module was assigned is applied. q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) if err != nil { t.Fatal(err) } if ok, _ := inv.ClaimPending(ctx, q.ID); !ok { t.Fatal("claim") } if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { t.Fatal(err) } settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) rows, _ := inv.PendingFor(ctx, "anchor", "sensors") if len(rows) != 1 || rows[0].State != inventory.PendingApplied { t.Fatalf("a stale claim of an assigned module: %+v", rows) } } // A build that says nothing within the bound: the pending assignment expires at the next tick, saying so, // rather than waiting for ever; and a module registered by a process that kept no pending assignment is // assigned at the next tick. func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory for _, r := range []inventory.BuildRequest{ {ID: "build-lost", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/lost", Ref: "main", For: "merge", At: time.Now().Add(-buildRequestBound / 2)}, {ID: "build-heard", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/heard", Ref: "main", For: "merge", At: time.Now()}, } { if err := inv.RecordBuildRequest(ctx, r); err != nil { t.Fatal(err) } } for _, m := range []string{"lost", "heard"} { if _, err := assign(ctx, open, "laptop", m); err != nil { t.Fatal(err) } } register(t, open, catalogue.Manifest{Module: "heard", Version: "1"}) settlePending(ctx, open, time.Now().Add(buildRequestBound)) if !slices.Contains(assignedTo(t, open, "laptop"), "heard") { t.Fatal("a module registered meanwhile was not assigned at the next tick") } byModule := map[string]inventory.PendingAssignment{} for _, p := range pendingOf(t, open) { byModule[p.Module] = p } if p := byModule["lost"]; p.State != inventory.PendingExpired || !strings.Contains(p.Note, "no outcome of build build-lost") { t.Fatalf("a build that said nothing: %+v", p) } if p := byModule["heard"]; p.State != inventory.PendingApplied { t.Fatalf("a module registered meanwhile: %+v", p) } // Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is // still open (second review, bug B). if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil { t.Fatal(err) } said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) rows, err := inv.Pending(ctx, time.Unix(0, 0)) if err != nil { t.Fatal(err) } for _, r := range rows { if !r.Open() && (r.Raised == nil || r.Cleared != nil) { t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said) } } if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") { t.Fatalf("the applied row was not deleted: %v", said) } } // The seat's assign passes build on; unassign takes none. func TestTheSeatsAssignPassesBuild(t *testing.T) { argv, err := argvFor("assign", map[string]any{"node": "g14", "module": "sensors", "build": "true"}) if err != nil || !slices.Equal(argv, []string{"assign", "g14", "sensors", "--build"}) { t.Fatalf("assign with build: %v %v", argv, err) } if _, err := argvFor("unassign", map[string]any{"node": "g14", "module": "sensors", "build": "true"}); err == nil { t.Fatal("unassign took build") } } // The condition's words are plain. func TestAnAssignmentNotMadeIsSaidPlainly(t *testing.T) { o := pendingObservation(inventory.PendingAssignment{Node: "g14", Module: "sensors", Note: "the pending assignment of sensors to g14 expired: build b-1 of modules/sensors failed: boom"}) w := plainWordings[kindPendingEnded](o) if why, ok := conditions.PlainWords(w, "g14"); !ok { t.Fatalf("not plain: %s %+v", why, w) } if w.Headline != "sensors was not put on g14" { t.Fatalf("headline %q", w.Headline) } } // **One row's condition is never another's** (second review, bug A). The sequence: the first pending // assignment's build fails and the tick raises it; the person assigns again with build "true", a second // pending assignment; its build fails too before the next tick. That tick raises the second, and must not // clear it by judging the first answered by the second: each row has its own condition, and a newer row that // itself ended unmade answers nothing. func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() asked := 0 was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { asked++ return fmt.Sprintf("b-%d", asked), nil } t.Cleanup(func() { askABuild = was }) mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { t.Fatal(err) } failedBuild(t, open, "b-1") settlePending(ctx, open, time.Now()) first := pendingOf(t, open)[0] if !conditionOpen(t, first) { t.Fatal("the first assignment not made raised nothing") } if _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil { t.Fatal(err) } failedBuild(t, open, "b-2") settlePending(ctx, open, time.Now()) var second inventory.PendingAssignment for _, p := range pendingOf(t, open) { if p.Build == "b-2" { second = p } } if second.State != inventory.PendingExpired { t.Fatalf("the second: %+v", second) } if !conditionOpen(t, second) { t.Fatal("the second assignment's condition was cleared in the tick that raised it") } if !conditionOpen(t, rowOf(t, open, first.ID)) { t.Fatal("the first was judged answered by a second that itself was not made") } // unassign takes both back, and the next tick clears both. if said, err := unassign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(said, "b-1") || !strings.Contains(said, "b-2") { t.Fatalf("unassign: %q %v", said, err) } settlePending(ctx, open, time.Now()) if conditionOpen(t, first) || conditionOpen(t, second) { t.Fatal("a condition stayed open after both were taken back") } } // **A raised row outlives the pruning until its condition clears** (second review, bug B); and a machine's // removal, which takes its rows with it, clears their conditions at the next tick. func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory asksWithPaths(t) mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { t.Fatal(err) } failedBuild(t, open, "b-modules/sensors") settlePending(ctx, open, time.Now()) row := pendingOf(t, open)[0] settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) if rowOf(t, open, row.ID).Raised == nil || !conditionOpen(t, row) { t.Fatal("a raised row was pruned, or its condition closed, while the condition was open") } if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { t.Fatal(err) } settlePending(ctx, open, time.Now()) settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] { t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err) } // On anchor, then anchor removed: the row goes, and its condition with it at the next tick. if _, err := assign(ctx, open, "anchor", "sensors"); err == nil { t.Fatal("known and not built was not refused") } q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) if err != nil { t.Fatal(err) } if _, err := inv.SettlePending(ctx, q.ID, inventory.PendingWaiting, inventory.PendingExpired, "boom"); err != nil { t.Fatal(err) } settlePending(ctx, open, time.Now()) q.Node = "anchor" if !conditionOpen(t, q) { t.Fatal("not raised") } if _, err := inv.RemoveNodeForTest(ctx, "anchor"); err != nil { t.Fatal(err) } settlePending(ctx, open, time.Now()) if conditionOpen(t, q) { t.Fatal("a removed machine's condition stayed open") } } // **A claim left by a controller that stopped** (second review): the tick settles it from what the mesh // holds — back to waiting when the module is not assigned there, applied when it is — and leaves a fresh // claim alone. func TestAStaleClaimIsSettledFromWhatTheMeshHolds(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory claimed := func(node string) inventory.PendingAssignment { p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: "sensors", Build: "b-x"}) if err != nil { t.Fatal(err) } if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { t.Fatalf("claim: %v %v", ok, err) } return p } notAssigned, assigned := claimed("laptop"), claimed("anchor") register(t, open, catalogue.Manifest{Module: "unrelated", Version: "1"}) if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "sensors", Version: "1"}, inventory.Source{}); err != nil { t.Fatal(err) } if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { t.Fatal(err) } settlePending(ctx, open, time.Now()) if rowOf(t, open, notAssigned.ID).State != inventory.PendingApplying || rowOf(t, open, assigned.ID).State != inventory.PendingApplying { t.Fatal("a fresh claim was settled") } settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) if got := rowOf(t, open, assigned.ID); got.State != inventory.PendingApplied { t.Fatalf("a stale claim of an assigned module: %+v", got) } // Released to waiting, and in the same tick made, since sensors is registered now. if got := rowOf(t, open, notAssigned.ID); got.State != inventory.PendingApplied || !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { t.Fatalf("a stale claim of a module not assigned: %+v", got) } } // **A waited build whose asker stopped waiting is asked, outcome unknown** (second review): it may still run, // so it reads as in flight until its outcome or its bound, never as not asked; a build never handed over // is not asked. func TestABuildNoLongerWaitedForIsStillInFlight(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory for _, id := range []string{"build-timeout", "build-nothandedover"} { dir := "modules/" + strings.TrimPrefix(id, "build-") if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-catalog", Seat: "git", Path: dir, Ref: "main", For: "build"}); err != nil { t.Fatal(err) } } markWaitFailed(ctx, "build-timeout", errors.New("no build machine answered within 10m0s")) markWaitFailed(ctx, "build-nothandedover", fmt.Errorf("%w: cannot submit a build: nats: timeout", link.ErrNotHandedOver)) said, err := assign(ctx, open, "laptop", "timeout") if err != nil || !strings.Contains(said, "being built") || !strings.Contains(said, "kept as pending") { t.Fatalf("a build no longer waited for: %q %v", said, err) } if _, err := assign(ctx, open, "laptop", "nothandedover"); err == nil || !strings.Contains(err.Error(), "was not handed over") { t.Fatalf("a build never handed over: %v", err) } settlePending(ctx, open, time.Now().Add(buildRequestBound+time.Minute)) rows, _ := inv.PendingFor(ctx, "laptop", "timeout") if len(rows) != 1 || rows[0].State != inventory.PendingExpired || !strings.Contains(rows[0].Note, "its asker stopped waiting") { t.Fatalf("past the bound: %+v", rows) } }