228 lines
9.6 KiB
Go
228 lines
9.6 KiB
Go
//go:build integration
|
|
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/conditions"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
)
|
|
|
|
func TestReplay352(t *testing.T) {
|
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
|
judgeLeavesAsTheCore(t)
|
|
t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn)
|
|
t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging)
|
|
}
|
|
|
|
func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) {
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied
|
|
// 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on.
|
|
carried, _, _ := inv.SentBuilds(ctx, "anchor")
|
|
if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reports, _ := inv.LastReports(ctx)
|
|
for _, r := range reports {
|
|
if r.Node == "anchor" && r.Current {
|
|
t.Fatal("the fixture's newer send reads as reported")
|
|
}
|
|
}
|
|
gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here
|
|
for i := 0; i < 4; i++ {
|
|
advancePlans(ctx, b.open)
|
|
}
|
|
p := b.release(t)
|
|
if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") {
|
|
t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note)
|
|
}
|
|
if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") {
|
|
t.Fatalf("the gate does not keep what it sent: %+v", g)
|
|
}
|
|
if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed {
|
|
t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err)
|
|
}
|
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
|
t.Fatalf("app was put back to %s", current["app"].Commit)
|
|
}
|
|
}
|
|
|
|
func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) {
|
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
|
judgeLeavesAsTheCore(t)
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
advancePlans(ctx, b.open) // the release judges app c2 on anchor
|
|
_, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}})
|
|
if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") {
|
|
t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err)
|
|
}
|
|
if len(b.sent) != 1 {
|
|
t.Fatalf("sent %v", b.sent)
|
|
}
|
|
}
|
|
|
|
func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) {
|
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
|
judgeLeavesAsTheCore(t)
|
|
g := aGateMesh(t)
|
|
ctx := t.Context()
|
|
inv := g.open.inventory
|
|
advancePlans(ctx, g.open) // anchor is sent app c2 first
|
|
if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gateEvery = 0
|
|
advancePlans(ctx, g.open)
|
|
p := g.plan(t)
|
|
if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
|
|
t.Fatalf("the plan is %s: %s", p.State, p.Note)
|
|
}
|
|
// Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no
|
|
// rollback (a release may walk what the other send left waiting on anchor; that is not a put-back).
|
|
if r := p.Modules["app"].Gate.Rollback; r != "" {
|
|
t.Fatalf("a superseded judging made a rollback: %q", r)
|
|
}
|
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
|
t.Fatalf("app was put back to %s", current["app"].Commit)
|
|
}
|
|
if failed, _ := inv.GateFailed(ctx, "build-2"); failed {
|
|
t.Fatal("a superseded build was marked failed")
|
|
}
|
|
}
|
|
|
|
func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) {
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
advancePlans(ctx, b.open)
|
|
// Another send moves app on anchor to a build this gate does not judge.
|
|
carried, _, _ := inv.SentBuilds(ctx, "anchor")
|
|
carried["app"] = "c3"
|
|
if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gateEvery = 0
|
|
advancePlans(ctx, b.open)
|
|
p := b.release(t)
|
|
if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") {
|
|
t.Fatalf("the release is %s: %s", p.State, p.Note)
|
|
}
|
|
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) {
|
|
t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent)
|
|
}
|
|
if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found {
|
|
t.Fatal("a superseded judging kept a verdict")
|
|
}
|
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" {
|
|
t.Fatalf("app was put back to %s", current["app"].Commit)
|
|
}
|
|
}
|
|
|
|
func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) {
|
|
withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks)
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
keeper, _ := withConditionsInMemory(t)
|
|
told := &conditions.Told{}
|
|
was := doctorFrom
|
|
doctorFrom = &doctor{open: open, keeper: keeper, teller: told}
|
|
t.Cleanup(func() { doctorFrom = was })
|
|
wasSend := sendRollout
|
|
var sent [][]string
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
sent = append(sent, names)
|
|
return names, nil
|
|
}
|
|
t.Cleanup(func() { sendRollout = wasSend })
|
|
|
|
build := func(id, commit, digest string, asked time.Time) inventory.Build {
|
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit})
|
|
b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".",
|
|
Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle,
|
|
Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}}
|
|
if err := inv.RecordBuild(ctx, b); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit},
|
|
inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return b
|
|
}
|
|
previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour))
|
|
failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute))
|
|
if versionOfBuild(previous) != strings.Repeat("1", 12) {
|
|
t.Fatalf("the build's version is %q", versionOfBuild(previous))
|
|
}
|
|
applied, err := inv.SchemaApplied(ctx)
|
|
if err != nil || applied < 87 {
|
|
t.Fatalf("the store's schema reaches %d (%v)", applied, err)
|
|
}
|
|
// The build before never recorded what it reads: not proved, refused.
|
|
if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") {
|
|
t.Fatalf("an unknown reach: %v %q", ok, why)
|
|
}
|
|
// It reads less than the store has: refused, naming both.
|
|
if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") {
|
|
t.Fatalf("a reach behind the store: %v %q", ok, why)
|
|
}
|
|
// Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent.
|
|
at := time.Now().Add(-5 * time.Minute)
|
|
state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at}
|
|
p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at,
|
|
State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}}
|
|
if err := inv.SavePlan(ctx, &p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply")
|
|
if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") {
|
|
t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note)
|
|
}
|
|
if len(sent) != 0 {
|
|
t.Fatalf("sent %v: nothing is put back", sent)
|
|
}
|
|
if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit {
|
|
t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit)
|
|
}
|
|
if marked, _ := inv.GateFailed(ctx, failed.ID); !marked {
|
|
t.Fatal("the failed build is not marked failed at its gate")
|
|
}
|
|
open2, _ := keeper.Open(ctx)
|
|
var found bool
|
|
for _, c := range open2 {
|
|
if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2)
|
|
}
|
|
// Reaching the store: allowed.
|
|
if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok {
|
|
t.Fatalf("a build that reads the whole schema was refused: %q", why)
|
|
}
|
|
// A build with no bundle to know it by: refused.
|
|
if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") {
|
|
t.Fatalf("a build without a bundle: %v %q", ok, why)
|
|
}
|
|
}
|