jochen
2bfa6ae4a0
A send carries the machine's whole declaration, so at the switch to roll the next send of anything would have carried the old default's backlog, unjudged, to every machine. A gated send now carries and judges everything waiting on its machine; every other send is refused or leaves the machine; a release plan walks what waits one machine at a time, the control node last, and one that fails holds the next until a person releases it.
258 lines
8.9 KiB
Go
258 lines
8.9 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/lease"
|
|
)
|
|
|
|
// The backlog the old default left — builds registered and sent nowhere — is released by a plan, one
|
|
// machine at a time, each judged, never by the next send of something else (novox/hq ADR 0236).
|
|
|
|
type backlogMesh struct {
|
|
open *stores
|
|
sent [][]string
|
|
broken map[string]bool
|
|
}
|
|
|
|
// aBacklog is a mesh where `app` moved c1 → c2 on anchor and laptop, `late` moved on laptop only, and
|
|
// `same` was rebuilt with the very artifacts it had: two machines heard, every send applied at once.
|
|
func aBacklog(t *testing.T) *backlogMesh {
|
|
t.Helper()
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
b := &backlogMesh{open: open, broken: map[string]bool{}}
|
|
withConditionsInMemory(t)
|
|
was := doctorFrom
|
|
doctorFrom = nil
|
|
t.Cleanup(func() { doctorFrom = was })
|
|
|
|
build := func(module, commit, made string, asked time.Time) {
|
|
manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
|
|
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit,
|
|
Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked,
|
|
Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: made}}}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{
|
|
Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit,
|
|
Asked: asked}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute)
|
|
on := map[string][]string{"app": {"anchor", "laptop"}, "late": {"laptop"}, "same": {"anchor", "laptop"}}
|
|
for _, m := range []string{"app", "late", "same"} {
|
|
build(m, "c1", "sha256:"+m+"-1", old)
|
|
for _, n := range on[m] {
|
|
if _, err := inv.Assign(ctx, n, m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
for _, n := range []string{"anchor", "laptop"} {
|
|
sent := map[string]string{}
|
|
for m, nodes := range on {
|
|
for _, x := range nodes {
|
|
if x == n {
|
|
sent[m] = "c1"
|
|
}
|
|
}
|
|
}
|
|
if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
build("app", "c2", "sha256:app-2", now)
|
|
build("late", "c2", "sha256:late-2", now)
|
|
build("same", "c2", "sha256:same-1", now) // rebuilt, the same bytes
|
|
|
|
assigned := func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) {
|
|
modules, err := open.inventory.Assigned(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sent, known, err := open.inventory.SentBuilds(ctx, node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return f.moves(node, modules, sent, known, all), nil
|
|
}
|
|
wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts
|
|
machineMoves = assigned
|
|
releaseHeard = func(context.Context, *stores) (map[string]bool, error) {
|
|
return map[string]bool{"anchor": true, "laptop": true}, nil
|
|
}
|
|
n := 0
|
|
sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) {
|
|
b.sent = append(b.sent, append([]string(nil), names...))
|
|
current, err := open.inventory.CurrentBuilds(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, node := range names {
|
|
modules, _ := open.inventory.Assigned(ctx, node)
|
|
carried := map[string]string{}
|
|
for _, m := range modules {
|
|
carried[m] = current[m].Commit
|
|
}
|
|
n++
|
|
digest := fmt.Sprintf("d-%s-%d", node, n)
|
|
if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil {
|
|
return nil, err
|
|
}
|
|
if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node,
|
|
Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return names, nil
|
|
}
|
|
gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) {
|
|
f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{},
|
|
rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}}
|
|
reports, err := open.inventory.LastReports(ctx)
|
|
if err != nil {
|
|
return f, err
|
|
}
|
|
for _, r := range reports {
|
|
if b.broken[r.Node] {
|
|
r.Outcome = inventory.OutcomeFailed
|
|
}
|
|
f.reports[r.Node] = r
|
|
}
|
|
return f, nil
|
|
}
|
|
wasSettle, wasEvery, wasBound := gateSettle, gateEvery, gateBound
|
|
gateSettle, gateEvery = 0, time.Hour
|
|
t.Cleanup(func() {
|
|
machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather
|
|
gateSettle, gateEvery, gateBound = wasSettle, wasEvery, wasBound
|
|
})
|
|
return b
|
|
}
|
|
|
|
func (b *backlogMesh) release(t *testing.T) inventory.Plan {
|
|
t.Helper()
|
|
plans, err := b.open.inventory.RecentPlans(t.Context(), 10)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, p := range plans {
|
|
if p.Release != nil {
|
|
return p
|
|
}
|
|
}
|
|
t.Fatal("no release plan")
|
|
return inventory.Plan{}
|
|
}
|
|
|
|
// The backlog goes out one machine at a time: the first judged before the second is sent, each build's
|
|
// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused.
|
|
func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) {
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
|
|
f, err := readMoveFacts(ctx, inv)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
moves, _ := machineMoves(ctx, b.open, f, "laptop", false)
|
|
var names []string
|
|
for _, mv := range moves {
|
|
names = append(names, mv.Module)
|
|
}
|
|
if !reflect.DeepEqual(names, []string{"app", "late"}) {
|
|
t.Fatalf("laptop waits for %v; a rebuild with the same bytes is no move", names)
|
|
}
|
|
// Nothing else may carry them: a send that judges nothing is refused.
|
|
if _, err := ungatedIn(ctx, b.open, []string{"laptop"}, ""); !errors.Is(err, errUngated) {
|
|
t.Fatalf("a send that judges nothing would carry them: %v", err)
|
|
}
|
|
|
|
advancePlans(ctx, b.open)
|
|
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) {
|
|
t.Fatalf("sent %v: the first machine alone, before it is judged", b.sent)
|
|
}
|
|
p := b.release(t)
|
|
if !reflect.DeepEqual(p.Release.Order, []string{"anchor", "laptop"}) || p.Release.Gate == nil {
|
|
t.Fatalf("the release plan is %+v", p.Release)
|
|
}
|
|
gateEvery = 0
|
|
for i := 0; i < 4; i++ {
|
|
advancePlans(ctx, b.open)
|
|
}
|
|
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"laptop"}}) {
|
|
t.Fatalf("sent %v", b.sent)
|
|
}
|
|
p = b.release(t)
|
|
if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) {
|
|
t.Fatalf("the release plan is %s: %s %+v", p.State, p.Note, p.Release)
|
|
}
|
|
for _, build := range []string{"build-app-c2", "build-late-c2"} {
|
|
if v, found, err := inv.GateOf(ctx, build); err != nil || !found || v.Verdict != inventory.GatePassed {
|
|
t.Fatalf("%s's pass was not kept: %+v %v %v", build, v, found, err)
|
|
}
|
|
}
|
|
// Nothing waits now, and nothing opens again.
|
|
if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil {
|
|
t.Fatalf("a release opened with nothing waiting: %+v %v", p, err)
|
|
}
|
|
}
|
|
|
|
// A release that fails on its first machine puts back what it carried there, goes no further, and the
|
|
// next one waits for a person, said as a condition; a person releases it with why.
|
|
func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) {
|
|
b := aBacklog(t)
|
|
ctx := t.Context()
|
|
inv := b.open.inventory
|
|
gateEvery = 0
|
|
b.broken["anchor"] = true
|
|
advancePlans(ctx, b.open)
|
|
|
|
p := b.release(t)
|
|
if p.State != inventory.PlanFailed {
|
|
t.Fatalf("the release is %s: %s", p.State, p.Note)
|
|
}
|
|
if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"anchor"}}) {
|
|
t.Fatalf("sent %v: the first machine, then the put-back to it, and nothing to laptop", b.sent)
|
|
}
|
|
if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" {
|
|
t.Fatalf("app is at %s, not put back", current["app"].Commit)
|
|
}
|
|
if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed {
|
|
t.Fatal("app's build is not marked failed at its gate")
|
|
}
|
|
// late still waits on laptop, and is not released on its own after a failure.
|
|
if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil {
|
|
t.Fatalf("a release opened after a failed one: %+v %v", p, err)
|
|
}
|
|
if obs := backlogObservation(); len(obs) != 1 || !strings.Contains(obs[0].Summary, "release-backlog") {
|
|
t.Fatalf("the held release is not said: %+v", obs)
|
|
}
|
|
b.broken["anchor"] = false
|
|
if err := backlogCommand(ctx, "release-backlog", []string{"--why", "anchor is fixed"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for i := 0; i < 4; i++ {
|
|
advancePlans(ctx, b.open)
|
|
}
|
|
if p := b.release(t); p.State != inventory.PlanDone || p.Release.By == "" {
|
|
t.Fatalf("the person's release is %s (%+v)", p.State, p.Release)
|
|
}
|
|
if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" {
|
|
t.Fatalf("late was not released to laptop: %v", sent)
|
|
}
|
|
}
|