Drop the plan state from the store and the assignments: migration 0108 and placement-only assignments (release 6, issue 1945 B9, B4)
This commit is contained in:
@@ -495,52 +495,6 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is
|
||||
// composed after it — never sent after it with the node still adopted. And the send inside the
|
||||
// flip is not made to wait on the flip's own hold.
|
||||
func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var heldElsewhere, heldHere error
|
||||
sendNodes = func(inner context.Context, open *stores, names []string) error {
|
||||
// Another caller cannot hold the node while the flip sends it.
|
||||
waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
||||
defer cancel()
|
||||
if release, err := open.inventory.HoldNodes(waiting, names); err == nil {
|
||||
release()
|
||||
heldElsewhere = errors.New("another caller held the node while the flip sent it")
|
||||
}
|
||||
// The flip's own send holds it without waiting on itself.
|
||||
_, release, err := holdNodes(inner, open, names)
|
||||
if err != nil {
|
||||
heldHere = err
|
||||
return err
|
||||
}
|
||||
release()
|
||||
return nil
|
||||
}
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if heldElsewhere != nil || heldHere != nil {
|
||||
t.Fatalf("%v %v", heldElsewhere, heldHere)
|
||||
}
|
||||
// And given back once it is done.
|
||||
release, err := open.inventory.HoldNodes(ctx, []string{"anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
|
||||
// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found,
|
||||
// not only its files, and the digest changes when any of them does.
|
||||
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
||||
@@ -611,33 +565,3 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
||||
t.Fatal("a refused flip changed something")
|
||||
}
|
||||
}
|
||||
|
||||
// An assignment cannot land between a preview and the flip that takes every module: assigning
|
||||
// holds the node, so it waits for whatever is converging it.
|
||||
func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll
|
||||
inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond
|
||||
defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }()
|
||||
|
||||
var whileFlipping error
|
||||
sendNodes = func(context.Context, *stores, []string) error {
|
||||
_, whileFlipping = assign(ctx, open, "anchor", "notes")
|
||||
return nil
|
||||
}
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !errors.Is(whileFlipping, inventory.ErrNodeBusy) {
|
||||
t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// busWaitBound is how long a module whose ask the bus did not answer is asked again at every look before the plan fails
|
||||
// (issue 1334).
|
||||
const busWaitBound = 10 * time.Minute
|
||||
|
||||
// busWaiting says an open plan has a module of its current tier that the bus did not answer an ask of: the look must come back
|
||||
// for it, as nothing else will when no build of the tier is in flight (issue 1334).
|
||||
func busWaiting(p inventory.Plan) bool {
|
||||
return len(busWaitingIn(p)) > 0
|
||||
}
|
||||
|
||||
// busWaitingIn are the modules of the plan's current tier left unasked because the bus did not answer, sorted.
|
||||
func busWaitingIn(p inventory.Plan) []string {
|
||||
if !p.Open() || p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "" && s.BusWaitFrom != nil {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// busWaitWords is how a verb says modules are waiting for the bus: not asked yet, and asked again by the plan at its next look.
|
||||
func busWaitWords(p inventory.Plan, modules []string) string {
|
||||
return fmt.Sprintf("%s not asked yet: the bus did not answer; %s asks again at its next look", strings.Join(modules, ", "), p.ID)
|
||||
}
|
||||
|
||||
// slicesWithout is names without those in drop, in order.
|
||||
func slicesWithout(names, drop []string) []string {
|
||||
var out []string
|
||||
for _, n := range names {
|
||||
if !slices.Contains(drop, n) {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||
// delivery goes on with the machines heard, says the others were away, and the away ones get the builds by the send
|
||||
// they are given when heard again (ADR 0353, ADR 0360 §5). Away is the silent check's one verdict, machineFacts.silent
|
||||
// over the watchman's presence (ADR 0433 §3), never a second rule; a sleeping machine is away once presence says so.
|
||||
|
||||
// awayOf is the machines out of touch at a moment: those the silent check calls silent (a sleeping machine
|
||||
// is away once presence says so). A machine
|
||||
// never heard from has not joined and is not away.
|
||||
func awayOf(machines []machineFacts, now time.Time) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, m := range machines {
|
||||
if m.silent(now) {
|
||||
out[m.name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeAway is the walk's reading of which machines are away as a tier is sent: every node with its last-heard time and
|
||||
// the watchman's verdict on its presence key. A bus that cannot be asked leaves none away, which is what a walk did
|
||||
// before: a machine is not passed on a reading that was not made.
|
||||
var judgeAway = func(ctx context.Context, open *stores, now time.Time) (map[string]bool, error) {
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer js.Close()
|
||||
machines := make([]machineFacts, len(nodes))
|
||||
for i, n := range nodes {
|
||||
machines[i] = machineFacts{name: n.Name, lastHeard: n.LastSeen}
|
||||
}
|
||||
if err := withPresence(ctx, js.Conn(), machines, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return awayOf(machines, now), nil
|
||||
}
|
||||
|
||||
// rememberAway keeps on the plan the machines of running that are away: sorted, each once.
|
||||
func rememberAway(p *inventory.Plan, running []string, away map[string]bool) {
|
||||
for _, n := range running {
|
||||
if away[n] && !slices.Contains(p.Away, n) {
|
||||
p.Away = append(p.Away, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(p.Away)
|
||||
}
|
||||
|
||||
// runningHeard is the machines of running that the walk waits on for a module: those a send of the walk reached, and
|
||||
// those it did not pass as away.
|
||||
func runningHeard(p inventory.Plan, running []string, s *inventory.PlanModule) []string {
|
||||
return slices.DeleteFunc(slices.Clone(running), func(n string) bool {
|
||||
if !slices.Contains(p.Away, n) {
|
||||
return false
|
||||
}
|
||||
_, sent := s.Rest[n]
|
||||
return !sent
|
||||
})
|
||||
}
|
||||
|
||||
// awaySaying is what a plan says of the machines it passed as away, empty when it passed none.
|
||||
func awaySaying(p inventory.Plan) string {
|
||||
var said []string
|
||||
for _, n := range p.Away {
|
||||
said = append(said, n+" was away: sent when it is heard again")
|
||||
}
|
||||
return strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
// judgedAway reads which machines are away for a tier about to be sent, and says when it cannot.
|
||||
func judgedAway(ctx context.Context, open *stores, now time.Time) map[string]bool {
|
||||
away, err := judgeAway(ctx, open, now)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "which machines are away cannot be read, so none is passed over: %v\n", err)
|
||||
return nil // empty-on-error: none is passed over on a reading that was not made; the error is said just above
|
||||
}
|
||||
return away
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||
// walk judges it with the silent check's one verdict, machineFacts.silent, and a sleeping machine is away once presence says so.
|
||||
|
||||
// 1161/AC1: away is the silent check's verdict; AC4: a machine heard is not away.
|
||||
func TestAMachineIsAwayAsTheSilentCheckJudgesIt(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
machines := []machineFacts{
|
||||
{name: "heard", lastHeard: now.Add(-time.Minute)},
|
||||
{name: "quiet", lastHeard: now.Add(-silentBound - time.Minute)},
|
||||
{name: "stuck", lastHeard: now, presence: presenceSilent},
|
||||
{name: "back", lastHeard: now.Add(-time.Hour), presence: presenceHeard},
|
||||
{name: "new"},
|
||||
}
|
||||
got := awayOf(machines, now)
|
||||
for node, want := range map[string]bool{"heard": false, "quiet": true, "stuck": true, "back": false,
|
||||
"new": false} {
|
||||
if got[node] != want {
|
||||
t.Errorf("1161/AC1: %s away is %v; want %v (away: %v)", node, got[node], want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC1, AC2: the plan keeps the machines of the tier it passed as away, sorted and once.
|
||||
func TestAPlanKeepsTheMachinesItPassedAsAway(t *testing.T) {
|
||||
p := &inventory.Plan{}
|
||||
away := map[string]bool{"g14": true, "ace": true}
|
||||
rememberAway(p, []string{"anchor", "g14"}, away)
|
||||
rememberAway(p, []string{"g14", "ace"}, away)
|
||||
if !slices.Equal(p.Away, []string{"ace", "g14"}) {
|
||||
t.Fatalf("1161/AC2: the plan keeps %v away; want ace, g14", p.Away)
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC2: the plan says it, where its other passed machines are said.
|
||||
func TestAPlanSaysTheMachinesItPassedAsAway(t *testing.T) {
|
||||
say := awaySaying(inventory.Plan{Away: []string{"g14"}})
|
||||
if !strings.Contains(say, "g14 was away: sent when it is heard again") {
|
||||
t.Fatalf("1161/AC2: the plan says %q", say)
|
||||
}
|
||||
if got := awaySaying(inventory.Plan{}); got != "" {
|
||||
t.Fatalf("1161/AC2: a plan that passed no machine says %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC1: the gate does not wait for a machine passed as away, unless a send reached it; AC4: a machine heard is waited for.
|
||||
func TestAGateDoesNotWaitForAMachinePassedAsAway(t *testing.T) {
|
||||
p := inventory.Plan{Away: []string{"g14"}}
|
||||
none := &inventory.PlanModule{}
|
||||
if got := runningHeard(p, []string{"anchor", "g14"}, none); !slices.Equal(got, []string{"anchor"}) {
|
||||
t.Fatalf("1161/AC1: the gate waits on %v; want anchor alone", got)
|
||||
}
|
||||
sent := &inventory.PlanModule{Rest: map[string]inventory.SentDeclaration{"g14": {Digest: "d"}}}
|
||||
if got := runningHeard(p, []string{"anchor", "g14"}, sent); !slices.Equal(got, []string{"anchor", "g14"}) {
|
||||
t.Fatalf("1161/AC4: a machine a send reached is waited for: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -150,16 +150,6 @@ func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild)
|
||||
return notRegistered(err)
|
||||
}
|
||||
}
|
||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||
// and the next push would send it.
|
||||
if failed, err := inv.GateFailed(ctx, kept.ID); err != nil {
|
||||
return manifest, err
|
||||
} else if failed {
|
||||
return manifest, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
|
||||
"back: it is recorded and not registered again — a newer build is", j.by, manifest.Module,
|
||||
short(j.commit))
|
||||
}
|
||||
// Then the module is judged against the list the mesh publishes, as every manifest read again is: one naming a
|
||||
// key that list does not hold, or of another type, is refused here and never enters the catalogue.
|
||||
if j.carried != nil {
|
||||
@@ -283,8 +273,4 @@ type answers struct {
|
||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||
pending []inventory.PendingAssignment
|
||||
pendingUnread string
|
||||
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
|
||||
// issue 485); lastSentUnread why it could not be read.
|
||||
lastSent map[string]inventory.Send
|
||||
lastSentUnread string
|
||||
}
|
||||
|
||||
@@ -524,7 +524,6 @@ type registrations interface {
|
||||
SourceIdentity(ctx context.Context, module string) (int64, error)
|
||||
Catalogued(ctx context.Context) ([]inventory.Entry, error)
|
||||
BuildRequestByID(ctx context.Context, id string) (inventory.BuildRequest, bool, error)
|
||||
MergeOf(ctx context.Context, repository, commit string) (inventory.BatchedMerge, bool, error)
|
||||
}
|
||||
|
||||
// mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in.
|
||||
|
||||
@@ -129,41 +129,6 @@ func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "evil"})
|
||||
fork.Commit = "c0ffee0123456789" // the commit sudo was registered at
|
||||
keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false)
|
||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("the fork's build was taken in: %v", err)
|
||||
}
|
||||
// Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older
|
||||
// than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main.
|
||||
failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "2"})
|
||||
failed.Commit = "badbadbad0123456"
|
||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, _, err := inv.BuildByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found && previous.ID == fork.ID {
|
||||
t.Fatalf("a rollback would put back the fork's build %s", previous.ID)
|
||||
}
|
||||
if !found || previous.ID != "build-sudo" {
|
||||
t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
@@ -305,10 +270,11 @@ func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var forgeMerge keptMerge
|
||||
_ = json.Unmarshal(event, &forgeMerge)
|
||||
wasForge := keptForgeMerge
|
||||
t.Cleanup(func() { keptForgeMerge = wasForge })
|
||||
keptForgeMerge = func(context.Context, string, string, string) (keptMerge, bool, error) { return forgeMerge, true, nil }
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git",
|
||||
Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil {
|
||||
|
||||
@@ -7,14 +7,9 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
||||
@@ -165,43 +160,3 @@ func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
||||
t.Fatalf("a send that asserted everything left the condition open: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
||||
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
||||
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
// The controller starting, before either was assigned.
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
|
||||
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, nil)
|
||||
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
||||
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
|
||||
t.Fatal("a send that asserted everything raised a condition")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,7 +54,6 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
||||
// The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build
|
||||
// (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes.
|
||||
// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
|
||||
go copySentBuildsSoon(ctx, inv)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -62,39 +61,6 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
||||
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
|
||||
const kindSentBuildsNotCopied = "sent-builds-not-copied"
|
||||
|
||||
// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon
|
||||
// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails,
|
||||
// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key
|
||||
// that names a commit and empties each column it has copied.
|
||||
func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||
failed := false
|
||||
for {
|
||||
_, err := inv.CopySentBuilds(ctx)
|
||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied,
|
||||
Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "}
|
||||
if err != nil {
|
||||
fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err)
|
||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
||||
o.Summary += err.Error()
|
||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if keeper := conditionsFrom; keeper != nil && failed {
|
||||
_, _ = keeper.Clear(ctx, o.Key(), "copied")
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(chainMoveRetry):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// chainMoveRetry is how long the copy waits before it tries again.
|
||||
const chainMoveRetry = 30 * time.Second
|
||||
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The links a rollout makes (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): the send of a plan's tier puts a
|
||||
// build on a machine; the assignment of the module there names the build, its commit and the rollout (written by the record of
|
||||
// the send, inventory.RecordSent, from the context a send is made in), and the rollout lists the assignments it changed with the
|
||||
// build before and after.
|
||||
|
||||
// rolloutModules is what a plan sends of each module: the commit and the build of it.
|
||||
func rolloutModules(p *inventory.Plan) map[string]inventory.RolloutModule {
|
||||
out := make(map[string]inventory.RolloutModule, len(p.Modules))
|
||||
for m, s := range p.Modules {
|
||||
if s != nil && s.Commit != "" {
|
||||
out[m] = inventory.RolloutModule{Commit: s.Commit, Build: s.Build}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sendingAs is the context a plan's send is made in: what is recorded of the send names this rollout.
|
||||
func sendingAs(ctx context.Context, p *inventory.Plan) context.Context {
|
||||
return inventory.WithRollout(ctx, p.ID, rolloutModules(p))
|
||||
}
|
||||
|
||||
// ranBefore is what each module on a machine ran before a send, as the chain names it: its assignment's build and commit,
|
||||
// else the commit the machine was last sent for it. Read before the send, which writes the new ones.
|
||||
func ranBefore(ctx context.Context, inv *inventory.Inventory, node string) map[string]chain.Ref {
|
||||
was := map[string]chain.Ref{}
|
||||
if sent, known, err := inv.SentBuilds(ctx, node); err == nil && known {
|
||||
for m, commit := range sent {
|
||||
was[m] = chain.Ref{Commit: commit}
|
||||
}
|
||||
}
|
||||
if ran, err := inv.AssignmentRuns(ctx, node); err == nil {
|
||||
for m, r := range ran {
|
||||
was[m] = r
|
||||
}
|
||||
}
|
||||
return was
|
||||
}
|
||||
|
||||
// noteChanges writes into a plan that a send to a machine changed the assignments of the modules it carried: the build and
|
||||
// commit before and after, the builds the rollout carried, and the step taken. A change noted already for the machine and the
|
||||
// module is replaced, so a send made again says it once.
|
||||
func noteChanges(p *inventory.Plan, node string, modules []string, was map[string]chain.Ref, at time.Time) {
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
change := chain.Change{Node: node, Module: m, Before: was[m], After: chain.Ref{Build: s.Build, Commit: s.Commit}}
|
||||
if at := slices.IndexFunc(p.Changed, func(c chain.Change) bool { return c.Node == node && c.Module == m }); at >= 0 {
|
||||
change.Before = p.Changed[at].Before
|
||||
p.Changed[at] = change
|
||||
} else {
|
||||
p.Changed = append(p.Changed, change)
|
||||
}
|
||||
if s.Build != "" && !slices.Contains(p.Builds, s.Build) {
|
||||
p.Builds = append(p.Builds, s.Build)
|
||||
}
|
||||
}
|
||||
if !slices.ContainsFunc(p.Steps, func(s chain.Step) bool { return s.Tier == p.Tier && s.Node == node }) {
|
||||
sent := at.UTC()
|
||||
p.Steps = append(p.Steps, chain.Step{Tier: p.Tier, Node: node, Sent: &sent})
|
||||
}
|
||||
}
|
||||
|
||||
// refSaid is a build and the commit it was made from as a person reads them; "nothing" for a module that ran none.
|
||||
func refSaid(r chain.Ref) string {
|
||||
switch {
|
||||
case r.Build == "" && r.Commit == "":
|
||||
return "nothing"
|
||||
case r.Build == "":
|
||||
return "commit " + short(r.Commit)
|
||||
}
|
||||
return r.Build + " (" + short(r.Commit) + ")"
|
||||
}
|
||||
|
||||
// changedLines say the assignments a rollout changed: the machine, the module, and the build before and after.
|
||||
func changedLines(p inventory.Plan) []string {
|
||||
if len(p.Changed) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := []string{" changed:"}
|
||||
for _, c := range p.Changed {
|
||||
out = append(out, fmt.Sprintf(" %s/%s %s → %s", c.Node, c.Module, refSaid(c.Before), refSaid(c.After)))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// runLines say what each module assigned to a machine runs there, as its assignment names it: the build, the commit, the
|
||||
// rollout that put it there and since when.
|
||||
func runLines(assigned []chain.Assignment) []string {
|
||||
var out []string
|
||||
for _, a := range assigned {
|
||||
if a.Build == "" && a.Commit == "" {
|
||||
continue
|
||||
}
|
||||
line := fmt.Sprintf(" %-22s %s", a.Module, refSaid(chain.Ref{Build: a.Build, Commit: a.Commit}))
|
||||
if a.Rollout != "" {
|
||||
line += ", sent by " + a.Rollout
|
||||
}
|
||||
if !a.Since.IsZero() {
|
||||
line += " at " + a.Since.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]string{" what they run, as the chain names it:"}, out...)
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **A rollout lists the assignments it changed, each with the build before and after** (ADR 0408 §1, issue 1095): one
|
||||
// change for a machine and a module however often the send is noted, the build before kept from the first, the builds the
|
||||
// rollout carried once each, and one step for a machine and a tier.
|
||||
func TestARolloutListsTheAssignmentsItChangedWithTheBuildBeforeAndAfter(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Tier: 1, Modules: map[string]*inventory.PlanModule{
|
||||
"notes": {Commit: "c2", Build: "b2"}, "wiki": {Commit: "c2w", Build: "b2w"}, "nothing": nil}}
|
||||
was := map[string]chain.Ref{"notes": {Build: "b1", Commit: "c1"}, "wiki": {Commit: "c1w"}}
|
||||
|
||||
noteChanges(&p, "anchor", []string{"notes", "wiki", "nothing"}, was, at)
|
||||
noteChanges(&p, "laptop", []string{"notes"}, nil, at)
|
||||
// The send noted again, after what was sent had become what the machine runs: the first before stands.
|
||||
noteChanges(&p, "anchor", []string{"notes"}, map[string]chain.Ref{"notes": {Build: "b2", Commit: "c2"}}, at.Add(time.Minute))
|
||||
|
||||
want := []chain.Change{
|
||||
{Node: "anchor", Module: "notes", Before: chain.Ref{Build: "b1", Commit: "c1"}, After: chain.Ref{Build: "b2", Commit: "c2"}},
|
||||
{Node: "anchor", Module: "wiki", Before: chain.Ref{Commit: "c1w"}, After: chain.Ref{Build: "b2w", Commit: "c2w"}},
|
||||
{Node: "laptop", Module: "notes", After: chain.Ref{Build: "b2", Commit: "c2"}},
|
||||
}
|
||||
if len(p.Changed) != len(want) {
|
||||
t.Fatalf("changed %+v", p.Changed)
|
||||
}
|
||||
for k := range want {
|
||||
if p.Changed[k] != want[k] {
|
||||
t.Errorf("change %d is %+v, want %+v", k, p.Changed[k], want[k])
|
||||
}
|
||||
}
|
||||
if len(p.Builds) != 2 || p.Builds[0] != "b2" || p.Builds[1] != "b2w" {
|
||||
t.Errorf("builds %v", p.Builds)
|
||||
}
|
||||
if len(p.Steps) != 2 || p.Steps[0].Node != "anchor" || p.Steps[0].Tier != 1 || p.Steps[0].Sent == nil || p.Steps[1].Node != "laptop" {
|
||||
t.Errorf("steps %+v", p.Steps)
|
||||
}
|
||||
lines := changedLines(p)
|
||||
if len(lines) != 4 || !strings.Contains(lines[1], "anchor/notes b1 (c1) → b2 (c2)") ||
|
||||
!strings.Contains(lines[3], "laptop/notes nothing → b2 (c2)") {
|
||||
t.Errorf("changed lines %q", lines)
|
||||
}
|
||||
if got := changedLines(inventory.Plan{}); got != nil {
|
||||
t.Errorf("a plan that changed nothing says %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatAMachineRunsIsSaidAsItsAssignmentNamesIt(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
lines := runLines([]chain.Assignment{
|
||||
{Node: "anchor", Module: "notes", Build: "b2", Commit: "c2c2c2c2c2c2", Rollout: "plan-1", Since: at},
|
||||
{Node: "anchor", Module: "web", Commit: "c3"},
|
||||
{Node: "anchor", Module: "never-sent"},
|
||||
})
|
||||
if len(lines) != 3 || !strings.Contains(lines[1], "notes") || !strings.Contains(lines[1], "b2 (c2c2c2c2)") ||
|
||||
!strings.Contains(lines[1], "sent by plan-1") || !strings.Contains(lines[2], "web") || !strings.Contains(lines[2], "commit c3") {
|
||||
t.Fatalf("lines %q", lines)
|
||||
}
|
||||
if runLines([]chain.Assignment{{Node: "a", Module: "m"}}) != nil {
|
||||
t.Fatal("a module never sent is said to run something")
|
||||
}
|
||||
}
|
||||
@@ -248,25 +248,11 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento
|
||||
if out.Epoch, err = inv.SentEpoch(ctx, n.ID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.Generation, err = inv.SentGeneration(ctx, n.ID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.ReadsModules, err = inv.ReadsModules(ctx, n.ID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
// Every node-engine of release 6 reads a module per resource and a build per module.
|
||||
out.ReadsModules = true
|
||||
if out.SentBuilds, out.SentKnown, err = inv.SentBuilds(ctx, n.Name); err != nil {
|
||||
return out, err
|
||||
}
|
||||
runs, err := inv.AssignmentRuns(ctx, n.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if len(runs) > 0 {
|
||||
out.Runs = make(map[string]compose.Run, len(runs))
|
||||
for module, ref := range runs {
|
||||
out.Runs[module] = compose.Run{Commit: ref.Commit, Build: ref.Build}
|
||||
}
|
||||
}
|
||||
stated, has, err := inv.HealthOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
|
||||
@@ -243,54 +243,6 @@ func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696
|
||||
}
|
||||
}
|
||||
|
||||
// fallbackDelivery refuses the first declaration it is sent with the bucket's failure, as a bus that has not yet been given the
|
||||
// controller's grant does, and takes the second.
|
||||
type fallbackDelivery struct {
|
||||
bodies [][]byte
|
||||
first error
|
||||
}
|
||||
|
||||
func (f *fallbackDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *fallbackDelivery) declare(_ context.Context, _ readyNode, body []byte) (string, error) {
|
||||
f.bodies = append(f.bodies, body)
|
||||
if len(f.bodies) == 1 && f.first != nil {
|
||||
return "", fmt.Errorf("x: %w: %w", errFilesNotPut, f.first)
|
||||
}
|
||||
return "digest", nil
|
||||
}
|
||||
|
||||
// 1696/AC5: when the bucket cannot be written yet (the grant travels in the declaration of the machine holding the bus), that send goes
|
||||
// with every content inline and is made; a send that fails for another reason is not retried inline.
|
||||
func TestASendWhoseFilesCannotBePutGoesInlineOnce(t *testing.T) { // 1696/AC5
|
||||
users := contentOf("users", 100<<10)
|
||||
s := readyNode{node: "novox", declared: sendable{fileReferences: true, Resources: []map[string]any{aFileResource("nats.bus-users", users)}}}
|
||||
f := &fallbackDelivery{first: errors.New("timeout")}
|
||||
digests, err := deliver(t.Context(), f, "novox", []readyNode{s})
|
||||
if err != nil || digests["novox"] != "digest" || len(f.bodies) != 2 {
|
||||
t.Fatalf("%v %v %d", digests, err, len(f.bodies))
|
||||
}
|
||||
if strings.Contains(string(f.bodies[0]), users[:100]) || !strings.Contains(string(f.bodies[1]), users[:100]) {
|
||||
t.Error("the first body should name the file by digest and the second carry it inline")
|
||||
}
|
||||
other := &fallbackDelivery{}
|
||||
other.first = nil
|
||||
failing := &failingDelivery{err: errors.New("the bus is down")}
|
||||
if _, err := deliver(t.Context(), failing, "novox", []readyNode{s}); err == nil || failing.calls != 1 {
|
||||
t.Fatalf("a failure of another kind: %v after %d send(s)", err, failing.calls)
|
||||
}
|
||||
}
|
||||
|
||||
type failingDelivery struct {
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *failingDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *failingDelivery) declare(context.Context, readyNode, []byte) (string, error) {
|
||||
f.calls++
|
||||
return "", f.err
|
||||
}
|
||||
|
||||
// A bucket for the sweep.
|
||||
type sweepBucketFake struct {
|
||||
objects map[string]time.Time // zero: named at the last look
|
||||
@@ -502,33 +454,3 @@ func whileHeld(node string) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC9: a send holds the node's lock from the put to the publication, so the sweep never removes an object between them. The
|
||||
// test goes through overTheBus.declare: while another holds the lock the declaration is not published, and at the publication the
|
||||
// lock is held.
|
||||
func TestASendHoldsTheNodesLockFromThePutToThePublication(t *testing.T) { // 1696/AC9
|
||||
const node = "lock-send-node"
|
||||
bus := &aBusThatSays{published: make(chan string, 1)}
|
||||
bus.heldAtPublish = func() bool { return whileHeld(node) }
|
||||
send := overTheBus{server: link.SendingOn(bus), signer: aSignerOfZeros{}}
|
||||
release := lockFilesOf(node) // a sweep is judging this node's bucket
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := send.declare(t.Context(), readyNode{node: node}, []byte(`{}`))
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case n := <-bus.published:
|
||||
t.Fatalf("%s was sent its declaration while the sweep held the node's bucket", n)
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-bus.published:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the send did not go on once the sweep let go")
|
||||
}
|
||||
if err := <-done; err == nil || !strings.Contains(err.Error(), "nothing is recorded") {
|
||||
t.Fatalf("the send did not reach the publication (a lock held at it): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||
//
|
||||
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||
// build's outcome — and summarised here per machine, repository or module.
|
||||
|
||||
// durationSummary is one subject's measurements of one kind.
|
||||
type durationSummary struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Count int `json:"count"`
|
||||
Median string `json:"median"`
|
||||
P90 string `json:"p90"`
|
||||
Max string `json:"max"`
|
||||
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||
Suggests string `json:"suggests,omitempty"`
|
||||
}
|
||||
|
||||
func summarise(ds []inventory.Duration) []durationSummary {
|
||||
type key struct{ kind, subject string }
|
||||
by := map[key][]time.Duration{}
|
||||
for _, d := range ds {
|
||||
k := key{d.Kind, d.Subject}
|
||||
by[k] = append(by[k], d.Took)
|
||||
}
|
||||
var out []durationSummary
|
||||
for k, took := range by {
|
||||
slices.Sort(took)
|
||||
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||
switch k.kind {
|
||||
case inventory.DurationApply:
|
||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||
if ki != kj {
|
||||
return ki < kj
|
||||
}
|
||||
return out[i].Subject < out[j].Subject
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func round(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Second:
|
||||
return d.Round(time.Millisecond).String()
|
||||
case d < time.Minute:
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
default:
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
|
||||
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||
func durationsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "the summary as data")
|
||||
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *all {
|
||||
body, err := json.MarshalIndent(ds, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
summary := summarise(ds)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(summary) == 0 {
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, "+
|
||||
"plan-tier and build durations as it hears them\n", *days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||
for _, s := range summary {
|
||||
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||
if s.Suggests != "" {
|
||||
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,266 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The assignment generation a declaration was composed from, and the record of every send (novox/hq
|
||||
// issue 234).
|
||||
//
|
||||
// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the
|
||||
// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and
|
||||
// cannot tell a later send that carries an older view of the assignments. So a declaration now carries
|
||||
// the generation of the assignments it was composed from — a counter the store raises in the same
|
||||
// transaction as every assignment change — and a machine refuses one older than it applied, unless it is
|
||||
// a gate's put-back. And every send is written down with who sent it, from which generation and naming
|
||||
// which modules: the controller logged no send then, and which process sent the stale declaration could
|
||||
// not be read back from anything the mesh kept.
|
||||
|
||||
// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from.
|
||||
const kindOlderGeneration = "older-generation"
|
||||
|
||||
// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is.
|
||||
const generationRefusalsSaid = advisoryQuiet
|
||||
|
||||
// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and
|
||||
// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the
|
||||
// record of the send whether or not the machine is sent them.
|
||||
//
|
||||
// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its
|
||||
// declaration is allotted here like any other and carries the generation as it stands — never older than
|
||||
// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never
|
||||
// needs to send one.
|
||||
func (o order) stamp(d *sendable) {
|
||||
d.Sequence, d.Epoch = o.sequence, o.epoch
|
||||
d.composedFrom, d.actingEpoch = o.generation, o.acting
|
||||
d.Generation = 0
|
||||
if o.readsGeneration && o.generation > 0 {
|
||||
d.Generation = o.generation
|
||||
}
|
||||
}
|
||||
|
||||
// sentRecord is what the record of a send keeps beside its digest.
|
||||
type sentRecord struct {
|
||||
sequence int64
|
||||
epoch uint64
|
||||
generation int64
|
||||
// toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine
|
||||
// whose node-engine has not said it reads one.
|
||||
toldGeneration int64
|
||||
sender string
|
||||
modules []string
|
||||
// plans is each module the send handed to the provisioner, its plan, as JSON (design 57 P6); nil for none.
|
||||
plans []byte
|
||||
}
|
||||
|
||||
// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
|
||||
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
|
||||
r := sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
|
||||
toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
|
||||
if len(d.plans) > 0 {
|
||||
kept := map[string]keptPlan{}
|
||||
for module, p := range d.plans {
|
||||
kept[module] = keptPlan{Plan: p, RequiredBy: d.requiredBy[module]}
|
||||
}
|
||||
// A map of plain strings and slices: it marshals.
|
||||
r.plans, _ = json.Marshal(kept)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// keptPlan is a handed module's plan as a send keeps it (migration 0100): the plan, and for a module only pulled in
|
||||
// by another, the module that requires it (issue 765, inventory.SentProvisioningPlans).
|
||||
type keptPlan struct {
|
||||
catalogue.Plan
|
||||
RequiredBy string `json:"required-by,omitempty"`
|
||||
}
|
||||
|
||||
// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
|
||||
// caller the controller ran this command for, or the account at the shell.
|
||||
func callerOf(ctx context.Context) string {
|
||||
if c := link.CallerIn(ctx); c != "" {
|
||||
return c
|
||||
}
|
||||
return link.Caller()
|
||||
}
|
||||
|
||||
// senderOf is a send's sender in words: who asked, and the process and build that composed it — the
|
||||
// answer issue 234 could not find, because two controllers and a one-shot push were all sending then.
|
||||
func senderOf(caller string) string {
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version)
|
||||
}
|
||||
|
||||
// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender
|
||||
// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that
|
||||
// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the
|
||||
// mesh cannot attribute is the louder fact, not a quieter one.
|
||||
//
|
||||
// **And raises the mesh's counter past what the machine applied, when the refused send was composed from
|
||||
// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind
|
||||
// the machine, which is a store put back from a backup — and every send after would be refused for ever. A
|
||||
// stale sender's generation is below the counter, and the counter is left alone for it.
|
||||
//
|
||||
// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that
|
||||
// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185.
|
||||
// S20 names `push <node>` for that case instead.
|
||||
func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) {
|
||||
r := report.OlderGeneration
|
||||
if r == nil || inv == nil || report.Node == "" {
|
||||
return
|
||||
}
|
||||
node, err := inv.NodeByName(ctx, report.Node)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+
|
||||
"read, so it is not raised: %v\n", report.Node, err)
|
||||
return
|
||||
}
|
||||
var raised int64
|
||||
if now, err := inv.AssignmentGeneration(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+
|
||||
"read: %v\n", report.Node, err)
|
||||
} else if counterBehind(*r, now) {
|
||||
if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+
|
||||
"and could not be raised: %v\n", now, report.Node, r.Applied, err)
|
||||
raised = 0
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+
|
||||
"a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n",
|
||||
now, report.Node, r.Applied, raised, report.Node)
|
||||
}
|
||||
}
|
||||
send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised)
|
||||
if err == nil && !found {
|
||||
send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence,
|
||||
Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied,
|
||||
CounterRaisedTo: raised})
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
|
||||
"kept, so it is not raised: %v\n", report.Node, report.Sequence, err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
|
||||
"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
|
||||
}
|
||||
|
||||
// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the
|
||||
// refused send carried the counter as it stands now, and the machine applied more than that.
|
||||
func counterBehind(r link.GenerationRefusal, now int64) bool {
|
||||
return r.Generation >= now && r.Applied > r.Generation
|
||||
}
|
||||
|
||||
// modulesWords is a send's modules as a sentence says them.
|
||||
func modulesWords(modules []string) string {
|
||||
if len(modules) == 0 {
|
||||
return "no module the mesh recorded"
|
||||
}
|
||||
return strings.Join(modules, ", ")
|
||||
}
|
||||
|
||||
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
|
||||
// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
|
||||
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
|
||||
s, found, err := inv.LastSend(ctx, node)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(w, "%s:\n", node)
|
||||
for _, line := range lastSendLines(s) {
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
|
||||
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
|
||||
// machine answered of it — the refusal in its own words.
|
||||
func lastSendLines(s inventory.Send) []string {
|
||||
const stamp = "2006-01-02 15:04:05"
|
||||
epoch := "under no lease epoch"
|
||||
if s.Epoch > 0 {
|
||||
epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
|
||||
}
|
||||
generation := "no assignment generation recorded"
|
||||
if s.Generation > 0 {
|
||||
generation = fmt.Sprintf("assignment generation %d", s.Generation)
|
||||
}
|
||||
lines := []string{
|
||||
fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
|
||||
fmt.Sprintf(" by %s", s.Sender),
|
||||
fmt.Sprintf(" %s, composed from %s", epoch, generation),
|
||||
fmt.Sprintf(" naming %s", modulesWords(s.Modules)),
|
||||
}
|
||||
answer := func(words string) { lines = append(lines, " its answer "+words) }
|
||||
switch a := s.Answer; {
|
||||
case s.RefusedAt != nil:
|
||||
// Before the report: a refusal for the generation is kept on the send itself and is the machine's
|
||||
// answer to this send in particular.
|
||||
answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
|
||||
s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
|
||||
case a.Outcome == "" || a.At == nil:
|
||||
answer("the machine has not reported on it yet")
|
||||
case a.Outcome == inventory.OutcomeApplied:
|
||||
answer("applied it at " + a.At.Local().Format(stamp))
|
||||
case a.Outcome == inventory.OutcomeRefused:
|
||||
answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
|
||||
for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
|
||||
lines = append(lines, " "+strings.TrimSpace(line))
|
||||
}
|
||||
default:
|
||||
answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
|
||||
a.At.Local().Format(stamp), a.Failed))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
|
||||
func sendAnswerWord(s inventory.Send) string {
|
||||
switch a := s.Answer; {
|
||||
case s.RefusedAt != nil:
|
||||
return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
|
||||
case a.Outcome == "" || a.At == nil:
|
||||
return "not reported on yet"
|
||||
case a.Outcome == inventory.OutcomeFailed:
|
||||
return fmt.Sprintf("failed (%d resource(s))", a.Failed)
|
||||
default:
|
||||
return a.Outcome
|
||||
}
|
||||
}
|
||||
|
||||
// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
|
||||
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
|
||||
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
|
||||
if unread != "" {
|
||||
fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
|
||||
return
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Println("last sent, per machine:")
|
||||
for _, n := range nodes {
|
||||
s, found := sends[n.Name]
|
||||
if !found {
|
||||
fmt.Printf(" %-12s no send recorded\n", n.Name)
|
||||
continue
|
||||
}
|
||||
generation := "no generation"
|
||||
if s.Generation > 0 {
|
||||
generation = fmt.Sprintf("generation %d", s.Generation)
|
||||
}
|
||||
fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
|
||||
generation, sendAnswerWord(s))
|
||||
}
|
||||
fmt.Printf("\n `node show <node>` says a machine's last send in full\n\n")
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1394/AC1 of C9 (ADR 0528 clause 2): when the controller has issued mesh-x's credential for fingerprint F (mesh-x
|
||||
// registered at a build of F, its account minted, the bus's machine sent its declaration with the user list, through
|
||||
// overTheBus.declare as a push sends it), then grants.mesh-x.F exists on the bus,
|
||||
// and a second pass writes it again as nothing new.
|
||||
func TestTheGrantOfAnIssuedCredentialIsOnTheBus(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
fp := grantFP('d')
|
||||
m := catalogue.Manifest{Module: "mesh-x", Version: "1", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh-x/broker"}}}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-x1", Repository: "novox/mesh-x", Module: "mesh-x", Commit: "c0ffee1",
|
||||
SourceFingerprint: fp}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-x", BuiltFrom: "c0ffee1", Head: "c0ffee1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: "mesh-x"}.Username()
|
||||
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule, Node: "anchor", Module: "mesh-x"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.OnConn(conn).EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The live path: a send of the bus's machine's declaration carrying the user list (overTheBus.declare), which records
|
||||
// the list sent and then writes the grants it issued.
|
||||
legacy, err := conn.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = js.DeleteStream(ctx, "GRANTS_DECLARED")
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "GRANTS_DECLARED", Subjects: []string{link.DeclareSubject("anchor")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.DeleteStream(context.Background(), "GRANTS_DECLARED") })
|
||||
send := overTheBus{open: open, server: link.SendingOn(link.OverNATS{Conn: conn, JS: legacy}), signer: aSignerOfZeros{}}
|
||||
ready := readyNode{node: "anchor", declared: sendable{BusUsers: "the user list the bus's machine is sent"}}
|
||||
if _, err := send.declare(ctx, ready, []byte(`{}`)); err != nil {
|
||||
t.Fatalf("the send of the bus's machine: %v", err)
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, broker.GrantsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := sdkdelivery.ControllerGrantKey("mesh-x", fp)
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("no grant %s after the list was issued: %v", key, err)
|
||||
}
|
||||
g, err := sdkdelivery.DecodeControllerGrant(e.Value())
|
||||
if err != nil || len(g.Accounts) != 1 || g.Accounts[0] != user {
|
||||
t.Fatalf("the grant reads %+v, %v; want the account %s", g, err, user)
|
||||
}
|
||||
if _, err := send.declare(ctx, ready, []byte(`{}`)); err != nil {
|
||||
t.Fatalf("the second send: %v", err)
|
||||
}
|
||||
if again, _ := kv.Get(ctx, key); again.Revision() != e.Revision() {
|
||||
t.Errorf("a second pass wrote the grant again (revision %d, was %d)", again.Revision(), e.Revision())
|
||||
}
|
||||
}
|
||||
@@ -4,9 +4,6 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
|
||||
@@ -27,20 +24,6 @@ func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The summary of durations says, per kind and subject, what a bound would be set from.
|
||||
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||
var ds []inventory.Duration
|
||||
for i := 1; i <= 10; i++ {
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||
Took: time.Duration(i) * time.Second})
|
||||
}
|
||||
got := summarise(ds)
|
||||
if len(got) != 1 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses
|
||||
// the cause, so a repair cannot pass for a drill by the word it gives.
|
||||
func TestADrillIsRecordedThroughItsOwnVerb(t *testing.T) {
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// heldKey carries the nodes this call already holds, so an act that holds a node and then sends
|
||||
// through sendTo does not wait on itself.
|
||||
type heldKey struct{}
|
||||
|
||||
// holdNodes holds the named nodes for composing and sending their declarations (novox/hq ADR
|
||||
// 0100), skipping any the context already holds, and returns a context that says it holds them.
|
||||
// Release gives back only what this call took.
|
||||
func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) {
|
||||
already, _ := ctx.Value(heldKey{}).(map[string]bool)
|
||||
var take []string
|
||||
for _, n := range names {
|
||||
if !already[n] {
|
||||
take = append(take, n)
|
||||
}
|
||||
}
|
||||
if len(take) == 0 {
|
||||
return ctx, func() {}, nil
|
||||
}
|
||||
release, err := open.inventory.HoldNodes(ctx, take)
|
||||
if err != nil {
|
||||
return ctx, nil, err
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for n := range already {
|
||||
held[n] = true
|
||||
}
|
||||
for _, n := range take {
|
||||
held[n] = true
|
||||
}
|
||||
return context.WithValue(ctx, heldKey{}, held), release, nil
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 348: on 2026-10-09 the control node's resolver stopped answering on its private address
|
||||
// at 10:57:57 UTC; the machine's network condition was raised at 10:58:45. The node-engine and the
|
||||
// controller were sent at 10:59:34. The new node-engine's first statement judged the names once — unknown,
|
||||
// "one look failed; a second decides" — and that statement cleared the condition, though its last evidence
|
||||
// still said "connection refused". The next look raised it again at 11:00:23, after the send, and both
|
||||
// builds failed their gate at 11:10 with "raised since it was sent" and were put back, for a fault that
|
||||
// began before they were sent.
|
||||
|
||||
// TestReplay348 replays the statements of the outage: the condition raised before the send is not
|
||||
// cleared by the restarted engine's first, undecided statement, and the gate does not count it against
|
||||
// the builds sent after it began.
|
||||
|
||||
// A release walks its modules without a record per module: D10 counts what its tier names as rolling,
|
||||
// so the node-engine a release walks is not "behind, and no plan is rolling it out" on its first machine.
|
||||
func TestAReleaseRollsOutWhatItsTierNames(t *testing.T) {
|
||||
plans := []inventory.Plan{
|
||||
{ID: "release-1", State: inventory.PlanRolling, Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
{ID: "plan-2", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"letta": {}}},
|
||||
}
|
||||
got := rollingModules(plans)
|
||||
if !got["mesh-host"] || !got["letta"] || len(got) != 2 {
|
||||
t.Fatalf("rolling: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"})
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
t.Fatalf("recording a send after cancellation failed: %v", err)
|
||||
}
|
||||
outstanding, err := inv.Outstanding(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outstanding != digest || digest != digestOf(body) {
|
||||
t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest)
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A declaration composed earlier is numbered lower than one composed later, whatever order the two
|
||||
// are sent in (novox/hq issue 204). The number used to be taken at send time, after composing, so a
|
||||
// declaration composed before an assignment changed and sent after a newer one carried the higher
|
||||
// number — and the machine, which refuses a lower number, took the older content as the mesh's
|
||||
// newest word. Taken before the composition reads anything, the order of numbers is the order of
|
||||
// compositions, and the host's refusal does what it is for.
|
||||
func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *testing.T) {
|
||||
allot := numbered()
|
||||
var composed []string
|
||||
compose := func(stamp string) func(string) (sendable, error) {
|
||||
return func(node string) (sendable, error) {
|
||||
composed = append(composed, stamp)
|
||||
return sendable{Resources: []map[string]any{{"id": node + "." + stamp}}}, nil
|
||||
}
|
||||
}
|
||||
// Composed first — before an assignment changed — and sent last.
|
||||
stale, _ := composeEach([]string{"anchor"}, allot, compose("before"))
|
||||
// Composed after the change, sent first.
|
||||
fresh, _ := composeEach([]string{"anchor"}, allot, compose("after"))
|
||||
|
||||
if stale[0].declared.Sequence != 1 || fresh[0].declared.Sequence != 2 {
|
||||
t.Fatalf("the numbers do not follow the compositions: before=%d after=%d",
|
||||
stale[0].declared.Sequence, fresh[0].declared.Sequence)
|
||||
}
|
||||
// Sent in the other order, the numbers do not change — so the machine that has applied the
|
||||
// fresh one (2) refuses the stale one (1) when it arrives late.
|
||||
if !(stale[0].declared.Sequence < fresh[0].declared.Sequence) {
|
||||
t.Fatal("a declaration composed earlier must carry the lower number, however late it is sent")
|
||||
}
|
||||
if len(composed) != 2 || composed[0] != "before" {
|
||||
t.Fatalf("compositions happened in an unexpected order: %v", composed)
|
||||
}
|
||||
}
|
||||
|
||||
// The number is taken before the first read of the composition, not after it: an allotter that
|
||||
// fails leaves nothing composed for that machine, and the others are still composed.
|
||||
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
|
||||
calls := 0
|
||||
allot := func(node string) (order, error) {
|
||||
if node == "anchor" {
|
||||
return order{}, context.DeadlineExceeded
|
||||
}
|
||||
return order{sequence: 7}, nil
|
||||
}
|
||||
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
|
||||
calls++
|
||||
if node == "anchor" {
|
||||
t.Fatal("anchor was composed although its number could not be taken")
|
||||
}
|
||||
return sendable{}, nil
|
||||
})
|
||||
if calls != 1 || len(sending) != 1 || sending[0].node != "laptop" || sending[0].declared.Sequence != 7 {
|
||||
t.Fatalf("laptop should be composed with its number and anchor refused: %v / %v", sending, refusals)
|
||||
}
|
||||
if len(refusals) != 1 {
|
||||
t.Fatalf("anchor's failed number should be a refusal naming it: %v", refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// What was sent is written down even when the sender's context is already cancelled (issue 204): a
|
||||
// controller replaced mid-send had told the machine and never recorded it, so status read "applied,
|
||||
// current" over a machine that had just been sent something else.
|
||||
@@ -1,51 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
|
||||
Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
||||
Modules: []string{"docker", "sudo"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
|
||||
Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
|
||||
for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
|
||||
"docker, sudo", "unknown field \"generation\""} {
|
||||
if !strings.Contains(shown, want) {
|
||||
t.Errorf("node show laptop does not say %q:\n%s", want, shown)
|
||||
}
|
||||
}
|
||||
body := printed(t, func() error { return statusFor(ctx, open, true) })
|
||||
var got struct {
|
||||
LastSent []struct {
|
||||
Node string `json:"node"`
|
||||
Sender string `json:"sender"`
|
||||
Answer string `json:"answer"`
|
||||
} `json:"lastSent"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &got); err != nil {
|
||||
t.Fatalf("%v:\n%s", err, body)
|
||||
}
|
||||
if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
|
||||
got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
|
||||
t.Fatalf("status --json's last sends read %+v", got.LastSent)
|
||||
}
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
|
||||
// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
|
||||
// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
|
||||
// the store's reader may not read the controller's tables and these verbs are the only window onto them.
|
||||
|
||||
func aLastSend(answer inventory.SendAnswer) inventory.Send {
|
||||
sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
|
||||
return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
|
||||
Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
||||
Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
|
||||
}
|
||||
|
||||
func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
||||
asked := answers{
|
||||
nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
|
||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
|
||||
Outcome: inventory.OutcomeApplied, At: &at})},
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
var laptop, anchor []string
|
||||
for _, line := range strings.Split(shown, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) > 0 && fields[0] == "laptop" {
|
||||
laptop = append(laptop, line)
|
||||
}
|
||||
if len(fields) > 0 && fields[0] == "anchor" {
|
||||
anchor = append(anchor, line)
|
||||
}
|
||||
}
|
||||
if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
|
||||
!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
|
||||
!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
|
||||
t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
|
||||
}
|
||||
if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
|
||||
t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
|
||||
asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
|
||||
lastSentUnread: "the store went away"}
|
||||
if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
|
||||
t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Error("a mesh whose sends could not be read is called well")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
||||
refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
||||
Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
|
||||
for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
|
||||
"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
|
||||
if !strings.Contains(refused, want) {
|
||||
t.Errorf("node show's last send does not say %q:\n%s", want, refused)
|
||||
}
|
||||
}
|
||||
if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
|
||||
At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
|
||||
t.Errorf("an applied send is not said applied:\n%s", applied)
|
||||
}
|
||||
if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
|
||||
"not reported on it yet") {
|
||||
t.Errorf("a send not reported on is not said so:\n%s", waiting)
|
||||
}
|
||||
stale := aLastSend(inventory.SendAnswer{})
|
||||
when := at
|
||||
stale.RefusedAt, stale.RefusedApplied = &when, 44
|
||||
if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
|
||||
!strings.Contains(s, "generation 44") {
|
||||
t.Errorf("a send refused for its generation is not said so:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
|
||||
asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
||||
Refused: "unknown field", At: &at})}}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
LastSent []map[string]any `json:"lastSent"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.LastSent) != 1 {
|
||||
t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
|
||||
}
|
||||
s := got.LastSent[0]
|
||||
if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
|
||||
s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
|
||||
s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
|
||||
s["answeredAt"] == nil {
|
||||
t.Fatalf("laptop's last send reads %v", s)
|
||||
}
|
||||
if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
|
||||
t.Fatalf("the modules it named read %v", s["modules"])
|
||||
}
|
||||
}
|
||||
|
||||
// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
|
||||
// read back.
|
||||
@@ -135,11 +135,6 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(),
|
||||
State: inventory.PlanBuilding}
|
||||
if err := inv.SavePlan(ctx, &saved); err == nil {
|
||||
t.Fatal("B wrote a plan after losing the lease")
|
||||
}
|
||||
store := conditions.NewInMemory()
|
||||
keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store,
|
||||
Epoch: func() (uint64, error) { return b.epoch(ctx) }})
|
||||
|
||||
@@ -86,7 +86,7 @@ func run() error {
|
||||
case "builds":
|
||||
return buildsCommand(ctx, args[1:])
|
||||
case "forget-checks", "queue", "cancel", "clear", "kill", "replay", "pause", "resume", "plans", "plan", "push", "deliver",
|
||||
"delivery-stop", "delivery-backups-again", "delivery-walks", "delivery-order", "delivery", "upgrade":
|
||||
"delivery-stop", "delivery-backups-again", "delivery-walks", "delivery-order", "delivery":
|
||||
return removedByName(args[0])
|
||||
case "try":
|
||||
return tryCommand(ctx, args[1:])
|
||||
@@ -128,6 +128,8 @@ func run() error {
|
||||
case "facts":
|
||||
return factsCommand(ctx, args[1:])
|
||||
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0236).
|
||||
case "upgrade":
|
||||
return upgradeCommand(ctx, args[1:])
|
||||
case "bus":
|
||||
return busCommand(ctx, args[1:])
|
||||
case "overlay":
|
||||
@@ -163,7 +165,7 @@ func run() error {
|
||||
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
||||
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
||||
case "durations":
|
||||
return durationsCommand(ctx, args[1:])
|
||||
return removedByName(args[0])
|
||||
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||
case "conditions":
|
||||
return conditionsCommand(ctx, args[1:])
|
||||
|
||||
@@ -284,14 +284,14 @@ func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, as
|
||||
if !onForge {
|
||||
return "", errors.New("the repository is not on the mesh's forge")
|
||||
}
|
||||
merged, known, err := inv.MergeOf(ctx, owner+"/"+name, commit)
|
||||
merged, known, err := keptForgeMerge(ctx, owner, name, commit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !known {
|
||||
return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, commit)
|
||||
}
|
||||
return approvedAnnounced(ctx, inv, owner, name, announcedOf(merged))
|
||||
return approvedAnnounced(ctx, inv, owner, name, merged)
|
||||
}
|
||||
|
||||
// errWordNotAnApprove says the merge cannot be shown to be the operator's Approve of its head: its announcement names no
|
||||
@@ -343,15 +343,67 @@ func claimantsOf(entries []inventory.Entry, seat string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none.
|
||||
func announcedOf(m inventory.BatchedMerge) keptMerge {
|
||||
var k keptMerge
|
||||
_ = json.Unmarshal(m.Event, &k)
|
||||
return k
|
||||
// forgeMergeKey is where the forge's announcement of a merge is kept, by the merge commit: the one thing a build of a merge
|
||||
// names, from which the pull request and its head are found (novox/hq ADR 0420 §7).
|
||||
func forgeMergeKey(owner, repo, commit string) string {
|
||||
return fmt.Sprintf("merge/%s/%s/%s", owner, repo, commit)
|
||||
}
|
||||
|
||||
// keptMerge is a merge as the record keeps it: the forge's announcement, and the modules it moved when it was
|
||||
// heard. The announcement reads back as a SourceMoved alone, which ignores the rest.
|
||||
// keepForgeMerge keeps the forge's announcement of a merge in the bucket of the merges, a week at most as that bucket
|
||||
// keeps them. The controller plans nothing from a merge any more (ADR 0539 §17); this is all it keeps of one.
|
||||
var keepForgeMerge = func(ctx context.Context, m link.SourceMoved) error {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return fmt.Errorf("%w: the bus is not held", link.ErrTryAgain)
|
||||
}
|
||||
api, err := jetstream.New(serving.Conn())
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: the bucket of the merges: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
body, err := json.Marshal(keptMerge{SourceMoved: m})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := kv.Put(ctx, forgeMergeKey(m.Owner, m.Repo, m.Commit), body); err != nil {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keptForgeMerge is the forge's announcement of a merge, by its commit; false when none was kept. A variable so a test can
|
||||
// hand it one.
|
||||
var keptForgeMerge = func(ctx context.Context, owner, repo, commit string) (keptMerge, bool, error) {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return keptMerge{}, false, errors.New("only the serving controller keeps what the forge announced")
|
||||
}
|
||||
api, err := jetstream.New(serving.Conn())
|
||||
if err != nil {
|
||||
return keptMerge{}, false, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket)
|
||||
if err != nil {
|
||||
return keptMerge{}, false, err
|
||||
}
|
||||
e, err := kv.Get(ctx, forgeMergeKey(owner, repo, commit))
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return keptMerge{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return keptMerge{}, false, err
|
||||
}
|
||||
var k keptMerge
|
||||
if err := json.Unmarshal(e.Value(), &k); err != nil {
|
||||
return keptMerge{}, false, err
|
||||
}
|
||||
return k, true, nil
|
||||
}
|
||||
|
||||
// keptMerge is a merge as the record keeps it: the forge's announcement.
|
||||
type keptMerge struct {
|
||||
link.SourceMoved
|
||||
Moves []string `json:"moves,omitempty"`
|
||||
|
||||
@@ -103,8 +103,11 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh
|
||||
const commit = "c0ffee0123456789"
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
var forgeMerge link.SourceMoved
|
||||
if err := json.Unmarshal(event, &forgeMerge); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := keepForgeMerge(ctx, forgeMerge); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
@@ -19,7 +18,7 @@ import (
|
||||
type anEmptyMesh struct {
|
||||
request inventory.BuildRequest
|
||||
asked bool
|
||||
merge inventory.BatchedMerge
|
||||
merge keptMerge
|
||||
}
|
||||
|
||||
func (m *anEmptyMesh) Catalogue(context.Context) (map[string]catalogue.Manifest, error) {
|
||||
@@ -33,9 +32,6 @@ func (m *anEmptyMesh) Catalogued(context.Context) ([]inventory.Entry, error) { r
|
||||
func (m *anEmptyMesh) BuildRequestByID(_ context.Context, id string) (inventory.BuildRequest, bool, error) {
|
||||
return m.request, m.asked && id == m.request.ID, nil
|
||||
}
|
||||
func (m *anEmptyMesh) MergeOf(context.Context, string, string) (inventory.BatchedMerge, bool, error) {
|
||||
return m.merge, true, nil
|
||||
}
|
||||
|
||||
// aDeliveryMergeOf is the Approved merge of a pull request of novox/mesh-delivery, its heads keyed by that repository,
|
||||
// and the ask and warrant bound to exactly those heads (the digest the Approve left covers them).
|
||||
@@ -102,8 +98,12 @@ func TestANewModuleIsRegisteredOnTheOperatorsApproveOfItsMergeAndRefusedOtherwis
|
||||
}
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7,
|
||||
"head_sha": head, "merge_commit_sha": commit})
|
||||
mesh := &anEmptyMesh{asked: true, merge: inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main",
|
||||
Commit: commit, Merged: time.Now(), Event: event},
|
||||
var forgeMerge keptMerge
|
||||
_ = json.Unmarshal(event, &forgeMerge)
|
||||
wasForge := keptForgeMerge
|
||||
t.Cleanup(func() { keptForgeMerge = wasForge })
|
||||
keptForgeMerge = func(context.Context, string, string, string) (keptMerge, bool, error) { return forgeMerge, true, nil }
|
||||
mesh := &anEmptyMesh{asked: true,
|
||||
request: inventory.BuildRequest{ID: "build-1", Repository: "novox/mesh-delivery", Seat: "git", Path: "",
|
||||
Ref: "main", Commit: commit, For: c.request}}
|
||||
forms := &sourceForms{bases: map[string]string{}, base: func(seat string) string {
|
||||
|
||||
@@ -104,8 +104,11 @@ func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheB
|
||||
const commit = "c0ffee0123456789"
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
var forgeMerge link.SourceMoved
|
||||
if err := json.Unmarshal(event, &forgeMerge); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := keepForgeMerge(ctx, forgeMerge); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
|
||||
@@ -39,9 +39,6 @@ func TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild(t *testing.T)
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 {
|
||||
t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err)
|
||||
}
|
||||
// The indexer writes the module's trunk value; the builder builds it, then its key is done.
|
||||
fp := fpOf('7')
|
||||
manifest := b.indexed(t, "systemd-resolved", "1", "novox/mesh-catalog", "modules/systemd-resolved", commit, fp)
|
||||
|
||||
@@ -796,19 +796,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
if err := showMode(ctx, inv, node); err != nil {
|
||||
return err
|
||||
}
|
||||
// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
|
||||
// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
|
||||
// Unreadable is said, not taken for none.
|
||||
switch last, found, err := inv.LastSend(ctx, name); {
|
||||
case err != nil:
|
||||
fmt.Printf(" what it was last sent could NOT be read: %v\n", err)
|
||||
case !found:
|
||||
fmt.Printf(" last sent no send recorded since the mesh began keeping them\n")
|
||||
default:
|
||||
for _, line := range lastSendLines(last) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
||||
for _, line := range agentAccountLines(ctx, inv, node) {
|
||||
fmt.Println(line)
|
||||
@@ -915,12 +902,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
}
|
||||
if len(assigned) > 0 {
|
||||
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
||||
// And what each runs, as its assignment names it (novox/hq ADR 0408, issue 1095).
|
||||
if on, err := inv.AssignmentsOn(ctx, name); err == nil {
|
||||
for _, line := range runLines(on) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
// And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same
|
||||
// reason its condition says. Not computable is said, never read as "all applied".
|
||||
|
||||
@@ -10,7 +10,6 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -697,16 +696,6 @@ func namesWords(names []string, most int) string {
|
||||
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
|
||||
}
|
||||
|
||||
// planModules are the modules a walk moves, sorted.
|
||||
func planModules(p inventory.Plan) []string {
|
||||
out := make([]string, 0, len(p.Modules))
|
||||
for m := range p.Modules {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else
|
||||
// their count and the repository.
|
||||
func deliveryWhat(modules []string, repository string) string {
|
||||
@@ -727,18 +716,6 @@ func deliveryName(modules []string, repository string) string {
|
||||
return repoName(repository) + " delivery"
|
||||
}
|
||||
|
||||
// walkWaitingWords explains a walk waiting for its delivery's word.
|
||||
func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string {
|
||||
what := deliveryWhat(w.modules, w.repository)
|
||||
if severity == conditions.Urgent {
|
||||
return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+
|
||||
"delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in))
|
||||
}
|
||||
return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+
|
||||
"decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.",
|
||||
what, humanDuration(in), humanDuration(waitUrgentAfter))
|
||||
}
|
||||
|
||||
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
|
||||
// Start and Stop are also asked of the operator (novox/hq ADR 0259); the condition's own words keep saying
|
||||
// where they are given without a channel, and the ask's text drops that (askText).
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -278,182 +275,6 @@ func listensLines(m catalogue.Manifest) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
func planCommand(ctx context.Context, args []string) error {
|
||||
// One reading of the mesh for every machine this looks at.
|
||||
ctx = withInputCache(ctx)
|
||||
set := flag.NewFlagSet("plan", flag.ContinueOnError)
|
||||
// Because "one resource" does not tell you whether the settings landed. Being able to read
|
||||
// the file before it is sent is the difference between believing a merge worked and knowing.
|
||||
show := set.Bool("files", false, "print the files this node would be given")
|
||||
// The declaration exactly as the node would receive it. For handing to something else --
|
||||
// checking it against the host's own parser, most usefully, which is the only way to know
|
||||
// that what the control plane emits is what the host accepts.
|
||||
asJSON := set.Bool("json", false, "print the declaration this node would be sent")
|
||||
// What a push would change, against what the machine was last sent (novox/hq ADR 0217): read
|
||||
// before sending, so a change that removes, moves or recreates something is seen first.
|
||||
asDiff := set.Bool("diff", false, "what a push would change on this node, against what it was last sent")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("plan <node> [--files] [--json] [--diff]")
|
||||
}
|
||||
args = positionals
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
plan, settings, err := resolveNode(ctx, open, args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(plan.Modules) == 0 {
|
||||
fmt.Printf("%s is assigned nothing\n", args[0])
|
||||
return nil
|
||||
}
|
||||
// Which optional requirements are unbound (novox/hq ADR 0330 §7): beside the document on the
|
||||
// error stream where the answer is the declaration itself, which carries no such field — the
|
||||
// node-engine refuses an envelope key it does not know.
|
||||
if *asDiff || *asJSON {
|
||||
for _, line := range unboundLines(plan) {
|
||||
fmt.Fprintf(os.Stderr, "%s:%s\n", args[0], line)
|
||||
}
|
||||
}
|
||||
if *asDiff {
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A module left out is not in the body, so the diff alone would say "nothing would change" for
|
||||
// a module just assigned whose settings cannot compose — success-shaped silence. Said first, with
|
||||
// why, as push and the plain plan say it (novox/hq ADR 0163, rule 6).
|
||||
reportLeftOut(args[0], declared)
|
||||
return writePlanDiff(ctx, open.inventory, args[0], body)
|
||||
}
|
||||
if *asJSON {
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The bytes a push would send, indented: one marshaller, so `plan --json` cannot show an
|
||||
// envelope other than the one sent.
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var indented bytes.Buffer
|
||||
if err := json.Indent(&indented, body, "", " "); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(indented.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
// What the machine was last told, by whom and from which assignment generation (novox/hq issue 234),
|
||||
// beside what it would be told now. A record that cannot be read is said, not taken for none.
|
||||
if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil {
|
||||
fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err)
|
||||
}
|
||||
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
||||
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
||||
// without --json allocates nothing.
|
||||
if record, err := open.inventory.NodeByName(ctx, args[0]); err == nil {
|
||||
left := plan.LeftOut(settings, record.Adopted)
|
||||
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
||||
}
|
||||
// And which of its modules no provider will grant, because the identity overflows the bound of
|
||||
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
|
||||
for _, o := range plan.Overflowing() {
|
||||
fmt.Printf("%s: %s\n", args[0], o)
|
||||
}
|
||||
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
||||
// stored before its definition moved from under it.
|
||||
for _, m := range plan.Modules {
|
||||
for _, stray := range catalogue.UnusedSettings(m, settings[m.Module]) {
|
||||
fmt.Printf("%s: a setting reaches nothing — %s\n", args[0], stray)
|
||||
}
|
||||
}
|
||||
fmt.Printf("%s would run:\n", args[0])
|
||||
for _, m := range plan.Modules {
|
||||
fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module])
|
||||
for _, line := range listensLines(m) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
// What was assigned here and cannot run here. Said with the rest rather than as a refusal: it is
|
||||
// one module on the wrong machine, the others still run, and the remedy is to move this one.
|
||||
for _, u := range plan.Unhostable {
|
||||
for _, c := range u.Missing {
|
||||
fmt.Printf(" %-20s not applied — %s\n", u.Module, catalogue.WrongMachine(u.Module, c, args[0]))
|
||||
}
|
||||
}
|
||||
for _, c := range plan.Claims {
|
||||
fmt.Printf(" holds %s, one per %s\n", c.Claim, c.Scope)
|
||||
}
|
||||
// What this machine depends on that is not on it. Worth saying out loud: it is the only part
|
||||
// of a node's set that stops working when a *different* machine goes away, and nothing else
|
||||
// in this output would have told anybody that.
|
||||
for _, n := range plan.Needs {
|
||||
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
|
||||
// And a served key the provider left out, because its machine composes no name it asks for
|
||||
// (novox/hq ADR 0346 §3).
|
||||
for _, line := range n.LeftOut {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
}
|
||||
// What the providers on this machine leave out of what they serve, and why: said where the remedy
|
||||
// is (novox/hq ADR 0346 §3).
|
||||
for _, line := range plan.ServedLeftOut(settings) {
|
||||
fmt.Printf("%s: %s\n", args[0], line)
|
||||
}
|
||||
for _, line := range unboundLines(plan) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
// And a container's busy that this machine's node-engine is too old to be sent (novox/hq ADR 0335): its
|
||||
// container is replaced at once, as before. A statement that cannot be read is said as such.
|
||||
if reads, err := engineReadsBusy(ctx, open.inventory, args[0]); err != nil {
|
||||
fmt.Printf("%s: whether its node-engine reads a container's busy could not be read: %v\n", args[0], err)
|
||||
} else {
|
||||
for _, line := range busyNotSent(plan.Modules, reads) {
|
||||
fmt.Printf("%s: %s\n", args[0], line)
|
||||
}
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resources := declared.Resources
|
||||
for module, layers := range settings {
|
||||
for _, layer := range layers {
|
||||
fmt.Printf(" %-20s settings from %s\n", module, layer.From)
|
||||
}
|
||||
}
|
||||
// What each port reached from the local network resolved to on this machine (novox/hq ADR 0340 §8),
|
||||
// or that it resolved to none.
|
||||
for _, p := range declared.localNetwork {
|
||||
fmt.Println(" " + p.Said())
|
||||
}
|
||||
fmt.Printf("\n%d resource(s)\n", len(resources))
|
||||
|
||||
if *show {
|
||||
for _, r := range resources {
|
||||
content, ok := r["content"].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
fmt.Printf("\n--- %s ---\n%s", shownAs(r), content)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shownAs is the heading `plan --show` puts over a resource's content.
|
||||
//
|
||||
// **A file written into says so.** Its content is the mesh's part of a file that is otherwise the
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan
|
||||
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there after the
|
||||
// write; so no code but chainSavePlans (which SavePlansMoving calls, and no other function) writes a plan to the rollouts
|
||||
// bucket by anything but a change of the merges it holds. Checked by reading the sources, no store needed.
|
||||
func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||
write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`)
|
||||
var found []string
|
||||
for _, root := range []string{"../../cmd", "../../internal"} {
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
text := string(raw)
|
||||
// The regular expression is the costly part of reading every source: only a file that names the table can match.
|
||||
if strings.Contains(strings.ToLower(text), "release_plan") && write.MatchString(text) {
|
||||
found = append(found, path+" writes the plan table")
|
||||
}
|
||||
// A plan record is written by the bucket's raw create and update, which only the plans' own file calls.
|
||||
if filepath.ToSlash(path) != "../../internal/inventory/chain_rollouts.go" && filepath.ToSlash(path) != "../../internal/chain/chain.go" &&
|
||||
strings.Contains(text, "chain.RolloutsOf") {
|
||||
found = append(found, path+" reaches the rollouts bucket")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(found) > 0 {
|
||||
t.Fatalf("a plan is written without plan-moved being said:\n%s", strings.Join(found, "\n"))
|
||||
}
|
||||
raw, err := os.ReadFile("../../internal/inventory/plans.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(raw)
|
||||
save := strings.Index(text, "func (i *Inventory) SavePlansMoving")
|
||||
next := strings.Index(text[save:], "\nfunc ")
|
||||
if next < 0 {
|
||||
next = len(text) - save
|
||||
}
|
||||
if !strings.Contains(text[save:save+next], "chainSavePlans(") {
|
||||
t.Error("SavePlansMoving does not write through chainSavePlans")
|
||||
}
|
||||
chainRaw, err := os.ReadFile("../../internal/inventory/chain_rollouts.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chainText := string(chainRaw)
|
||||
at := strings.Index(chainText, "func (i *Inventory) chainSavePlans")
|
||||
end := strings.Index(chainText[at:], "\nfunc ")
|
||||
if !strings.Contains(chainText[at:at+end], "PlanSaved(i, *p)") {
|
||||
t.Error("chainSavePlans does not say each plan it wrote")
|
||||
}
|
||||
if n := strings.Count(chainText, "createRaw(ctx, c, p.ID"); n != 1 {
|
||||
t.Errorf("a plan is made from %d places of chain_rollouts.go, want the one in chainSavePlans", n)
|
||||
}
|
||||
}
|
||||
|
||||
// movesSaid records what the hook hears: for each plan, the revisions it was said at.
|
||||
type movesSaid struct {
|
||||
mu sync.Mutex
|
||||
revs map[string][]int64
|
||||
}
|
||||
|
||||
func (s *movesSaid) hear(_ *inventory.Inventory, p inventory.Plan) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.revs[p.ID] = append(s.revs[p.ID], p.Revision)
|
||||
}
|
||||
|
||||
// sayingEverySave installs a PlanSaved that records, as the serving controller's says plan-moved.
|
||||
func sayingEverySave(t *testing.T) *movesSaid {
|
||||
t.Helper()
|
||||
s := &movesSaid{revs: map[string][]int64{}}
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = s.hear
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
return s
|
||||
}
|
||||
|
||||
// noUnsaid fails the test for a plan whose stored revision is not one the hook heard: a move nobody said.
|
||||
func (s *movesSaid) noUnsaid(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
recent, err := open.inventory.RecentPlans(t.Context(), 50)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if len(recent) == 0 {
|
||||
t.Fatal("no plan was kept")
|
||||
}
|
||||
for _, p := range recent {
|
||||
revs := s.revs[p.ID]
|
||||
if len(revs) == 0 || revs[len(revs)-1] != p.Revision {
|
||||
t.Errorf("%s is kept at revision %d and was said at %v: a move nobody said", p.ID, p.Revision, revs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store: the moves a merge makes of a plan — the batch it opens, the walk it is cut into, the tier that
|
||||
// is asked — are each said, and a look that changes nothing says nothing (novox/hq ADR 0406 §2).
|
||||
@@ -0,0 +1,35 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// **The controller opens on a store whose plan tables are gone** (novox/hq ADR 0539 §17, issue 1945 B9): migration 0108 has
|
||||
// dropped the record of sends, the assignment generation, the gate's verdicts and the durations, and the controller still
|
||||
// assigns a module, reads where it runs, takes it off and reads the nodes.
|
||||
func TestTheControllerOpensOnAStoreWithoutThePlanTables(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
|
||||
switch name := strings.Trim(row[0], `"`); name {
|
||||
case "declaration_send", "assignment_generation", "build_gate", "duration":
|
||||
t.Errorf("the table %s is in the store", name)
|
||||
}
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1"})
|
||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||
t.Fatalf("assign on the migrated store: %v", err)
|
||||
}
|
||||
if on, err := inv.Assigned(ctx, "anchor"); err != nil || len(on) != 1 || on[0] != "notes" {
|
||||
t.Fatalf("assigned %v: %v", on, err)
|
||||
}
|
||||
if nodes, err := inv.Nodes(ctx); err != nil || len(nodes) == 0 {
|
||||
t.Fatalf("nodes %v: %v", nodes, err)
|
||||
}
|
||||
}
|
||||
@@ -1087,16 +1087,12 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hostVersions := deliveredVersions(shelf[hostModule])
|
||||
rolling := rollingModules(plans)
|
||||
rolling := map[string]bool{} // no plan rolls a module out any more (ADR 0539 §17)
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1154,26 +1150,6 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// rollingModules is every module an open plan is rolling out: those it keeps a record of, and those its
|
||||
// tiers name. **A release keeps no record per module** — its walk is per machine, its modules only in its
|
||||
// tier — so reading the records alone, D10 said "no plan is rolling them out" about the node-engine while
|
||||
// a release walked it, and the gate on that release's first machine waited on what its own send causes
|
||||
// (novox/hq issue 348). Pure.
|
||||
func rollingModules(plans []inventory.Plan) map[string]bool {
|
||||
rolling := map[string]bool{}
|
||||
for _, p := range plans {
|
||||
for m := range p.Modules {
|
||||
rolling[m] = true
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
rolling[m] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return rolling
|
||||
}
|
||||
|
||||
// deliveredVersions are the versions a module's registered build is delivered as: the last element
|
||||
// of every resource path under a `versions/` directory, which registration filled from the artifact's
|
||||
// digest (catalogue `${version}`). The node-engine names itself by that directory.
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What a machine's provisioner client said of a module's files (novox/hq ADR 0362, ADR 0349 §15, design 57 P6,
|
||||
// issue 672), read where a walk's end and a gate count a machine's send applied: a module the send handed to the
|
||||
// provisioner is applied there only once the client reported `prov.applied` on that build's templates as well.
|
||||
|
||||
// provAppliedOn is the client's newest report of a module on a machine, from the stream that keeps it
|
||||
// (broker.ProvAppliedStream); false when it has said nothing of it. A variable so a test can say the client's words.
|
||||
var provAppliedOn = func(ctx context.Context, node, module string) (inventory.ProvReport, bool, error) {
|
||||
var raw []byte
|
||||
var said bool
|
||||
read := func(js nats.JetStreamContext) error {
|
||||
var err error
|
||||
raw, said, err = broker.LastProvApplied(js, node, module)
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
if d := doctorFrom; d != nil && d.js != nil {
|
||||
err = read(d.js.Context())
|
||||
} else {
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := conn.JetStream(nats.Context(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return read(js)
|
||||
})
|
||||
}
|
||||
if err != nil || !said {
|
||||
return inventory.ProvReport{}, false, err
|
||||
}
|
||||
var r inventory.ProvReport
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return inventory.ProvReport{}, false, fmt.Errorf("what %s's provisioner client said of %s is not its report: %w",
|
||||
node, module, err)
|
||||
}
|
||||
return r, true, nil
|
||||
}
|
||||
|
||||
// provLookup answers the clients' reports for one reading; a report that cannot be read is not counted, which
|
||||
// leaves the machine waiting, then silent, never applied — and is said.
|
||||
func provLookup(ctx context.Context) inventory.ProvLookup {
|
||||
return func(node, module string) (inventory.ProvReport, bool) {
|
||||
r, said, err := provAppliedOn(ctx, node, module)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "what %s's provisioner client said of %s could not be read: %v\n", node, module, err)
|
||||
return inventory.ProvReport{}, false
|
||||
}
|
||||
return r, said
|
||||
}
|
||||
}
|
||||
|
||||
// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from
|
||||
// the stream that keeps their reports.
|
||||
func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) {
|
||||
all, err := broker.ProvAppliedReports(js)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []inventory.ProvReport
|
||||
for subject, raw := range all {
|
||||
var r inventory.ProvReport
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err)
|
||||
}
|
||||
if r.Status == inventory.ProvFailed {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// provisionedOutcome is a machine's report of one send as a delivery's end counts it (novox/hq ADR 0362 §1-5, issue
|
||||
// 840): the engine's own report, combined with its provisioner client's for a send that handed modules to it. The
|
||||
// outcome of a client's failure begins inventory.OutcomeProvisioningFailed; a report not yet applied on both sides
|
||||
// is none (false). A send handing nothing is the engine's report unchanged. onSend says the report is about the send.
|
||||
func provisionedOutcome(node string, sent inventory.SentDeclaration, r inventory.Reported, onSend bool,
|
||||
prov inventory.ProvLookup) (string, bool) {
|
||||
if len(sent.Provisioned) == 0 {
|
||||
return r.Outcome, onSend
|
||||
}
|
||||
engine := inventory.AppliedReport{Outcome: r.Outcome}
|
||||
if r.At != nil {
|
||||
engine.At = *r.At
|
||||
}
|
||||
got, ok := inventory.ProvisionedSend(node, sent, engine, onSend, prov)
|
||||
return got.Outcome, ok
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from
|
||||
// the stream that keeps their reports.
|
||||
func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) {
|
||||
all, err := broker.ProvAppliedReports(js)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []inventory.ProvReport
|
||||
for subject, raw := range all {
|
||||
var r inventory.ProvReport
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err)
|
||||
}
|
||||
if r.Status == inventory.ProvFailed {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -4,22 +4,18 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -194,7 +190,6 @@ func serve(ctx context.Context) (err error) {
|
||||
// Never by a read.
|
||||
go settlingPending(ctx, open)
|
||||
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
||||
go forgettingOldDurations(ctx, inv)
|
||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||
// above; this is the other half — the control plane is the only one of the three that knows
|
||||
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
||||
@@ -374,108 +369,6 @@ func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest
|
||||
}
|
||||
}
|
||||
|
||||
// readyNode is one machine and the declaration it would be sent.
|
||||
type readyNode struct {
|
||||
node string
|
||||
declared sendable
|
||||
}
|
||||
|
||||
// composeEach works out what each named machine should be, and never lets one machine's answer
|
||||
// decide another's.
|
||||
//
|
||||
// **A machine whose set cannot be worked out is that machine's problem** (novox/hq ADR 0066). A
|
||||
// whole-mesh push used to refuse outright when any one node failed to resolve, so a single
|
||||
// unanswerable requirement on a single machine — one module requiring a provision nobody had
|
||||
// assigned a provider for — left every other machine in the mesh unconverged, including machines
|
||||
// with no relation to it at all. Nothing was sent anywhere, and the machines that could not be sent
|
||||
// were the ones with nothing wrong with them.
|
||||
//
|
||||
// It is the same rule a92c11b established one level down, where an un-hostable module stopped
|
||||
// taking down the healthy modules beside it, applied one level up: **the blast radius of a fault is
|
||||
// the thing that has it.** What could not be worked out is named and returned, so a push still ends
|
||||
// with a non-zero outcome and nobody mistakes a partial convergence for a whole one.
|
||||
//
|
||||
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
|
||||
// across two machines that must agree — and dropped here, where it never did.
|
||||
func composeEach(names []string, allot func(node string) (order, error),
|
||||
compose func(node string) (sendable, error)) ([]readyNode, []string) {
|
||||
|
||||
var sending []readyNode
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
// **Numbered before it is composed, not before it is sent** (novox/hq issue 204). The
|
||||
// number says where this declaration stands against every other the mesh composed for the
|
||||
// machine, and the host refuses one lower than the last it applied. Taken at send time, as
|
||||
// it was, a declaration composed a minute ago — before an assignment changed — went out with
|
||||
// a number higher than one composed after the change and sent before it, and the machine
|
||||
// took the older content as the newer word: on 2026-10-02 a runtime assigned and applied on
|
||||
// two machines was undone two seconds later by exactly that. Taken here, before the first
|
||||
// read, what was composed earlier is numbered lower whatever order the sends happen in.
|
||||
numbered, err := allot(name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared, err := compose(name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
numbered.stamp(&declared)
|
||||
if len(declared.Resources) == 0 {
|
||||
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||
// say — and skipping the empty declaration leaves that last resource in force
|
||||
// forever, re-applied by the node's own heartbeat, with no way for the mesh to say
|
||||
// it is gone. An empty declaration is the correction: the host drops what the mesh
|
||||
// owned and keeps what it found (the adoption envelope still rides along). A node
|
||||
// that never held anything applies it as the no-op it is.
|
||||
fmt.Printf("%s owns nothing now — sent so it drops what it last held\n", name)
|
||||
}
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
return sending, refusals
|
||||
}
|
||||
|
||||
// keeping is what a round of sends keeps of the machines it composed, before any is delivered: nil
|
||||
// keeps them all. A push holds back the machines whose running modules' data would move (holdingBack).
|
||||
type keeping func(held context.Context, sending []readyNode) ([]readyNode, error)
|
||||
|
||||
// holdingBack leaves out each machine whose declaration would move a running module's data, says so,
|
||||
// and names it among those held back (novox/hq ADR 0217); the rest go on to be delivered, grants first
|
||||
// (issue 249). A declaration's body is the same bytes however often it is read.
|
||||
func holdingBack(ctx context.Context, inv *inventory.Inventory, sending []readyNode, movable map[string]bool,
|
||||
heldBack *[]string) ([]readyNode, error) {
|
||||
var out []readyNode
|
||||
for _, s := range sending {
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
moves, err := heldMoves(ctx, inv, s.node, body, movable)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(moves) > 0 {
|
||||
sayDataHeld(os.Stdout, s.node, moves)
|
||||
*heldBack = append(*heldBack, s.node)
|
||||
continue
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// delivery is the two acts of sending machines what they should be, apart, so the order between
|
||||
// them is one function's and can be read and tested there (novox/hq issue 249).
|
||||
type delivery interface {
|
||||
// grant issues what the machines' modules may do — each module's state raised and its
|
||||
// membership issued — for every machine about to be sent.
|
||||
grant(ctx context.Context, sending []readyNode) error
|
||||
// declare sends one machine its declaration and records it sent, answering the digest.
|
||||
declare(ctx context.Context, s readyNode, body []byte) (string, error)
|
||||
}
|
||||
|
||||
// errGrants marks a send that stopped because what the machines' modules may do could not be issued
|
||||
// (novox/hq issue 249). Nothing about the machines is wrong; asked again, it is likely to work, so an
|
||||
// announcement that hits it is held and asked again.
|
||||
@@ -496,158 +389,9 @@ func (g *grantsRefused) Error() string {
|
||||
strings.Join(names, ", "), g.nodes[names[0]])
|
||||
}
|
||||
|
||||
// deliver sends the machines their declarations: **the machine holding the bus, then the grants,
|
||||
// then the rest** (novox/hq issue 249).
|
||||
//
|
||||
// A merge gave a module a new state; its bundle reached every machine within a minute, and the
|
||||
// machines' permissions on the bus did not include the state until somebody pushed by hand: the code
|
||||
// arrived before the right to use it. A module that read its new state on start failed its start; the
|
||||
// one that was there retried for two minutes. The memberships were issued after the declarations —
|
||||
// "because the runtime it is for arrives with it" — and a membership is retained last-per-subject on
|
||||
// the bus (internal/link/bus.go), so issued first it waits for the runtime that arrives after it. A
|
||||
// runtime still on the old code merely holds a grant it does not use yet.
|
||||
//
|
||||
// **The holder's declaration before the grants, though.** The bus's user list travels in it, and the
|
||||
// controller's own right to publish memberships and raise buckets is in that list (the precedent of
|
||||
// issue 183): grants first, and a grant the controller is not yet allowed to make would hold the very
|
||||
// declaration that allows it — a lock only a hand on the broker could open. A runtime already running
|
||||
// on that machine follows a membership issued after its declaration, as it always has.
|
||||
//
|
||||
// **A grant that cannot be issued holds back what it concerns, and says so as an error.** It used to
|
||||
// be said and passed over — "the machines keep what they derive until the next push" — which reported
|
||||
// a rollout done that had delivered code its machines could not run. A membership that failed holds
|
||||
// back its own machine; a failure that names no machine (the buckets) holds back every machine but the
|
||||
// holder, already sent. The error carries errGrants, so the caller's rollout is not marked sent and is
|
||||
// tried again.
|
||||
//
|
||||
// The grants are issued, not waited on: a membership is a retained message the runtime reads when it
|
||||
// comes, and the bus answers its publication; nothing here waits for a runtime to have read one.
|
||||
func deliver(ctx context.Context, d delivery, holder string, sending []readyNode) (map[string]string, error) {
|
||||
digests := map[string]string{}
|
||||
if len(sending) == 0 {
|
||||
return digests, nil
|
||||
}
|
||||
send := func(s readyNode) error {
|
||||
// The number is inside the signed bytes, so a replayed older declaration cannot borrow a
|
||||
// newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204).
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digest, err := d.declare(ctx, s, body)
|
||||
if errors.Is(err, errFilesNotPut) && s.declared.fileReferences {
|
||||
// **The grant that lets the controller write the bucket travels in the very declaration of the machine holding the
|
||||
// bus** (novox/hq ADR 0486 §2): until the bus has it, a put is refused, and the declaration that would deliver the
|
||||
// grant would wait for it. So that send, and only that, goes with every content inline, as to a node that reads no
|
||||
// references, and says so; the next one tries the bucket again.
|
||||
fmt.Printf("%s: the files that travel by digest could not be put in the bucket (%v): sending them inline\n", s.node, err)
|
||||
s.declared.fileReferences = false
|
||||
if body, err = s.declared.Body(); err != nil {
|
||||
return err
|
||||
}
|
||||
digest, err = d.declare(ctx, s, body)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digests[s.node] = digest
|
||||
return nil
|
||||
}
|
||||
var rest []readyNode
|
||||
for _, s := range sending {
|
||||
if holder != "" && s.node == holder {
|
||||
if err := send(s); err != nil {
|
||||
return digests, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
rest = append(rest, s)
|
||||
}
|
||||
held := map[string]error{}
|
||||
if err := d.grant(ctx, sending); err != nil {
|
||||
var some *grantsRefused
|
||||
if !errors.As(err, &some) {
|
||||
var names []string
|
||||
for _, s := range rest {
|
||||
names = append(names, s.node)
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return digests, fmt.Errorf("%w: %w", errGrants, err)
|
||||
}
|
||||
return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(names, ", "), err)
|
||||
}
|
||||
held = some.nodes
|
||||
}
|
||||
var notSent []string
|
||||
for _, s := range rest {
|
||||
if _, refused := held[s.node]; refused {
|
||||
notSent = append(notSent, s.node)
|
||||
continue
|
||||
}
|
||||
if err := send(s); err != nil {
|
||||
return digests, err
|
||||
}
|
||||
}
|
||||
if len(notSent) > 0 {
|
||||
return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(notSent, ", "),
|
||||
&grantsRefused{nodes: held})
|
||||
}
|
||||
if len(held) > 0 {
|
||||
// Only the holder's own memberships failed, and it was sent before them.
|
||||
return digests, fmt.Errorf("%w: %w", errGrants, &grantsRefused{nodes: held})
|
||||
}
|
||||
return digests, nil
|
||||
}
|
||||
|
||||
// overTheBus is delivery as the mesh does it: memberships on the bus, declarations signed.
|
||||
type overTheBus struct {
|
||||
open *stores
|
||||
server *link.Server
|
||||
signer link.Signer
|
||||
// indent is put before each "sent" line, for the callers whose output is nested.
|
||||
indent string
|
||||
}
|
||||
|
||||
// putFiles puts the contents s names by digest in the bucket.
|
||||
func (b overTheBus) putFiles(ctx context.Context, s readyNode) error {
|
||||
objects, err := s.declared.objects()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: the files that travel by digest cannot be read: %w", s.node, err)
|
||||
}
|
||||
if len(objects) == 0 {
|
||||
return nil
|
||||
}
|
||||
bus, ok := b.server.Bus().(link.OverNATS)
|
||||
if !ok {
|
||||
return fmt.Errorf("%s: the declaration names files by digest and this bus has no object bucket", s.node)
|
||||
}
|
||||
putting, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
store, err := openBucket(putting, bus.Conn, s.node)
|
||||
if err == nil {
|
||||
err = putObjects(putting, store, objects)
|
||||
}
|
||||
notePut(ctx, s.node, err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket.
|
||||
var errFilesNotPut = errors.New("the files it names by digest are not in the bucket")
|
||||
|
||||
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||
// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
|
||||
// when they cannot be, and never what holds the send back (assertOnSend).
|
||||
if bus, ok := b.server.Bus().(link.OverNATS); ok {
|
||||
js := broker.OnConn(bus.Conn)
|
||||
js.Note = func(format string, args ...any) { fmt.Printf(b.indent+" "+format+"\n", args...) }
|
||||
_ = assertOnSend(ctx, b.open.inventory, js, b.indent)
|
||||
}
|
||||
return issueMemberships(ctx, b.open, b.server, sending)
|
||||
}
|
||||
|
||||
// brokerFirst is the machines to send in the order a grant needs (novox/hq issue 249): the machine
|
||||
// holding the bus first — its declaration carries the bus's user list (composeBusUsers), and a
|
||||
// module's new permissions are refused by the bus until that list says them. Among the machines it
|
||||
@@ -755,219 +499,6 @@ func couldNotBeResolved(refusals []string, sent int) error {
|
||||
len(refusals), sent, strings.Join(refusals, "\n\n"))
|
||||
}
|
||||
|
||||
// sendTo resolves and sends to exactly the machines named, or refuses without sending anything.
|
||||
//
|
||||
// The all-or-nothing rule push deliberately does NOT follow, and for a reason that holds here and
|
||||
// not there: a rotation that reached the
|
||||
// consumer and refused on the provider would leave one end holding a credential the other has
|
||||
// never heard of — which is the state this whole mechanism exists to make impossible.
|
||||
func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
_, err := sendToEach(ctx, open, names)
|
||||
return err
|
||||
}
|
||||
|
||||
// sendToEach is sendTo, answering the machines it sent: those named, and before them the machine
|
||||
// holding the bus when its user list must go first (novox/hq issue 249) — so a caller that waits for
|
||||
// the machines it sent waits for that one too.
|
||||
func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
inv := open.inventory
|
||||
// **No send replaces the bus but its planned step** (novox/hq ADR 0236). A plan's send to the bus's
|
||||
// machine for some other module carried the bus's new build with it on 2026-10-06, and the bus
|
||||
// restarted under every machine with nobody having asked. Refused whole — the send cannot leave the
|
||||
// bus behind and carry the rest (ADR 0221 option 2) — and said with the remedy; the plan tries again.
|
||||
if !busStepSending(ctx) {
|
||||
held, err := busHeld(ctx, inv, names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, n := range names {
|
||||
if why, h := held[n]; h {
|
||||
return nil, fmt.Errorf("%w: %s", errBusWaits, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer ident.Close()
|
||||
|
||||
// The private network every declaration of the send is composed with: not computable refuses the send up front.
|
||||
if _, err := generatorsOf(ctx, open); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
holder, behind, err := brokerBehind(ctx, open, names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names = brokerFirst(names, holder, behind)
|
||||
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
|
||||
// converge, which flips the node and then sends it — is not made to wait on itself.
|
||||
ctx, release, err := holdNodes(ctx, open, names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
var sending []readyNode
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
// Numbered before composing, for the reason composeEach gives (novox/hq issue 204).
|
||||
numbered, err := allot(ctx, inv, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
// Read afresh for each machine: the send before it made and recorded things.
|
||||
composing := freshInput(ctx)
|
||||
plan, _, err := resolveNode(composing, open, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
reportUnhostable(name, plan)
|
||||
reportKept(ctx, plan)
|
||||
declared, err := composeSend(composing, open, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
numbered.stamp(&declared)
|
||||
reportLeftOut(name, declared)
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
if len(refusals) > 0 {
|
||||
return nil, fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
|
||||
len(refusals), strings.Join(refusals, "\n\n"))
|
||||
}
|
||||
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
// And every assignment on those machines its membership (novox/hq ADR 0160): composed from the
|
||||
// same records the bus's accounts are, so what a runtime serves and what its account may are one
|
||||
// composition. **Issued before the declarations** (novox/hq issue 249): the runtime the
|
||||
// membership is for arrives with the declaration, and a membership waits for it on the bus; the
|
||||
// code arriving first was refused its own state until somebody pushed.
|
||||
if _, err := deliver(ctx, overTheBus{open: open, server: server, signer: ident, indent: " "}, holder, sending); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent := make([]string, 0, len(sending))
|
||||
for _, s := range sending {
|
||||
sent = append(sent, s.node)
|
||||
}
|
||||
return sent, nil
|
||||
}
|
||||
|
||||
// issueMemberships publishes the membership of every module on the machines just sent.
|
||||
//
|
||||
// Each carries what its module receives and the private network's addresses, from the same
|
||||
// composition as the declaration it was sent (novox/hq ADR 0167): a provider reads what it is
|
||||
// given on the bus, and the file written beside it says the same thing.
|
||||
func issueMemberships(ctx context.Context, open *stores, server *link.Server, sent []readyNode) error {
|
||||
records, err := open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bus, ok := server.Bus().(link.OverNATS)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
// Which machines are root-free now (novox/hq ADR 0259 §8): a channel's verified sender is composed for the
|
||||
// router only from one, beside a router on one. Judged once per push, by the root-free verb's judgement.
|
||||
var whys map[string]string
|
||||
records.RootFree, whys = rootVerdictsOf(judgeRootFree(ctx, newRootReader(ctx, open.inventory, bus.Conn),
|
||||
records.Nodes, time.Now()))
|
||||
where := broker.PlacementsOf(records, records.Interchangeable)
|
||||
// What the judgement kept off is said, with its reason (novox/hq issue 930); the self-check gives it back.
|
||||
for _, line := range droppedVerifiedSenders(records, where, whys) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
// **Every declared state's bucket, before the memberships that name it** (novox/hq ADR 0201). The
|
||||
// raise at start asserts them too, but a module registered and assigned since would otherwise have
|
||||
// its bucket only after the control plane next restarts — found the first time a module declared
|
||||
// state: its bundle asked for a bucket that did not exist. Idempotent and cheap.
|
||||
//
|
||||
// **A failure here is the send's failure** (novox/hq issue 249). It was said and the push stood,
|
||||
// because the declarations were already away; they are sent after this now — all but the bus's
|
||||
// own machine, sent before it (deliver) — and a module whose state does not exist is a module
|
||||
// that fails its start, so they are not sent and the caller tries again rather than reporting the
|
||||
// rollout done.
|
||||
buckets, err := open.inventory.DeclaredBuckets(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the modules' state could not be read, so no bucket was asserted: %w", err)
|
||||
}
|
||||
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
||||
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
||||
}
|
||||
// **And every declared log, for the same reason** (novox/hq ADR 0297 §2): a membership names its
|
||||
// logs, and a module whose log does not exist fails its first append.
|
||||
logs, err := open.inventory.DeclaredLogs(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the modules' logs could not be read, so no log was asserted: %w", err)
|
||||
}
|
||||
if _, err := broker.RaiseLogs(broker.OnConn(bus.Conn), logs); err != nil {
|
||||
return fmt.Errorf("the modules' logs could not be asserted on the bus: %w", err)
|
||||
}
|
||||
// Every membership is tried, and the first failure named once.
|
||||
issued := 0
|
||||
refused := map[string]error{}
|
||||
for _, s := range sent {
|
||||
node := s.node
|
||||
for _, d := range records.Assigned[node] {
|
||||
membership := broker.MembershipFor(node, d, where)
|
||||
membership.Mesh = s.declared.Mesh
|
||||
for requirement, given := range s.declared.Received[d.Module] {
|
||||
raw, err := json.Marshal(given)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if membership.Receives == nil {
|
||||
membership.Receives = map[string]json.RawMessage{}
|
||||
}
|
||||
membership.Receives[requirement] = raw
|
||||
}
|
||||
body, err := json.Marshal(membership)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
|
||||
if refused[node] == nil {
|
||||
refused[node] = fmt.Errorf("%s: %w", d.Module, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
issued++
|
||||
}
|
||||
}
|
||||
if issued > 0 {
|
||||
fmt.Printf(" issued %d membership(s)\n", issued)
|
||||
}
|
||||
// **And what every other machine hears of a module's stream** (novox/hq ADR 0363 §6): a grant the provider's
|
||||
// setting changed reaches a consumer on a machine this send does not reach. Its membership is issued again
|
||||
// as last issued, with only what it hears changed, so nothing else of it moves before its own send.
|
||||
reheard := reissueHearing(ctx, bus, records, where, sent)
|
||||
if reheard > 0 {
|
||||
fmt.Printf(" issued %d membership(s) again for what they hear of a module's stream\n", reheard)
|
||||
}
|
||||
// **And says again what it keeps** (novox/hq ADR 0495 clause 2, issue 1790): its word at start is refused until a push
|
||||
// carries its grant; kept only when it changed, so saying it at every push changes nothing else.
|
||||
if js := servingBus.Load(); js != nil {
|
||||
if err := announceOwnStates(ctx, js); err != nil {
|
||||
fmt.Printf(" states: %v\n", err)
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
// Returned, never passed over (novox/hq issue 249): the declarations of the machines they are
|
||||
// for are not sent, and the rollout that asked is tried again rather than waiting for a push.
|
||||
return &grantsRefused{nodes: refused}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// digestOf is what the mesh compares to answer "has this machine been sent what it should be".
|
||||
//
|
||||
// Over the same bytes that are sent, so the comparison is of the thing itself rather than of
|
||||
@@ -1027,11 +558,6 @@ func wouldSendFrom(ctx context.Context, open *stores,
|
||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment
|
||||
// elsewhere in the mesh is not a change of this machine.
|
||||
if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1165,120 +691,10 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107).
|
||||
// allotting is allot over one inventory, in the shape composeEach takes.
|
||||
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (order, error) {
|
||||
return func(node string) (order, error) { return allot(ctx, inv, node) }
|
||||
}
|
||||
|
||||
// epochForActs is the lease's gate as a composition asks it; a variable so a test can act under an epoch
|
||||
// without a bus.
|
||||
var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) }
|
||||
|
||||
// order is what a declaration carries of its writer's order (link/order.go): its sequence, and the
|
||||
// epoch of the lease it is composed under — zero for a machine that has not said it reads one.
|
||||
type order struct {
|
||||
sequence int64
|
||||
epoch uint64
|
||||
// generation is the assignment generation read before the composition, and readsGeneration whether the
|
||||
// machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts
|
||||
// under, whether or not the machine is sent it — both kept for the record of the send.
|
||||
generation int64
|
||||
readsGeneration bool
|
||||
acting uint64
|
||||
}
|
||||
|
||||
// allot takes the next sequence for a machine — the number its next declaration carries — under the
|
||||
// lease: a process that may not act takes none, and composes nothing (novox/hq to-be 45 §6).
|
||||
func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, error) {
|
||||
epoch, err := epochForActs(ctx)
|
||||
if err != nil {
|
||||
return order{}, fmt.Errorf("nothing was composed for %s: %w", node, err)
|
||||
}
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
acting := epoch
|
||||
if epoch > 0 {
|
||||
reads, err := inv.ReadsEpoch(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
if !reads {
|
||||
epoch = 0
|
||||
}
|
||||
}
|
||||
// **The generation is read here, before the composition reads a single assignment** (novox/hq issue
|
||||
// 234). A generation only grows, so one read before is never newer than the view composed after it:
|
||||
// the declaration may claim a generation older than its content, never newer — and a claim newer than
|
||||
// the content is exactly the stale send the machine must be able to refuse.
|
||||
generation, err := inv.AssignmentGeneration(ctx)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
readsGeneration, err := inv.ReadsGeneration(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
seq, err := inv.NextSequence(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration,
|
||||
acting: acting}, nil
|
||||
}
|
||||
|
||||
// recordSent writes down what a machine was just sent, and returns the digest.
|
||||
//
|
||||
// **On a context that outlives the caller's** (novox/hq issue 204). The record is written after the
|
||||
// declaration is away, so a send that failed is never recorded as current — and a controller being
|
||||
// replaced mid-send had its context cancelled between the two, so the machine was told and the mesh
|
||||
// never wrote it down: status read "applied, current" over a machine that had just been sent
|
||||
// something else. What was sent was sent; the record of it must not depend on the sender living
|
||||
// another second. Bounded, so a store that is away does not hold a dying process open for ever.
|
||||
//
|
||||
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
||||
// what tells a machine held back by a policy or a plan from one a push left behind.
|
||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
||||
builds map[string]string, epoch uint64, sent sentRecord) (string, error) {
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
record, err := inv.NodeByName(kept, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest := digestOf(body)
|
||||
// The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the
|
||||
// would-send is never stamped with a generation the machine was not sent; and the send itself, who sent
|
||||
// it and from which generation, beside them. A record of the send that cannot be written does not make a
|
||||
// send that is away look failed: it is said, loudly, and the machine's own record stands.
|
||||
err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{
|
||||
Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation,
|
||||
Digest: digest, Modules: sent.modules, Provisioning: sent.plans})
|
||||
var unrecorded *inventory.SendNotRecordedError
|
||||
switch {
|
||||
case errors.As(err, &unrecorded):
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+
|
||||
"sent it and from which generation could not be written down, so a refusal of it will name no sender "+
|
||||
"(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err)
|
||||
fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err)
|
||||
case err != nil:
|
||||
return "", err
|
||||
}
|
||||
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
|
||||
// Never a reason to fail a send that is already away: a summary that cannot be kept means the
|
||||
// next comparison has nothing to hold against, which is how every machine starts.
|
||||
if summary, err := summarize(body); err == nil {
|
||||
if raw, err := json.Marshal(summary); err == nil {
|
||||
if err := inv.RecordSentSummary(kept, record.ID, raw); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: what it was sent is not kept for comparison: %v\n", node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// reportUnheldPushed says what a push's machines lack of the seats their modules depend on
|
||||
// (novox/hq ADR 0207): every line for a machine the push named, since that is the machine somebody
|
||||
// is looking at, and one line per machine otherwise — a list per machine across the mesh is the
|
||||
@@ -1314,134 +730,3 @@ func schemaReach() (int, error) {
|
||||
}
|
||||
return reach, nil
|
||||
}
|
||||
|
||||
// reissueHearing issues again, on every machine not just sent, the membership of each assignment whose
|
||||
// `consumes` names a module owning a stream, when what it hears changed: read as last issued from the bus,
|
||||
// its `hears` replaced, and published. One that was never issued is left to its own send. Failures are said
|
||||
// and passed over: the send that carries the grant is not the consumer's, and its own next send issues it.
|
||||
func reissueHearing(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements,
|
||||
sent []readyNode) int {
|
||||
reached := map[string]bool{}
|
||||
for _, s := range sent {
|
||||
reached[s.node] = true
|
||||
}
|
||||
issued := 0
|
||||
for node, declared := range records.Assigned {
|
||||
if reached[node] {
|
||||
continue
|
||||
}
|
||||
for _, d := range declared {
|
||||
if !d.HearsAStream {
|
||||
continue
|
||||
}
|
||||
subject := broker.MembershipSubject(node, d.Module)
|
||||
last, err := bus.JS.GetLastMsg(broker.AssignmentsStream, subject)
|
||||
if err != nil {
|
||||
if !errors.Is(err, nats.ErrMsgNotFound) {
|
||||
fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n",
|
||||
d.Module, node, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
body, changed, err := withHearing(last.Data, broker.HeardIssuedFor(node, d))
|
||||
if err != nil {
|
||||
fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n",
|
||||
d.Module, node, err)
|
||||
continue
|
||||
}
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
|
||||
fmt.Printf(" %s on %s: what it hears could not be issued again: %v\n", d.Module, node, err)
|
||||
continue
|
||||
}
|
||||
issued++
|
||||
}
|
||||
}
|
||||
return issued
|
||||
}
|
||||
|
||||
// withHearing is a membership as issued with its `hears` replaced, and whether that changed it. Pure.
|
||||
func withHearing(issued []byte, hears []broker.HeardIssued) ([]byte, bool, error) {
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(issued, &fields); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
var had []broker.HeardIssued
|
||||
if raw, ok := fields["hears"]; ok {
|
||||
if err := json.Unmarshal(raw, &had); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
if slices.Equal(had, hears) {
|
||||
return issued, false, nil
|
||||
}
|
||||
if len(hears) == 0 {
|
||||
delete(fields, "hears")
|
||||
} else {
|
||||
raw, err := json.Marshal(hears)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
fields["hears"] = raw
|
||||
}
|
||||
body, err := json.Marshal(fields)
|
||||
return body, true, err
|
||||
}
|
||||
|
||||
func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
// The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node
|
||||
// that applies it finds every one. A node reading no references has none.
|
||||
// **Under the node's lock** from the put to the publication, so the sweep of its bucket cannot remove an object between them.
|
||||
unlock := lockFilesOf(s.node)
|
||||
err := b.putFiles(ctx, s)
|
||||
if err != nil {
|
||||
unlock()
|
||||
return "", err
|
||||
}
|
||||
declared := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second)
|
||||
unlock()
|
||||
if declared != nil {
|
||||
return "", declared
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||
// the machine look current for a declaration it never received.
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch,
|
||||
sentRecordOf(ctx, s.declared))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(s.declared.Bindings) > 0 {
|
||||
// And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232):
|
||||
// what the next resolution keeps it at. On the same outliving context as the send's record.
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
err := b.open.inventory.RecordBindings(kept, s.node, s.declared.Bindings)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s was sent its declaration, and where its consumers are bound to their "+
|
||||
"data could not be recorded: %w", s.node, err)
|
||||
}
|
||||
}
|
||||
if s.declared.BusUsers != "" {
|
||||
// And the user list it carried, so the next send reads whether it must go first from the
|
||||
// list alone (novox/hq issue 249). On the same outliving context as the send's record.
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
err := b.open.inventory.RecordSentBusUsers(kept, s.node, digestOf([]byte(s.declared.BusUsers)))
|
||||
cancel()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// The list sent issued the credential of every module it carried, at the fingerprint each is registered at:
|
||||
// the grants the assigner's sixth rule reads (novox/hq ADR 0528 clause 2, issue 1394).
|
||||
if bus, ok := b.server.Bus().(link.OverNATS); ok {
|
||||
if js, err := jetstream.New(bus.Conn); err == nil {
|
||||
writing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
grantsIssued(writing, b.open.inventory, js, b.indent)
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf("%ssent %s %d resource(s)\n", b.indent, s.node, len(s.declared.Resources))
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// One machine that cannot be worked out is not a reason to leave the mesh unconverged.
|
||||
//
|
||||
// A whole-mesh push refused outright the moment any single node failed to resolve, so a module on
|
||||
// the anchor requiring a provision nobody had assigned a provider for stopped every OTHER machine
|
||||
// from being sent anything — machines with no relation to the fault, and nothing wrong with them.
|
||||
// The failure and the punishment were on different machines.
|
||||
//
|
||||
// It is the same rule an un-hostable module already follows one level down (a92c11b: one module on
|
||||
// the wrong machine no longer refuses the whole node), applied one level up.
|
||||
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||
sending, refusals := composeEach(
|
||||
[]string{"anchor", "home-server", "laptop"}, numbered(),
|
||||
func(node string) (sendable, error) {
|
||||
if node == "anchor" {
|
||||
return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
|
||||
}
|
||||
return sendable{Resources: []map[string]any{{"id": node + ".thing"}}}, nil
|
||||
})
|
||||
|
||||
var told []string
|
||||
for _, s := range sending {
|
||||
told = append(told, s.node)
|
||||
}
|
||||
if strings.Join(told, ",") != "home-server,laptop" {
|
||||
t.Errorf("a machine with nothing wrong with it was not sent: %v", told)
|
||||
}
|
||||
if len(refusals) != 1 || !strings.Contains(refusals[0], "anchor") ||
|
||||
!strings.Contains(refusals[0], "acme-ca") {
|
||||
t.Errorf("the machine that could not be worked out was not named with its reason: %v",
|
||||
refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine whose declaration composes to nothing is SENT the empty declaration, not skipped
|
||||
// (novox/hq issue 127): it may have held something before, and only sending the empty
|
||||
// declaration tells it to drop what the mesh owned. It is never a refusal.
|
||||
func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) {
|
||||
sending, refusals := composeEach([]string{"spare"}, numbered(),
|
||||
func(string) (sendable, error) { return sendable{}, nil })
|
||||
if len(sending) != 1 || len(refusals) != 0 {
|
||||
t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// A push that skipped a machine still ends badly, and says what was sent.
|
||||
//
|
||||
// **Skipping is not succeeding.** The mesh is not in the state somebody asked for, so the command
|
||||
// exits non-zero — but it says how many machines it did reach, because the old message ("nothing
|
||||
// was sent") was the very claim that had become untrue.
|
||||
func TestASkippedMachineIsStillAnError(t *testing.T) {
|
||||
if err := couldNotBeResolved(nil, 3); err != nil {
|
||||
t.Fatalf("a push that resolved every machine reported a problem: %v", err)
|
||||
}
|
||||
|
||||
err := couldNotBeResolved([]string{"anchor:\nnothing provides \"acme-ca\""}, 2)
|
||||
if err == nil {
|
||||
t.Fatal("a push that could not work out a machine reported success")
|
||||
}
|
||||
said := err.Error()
|
||||
if strings.Contains(said, "nothing was sent") {
|
||||
t.Errorf("the push says nothing was sent, and it sent two machines: %q", said)
|
||||
}
|
||||
for _, want := range []string{"anchor", "acme-ca", "2 other node(s) were"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Errorf("the refusal does not say %q: %q", want, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
|
||||
func numbered() func(string) (order, error) {
|
||||
var n int64
|
||||
return func(string) (order, error) { n++; return order{sequence: n}, nil }
|
||||
}
|
||||
@@ -111,33 +111,6 @@ type meshStatus struct {
|
||||
// why they could not be read.
|
||||
Pending []pendingStatus `json:"pending,omitempty"`
|
||||
PendingUnread string `json:"pendingUnread,omitempty"`
|
||||
// LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which
|
||||
// assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485).
|
||||
// A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read.
|
||||
LastSent []machineLastSent `json:"lastSent"`
|
||||
LastSentUnread string `json:"lastSentUnread,omitempty"`
|
||||
}
|
||||
|
||||
// machineLastSent is one machine's last send as status says it.
|
||||
type machineLastSent struct {
|
||||
Node string `json:"node"`
|
||||
Sequence int64 `json:"sequence"`
|
||||
SentAt time.Time `json:"sentAt"`
|
||||
Sender string `json:"sender"`
|
||||
Epoch int64 `json:"epoch"`
|
||||
Generation int64 `json:"generation"`
|
||||
Digest string `json:"digest"`
|
||||
Modules []string `json:"modules"`
|
||||
// Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it
|
||||
// has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed.
|
||||
Answer string `json:"answer"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Failed int `json:"failed,omitempty"`
|
||||
AnsweredAt *time.Time `json:"answeredAt,omitempty"`
|
||||
// RefusedForGeneration is when the machine refused it for coming from an older assignment generation than
|
||||
// it applied, and AppliedGeneration that generation; absent when it did not.
|
||||
RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"`
|
||||
AppliedGeneration int64 `json:"appliedGeneration,omitempty"`
|
||||
}
|
||||
|
||||
// pendingStatus is one pending assignment as status says it.
|
||||
@@ -297,7 +270,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
||||
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
||||
}
|
||||
out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
@@ -354,20 +326,3 @@ func say(value any) error {
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485).
|
||||
func lastSentStatus(sends map[string]inventory.Send) []machineLastSent {
|
||||
out := []machineLastSent{}
|
||||
for _, s := range sends {
|
||||
modules := s.Modules
|
||||
if modules == nil {
|
||||
modules = []string{}
|
||||
}
|
||||
out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender,
|
||||
Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome,
|
||||
Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At,
|
||||
RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// A membership issued again for what it hears changes nothing else of it (novox/hq ADR 0363 §6).
|
||||
func TestAMembershipIssuedAgainChangesOnlyWhatItHears(t *testing.T) {
|
||||
issued := []byte(`{"node":"ace","module":"reader","receives":{"x":[1]},"emits":"mesh.mod.reader.event.{event}"}`)
|
||||
hears := []broker.HeardIssued{{Stream: "MAIL", Consumer: "ace_reader"}}
|
||||
body, changed, err := withHearing(issued, hears)
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("changed %v, %v", changed, err)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["receives"] == nil || got["emits"] != "mesh.mod.reader.event.{event}" || got["hears"] == nil {
|
||||
t.Fatalf("issued again as %s", body)
|
||||
}
|
||||
if _, again, _ := withHearing(body, hears); again {
|
||||
t.Fatal("the same hearing issued again counts as a change")
|
||||
}
|
||||
gone, changed, err := withHearing(body, nil)
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("a withdrawal: %v, %v", changed, err)
|
||||
}
|
||||
got = nil
|
||||
if err := json.Unmarshal(gone, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := got["hears"]; still {
|
||||
t.Fatalf("a withdrawn hearing is still issued: %s", gone)
|
||||
}
|
||||
}
|
||||
@@ -99,7 +99,7 @@ var removedVerbs = map[string]bool{"build": true, "rebuild": true, "forget-check
|
||||
// Release 6 (ADR 0539 §5): a delivery is each machine following its wants, so the verbs that planned, started, stopped,
|
||||
// retried or closed one are gone.
|
||||
"plans": true, "plan": true, "push": true, "deliver": true, "delivery-stop": true, "delivery-backups-again": true,
|
||||
"delivery-walks": true, "delivery-order": true, "upgrade": true}
|
||||
"delivery-walks": true, "delivery-order": true}
|
||||
|
||||
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
|
||||
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
|
||||
@@ -618,6 +618,22 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
argv = append(argv, "--retired")
|
||||
}
|
||||
return argv, nil
|
||||
case "upgrade":
|
||||
m, policy := str("module"), str("policy")
|
||||
if m == "" {
|
||||
if policy != "" {
|
||||
return nil, errors.New("upgrade takes a policy only for a module")
|
||||
}
|
||||
return []string{"upgrade"}, nil
|
||||
}
|
||||
argv := []string{"upgrade", m}
|
||||
if policy != "" {
|
||||
argv = append(argv, policy)
|
||||
if w := str("why"); w != "" {
|
||||
argv = append(argv, "--why", w)
|
||||
}
|
||||
}
|
||||
return argv, nil
|
||||
case "bus":
|
||||
if !on("upgrade") {
|
||||
return []string{"bus"}, nil
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import "context"
|
||||
|
||||
// What was once a send is no longer one (ADR 0539 §5, §8): the controller writes the compositions that changed instead
|
||||
// (composeModules), and each machine follows its wants. The calls below stand where a send was asked until the part of the
|
||||
// controller that asked it writes compositions in its place; the subtask that replaces the last caller deletes this file.
|
||||
|
||||
// sendTo sends nothing; see sendToEach.
|
||||
func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
_, err := sendToEach(ctx, open, names)
|
||||
return err
|
||||
}
|
||||
|
||||
// sendToEach answers the machines it was asked for, and sends them nothing.
|
||||
func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// holdNodes held machines from composing to sending; nothing is sent, so it holds nothing.
|
||||
func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) {
|
||||
return ctx, func() {}, nil
|
||||
}
|
||||
@@ -89,16 +89,6 @@ func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) {
|
||||
t.Fatal("the digest the mesh compares is not of the declaration push sends")
|
||||
}
|
||||
|
||||
// plan --json prints that same envelope.
|
||||
printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) })
|
||||
var compact bytes.Buffer
|
||||
if err := json.Compact(&compact, []byte(printed)); err != nil {
|
||||
t.Fatalf("plan --json is not JSON: %v\n%s", err, printed)
|
||||
}
|
||||
if digestOf(compact.Bytes()) != digestOf(body) {
|
||||
t.Fatalf("plan --json shows something other than what push sends:\n%s", printed)
|
||||
}
|
||||
|
||||
// Taking the module moves its resources out of untaken.
|
||||
if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
client := aClientModule()
|
||||
moved, kept := aBot("moved-bot", true), aBot("kept-bot", false)
|
||||
moved.Provides = []catalogue.Offer{{Name: "moved-bot-api"}}
|
||||
kept.Provides = []catalogue.Offer{{Name: "kept-bot-api"}}
|
||||
desk := catalogue.Manifest{Module: "desk", Version: "1", Requires: []string{"moved-bot-api", "kept-bot-api"}}
|
||||
for _, m := range []catalogue.Manifest{client, moved, kept, desk} {
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, m := range []string{client.Module, desk.Module} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract,
|
||||
SaidAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
plan, settings, err := resolveNode(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.RequiredBy("moved-bot") != "desk" || plan.RequiredBy("kept-bot") != "desk" {
|
||||
t.Fatalf("the bots are on laptop because %v", plan.Because)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "laptop", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var movedToken, keptToken, movedConf map[string]any
|
||||
for _, r := range declared.Resources {
|
||||
switch r["id"] {
|
||||
case "moved-bot.needs-token":
|
||||
movedToken = r
|
||||
case "kept-bot.needs-token":
|
||||
keptToken = r
|
||||
case "moved-bot.conf":
|
||||
movedConf = r
|
||||
}
|
||||
}
|
||||
if movedToken == nil || movedToken["sealed"] != nil || movedToken["owner-at-runtime"] != "owned" {
|
||||
t.Fatalf("the pulled-in moved module's secret file is sent as %v: placed, with no value", movedToken)
|
||||
}
|
||||
if movedConf == nil || movedConf["content"] != nil || movedConf["owner-at-runtime"] != "owned" {
|
||||
t.Fatalf("the pulled-in moved module's templated file is sent as %v: placed, with no text", movedConf)
|
||||
}
|
||||
if keptToken == nil || keptToken["sealed"] == nil {
|
||||
t.Fatalf("the pulled-in module not moved has its secret sent as %v: sealed, as before", keptToken)
|
||||
}
|
||||
if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held {
|
||||
t.Fatalf("the send made the pulled-in handed module's secret in the controller's store (held %v, %v)", held, err)
|
||||
}
|
||||
if _, has := declared.plans["kept-bot"]; has || len(declared.plans) != 1 {
|
||||
t.Fatalf("the send hands %v", declared.plans)
|
||||
}
|
||||
|
||||
record, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, inventory.Send{Digest: "first",
|
||||
Provisioning: sentRecordOf(ctx, declared).plans}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
mirror := inv.PlanValuesForTest(t)
|
||||
var value inventory.AssignmentMirrored
|
||||
if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil {
|
||||
t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err)
|
||||
}
|
||||
if value.RequiredBy != "desk" || !value.Assigned.IsZero() || value.Provisioning == nil ||
|
||||
len(value.Provisioning.Files) != 1 || value.Provisioning.Files[0].Path != "/etc/moved-bot/bot.conf" ||
|
||||
len(value.Provisioning.Secrets) != 1 || value.Provisioning.Secrets[0].Name != "token" {
|
||||
t.Fatalf("the pulled-in module is mirrored as %s", mirror["laptop.moved-bot"])
|
||||
}
|
||||
if _, has := mirror["laptop.kept-bot"]; has {
|
||||
t.Fatalf("a pulled-in module not handed is mirrored: %s", mirror["laptop.kept-bot"])
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// A send keeps, beside each handed module's plan, what requires a module only pulled in (issue 765), so the
|
||||
// assignments mirror gives that plan a value of its own; an assigned module's plan is kept as the bytes it was.
|
||||
func TestASendKeepsWhatRequiresAPulledInModule(t *testing.T) {
|
||||
plan := catalogue.Plan{Templates: catalogue.PlanTemplates{Source: "http://store:5000/v2/notify/files",
|
||||
Digest: "sha256:" + strings.Repeat("d", 64)},
|
||||
Files: []catalogue.PlanFile{{ID: "rc", Path: "/home/op/rc", Template: "rc", OwnerAtRuntime: "owned"}}}
|
||||
r := sentRecordOf(t.Context(), sendable{plans: map[string]catalogue.Plan{"dunst": plan, "notify": plan},
|
||||
requiredBy: map[string]string{"notify": "desk"}})
|
||||
kept := string(r.plans)
|
||||
want := `{"dunst":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` +
|
||||
strings.Repeat("d", 64) + `"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}]},` +
|
||||
`"notify":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` + strings.Repeat("d", 64) +
|
||||
`"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}],"required-by":"desk"}}`
|
||||
if kept != want {
|
||||
t.Fatalf("the send keeps\n%s\nwant\n%s", kept, want)
|
||||
}
|
||||
}
|
||||
|
||||
// **A module only pulled in by another is handed on a machine that can take it** (issue 765): on a contract-11
|
||||
// node running a client, a moved module that nothing assigned there, required by one that is, has its files and its
|
||||
// own secret handed like an assigned one's, no secret of it made or sealed by the controller, and its plan mirrored
|
||||
// under its own key, saying what requires it. A module not moved, pulled in the same way, is sent as before.
|
||||
@@ -338,7 +338,6 @@ func printStatus(asked answers) error {
|
||||
|
||||
// What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not
|
||||
// break "all well"; a record that could not be read does.
|
||||
printLastSends(nodes, asked.lastSent, asked.lastSentUnread)
|
||||
|
||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||
@@ -474,12 +473,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
out.pendingUnread = err.Error()
|
||||
err = nil
|
||||
}
|
||||
// And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through
|
||||
// the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said.
|
||||
if out.lastSent, err = inv.LastSends(ctx); err != nil {
|
||||
out.lastSentUnread = err.Error()
|
||||
err = nil
|
||||
}
|
||||
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
||||
// to read the log from; a process with none has no log to count.
|
||||
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||
@@ -604,8 +597,7 @@ func (a answers) well() bool {
|
||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
||||
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) &&
|
||||
a.lastSentUnread == ""
|
||||
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
|
||||
}
|
||||
|
||||
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
|
||||
|
||||
@@ -196,11 +196,6 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
|
||||
if report.Ordered() {
|
||||
link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
|
||||
}
|
||||
// A send refused for the generation it was composed from is kept on the send it refused, so S20 names
|
||||
// its sender (novox/hq issue 234).
|
||||
if report.OlderGeneration != nil {
|
||||
heardGenerationRefusal(ctx, l.Enrolment.Inventory, report)
|
||||
}
|
||||
// What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its
|
||||
// probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey.
|
||||
if report.Superseded == "" && report.Rekey == nil && report.Node != "" {
|
||||
|
||||
@@ -34,9 +34,6 @@ func TestPlanAndNodeListAnOptionalRequirementLeftUnbound(t *testing.T) {
|
||||
}
|
||||
}
|
||||
const said = "unbound influxdb-api, optional for grafana"
|
||||
if shown := printed(t, func() error { return planCommand(ctx, []string{"laptop"}) }); !strings.Contains(shown, said) {
|
||||
t.Errorf("plan laptop does not say influxdb-api is unbound:\n%s", shown)
|
||||
}
|
||||
if shown := printed(t, func() error { return showNode(ctx, open, "laptop") }); !strings.Contains(shown, said) {
|
||||
t.Errorf("node show laptop does not say influxdb-api is unbound:\n%s", shown)
|
||||
}
|
||||
|
||||
+113
-23
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/gosource"
|
||||
"regexp"
|
||||
@@ -127,7 +128,7 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
|
||||
// notNow marks a store that could not be read right now, or plans another controller is working (issue 921), so
|
||||
// the announcement is held rather than lost; anything else is returned as it was.
|
||||
func notNow(err error) error {
|
||||
if inventory.Unreachable(err) || errors.Is(err, inventory.ErrPlansBusy) {
|
||||
if inventory.Unreachable(err) {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return err
|
||||
@@ -147,8 +148,9 @@ func unsaved(err error) error {
|
||||
// becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go.
|
||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
// A merge plans nothing here any more: mesh-delivery merges on the Approve, the indexer writes the fingerprints and
|
||||
// the builder builds them (ADR 0539 §5, §17). The controller holds no batch for it.
|
||||
return nil
|
||||
// the builder builds them (ADR 0539 §5, §17). The controller keeps the announcement, for the build of a merge that
|
||||
// moves a module between repositories to be shown the operator's Approve (ADR 0420 §7).
|
||||
return keepForgeMerge(ctx, m)
|
||||
}
|
||||
|
||||
// movesOfMerge is what one merge moves, acted on: every module recorded as built from that repository and
|
||||
@@ -587,26 +589,6 @@ func packagesSaid(read []inventory.ReadRepository, m link.SourceMoved) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// lookedAt is when a merge was last acted on for a module that packages another repository's source: its
|
||||
// newest build, or the newest plan that built it or is still building it, whichever is later. A build asked
|
||||
// after a merge clones that repository with the merge in it, so an older merge is history for it; a plan
|
||||
// that closed without building it looked at nothing.
|
||||
func lookedAt(name string, read []inventory.ReadRepository, plans []inventory.Plan) time.Time {
|
||||
var at time.Time
|
||||
for _, r := range read {
|
||||
if r.Looked.After(at) {
|
||||
at = r.Looked
|
||||
}
|
||||
}
|
||||
for _, p := range plans {
|
||||
s, in := p.Modules[name]
|
||||
if in && (p.Open() || (s != nil && s.State == "built")) && p.Created.After(at) {
|
||||
at = p.Created
|
||||
}
|
||||
}
|
||||
return at
|
||||
}
|
||||
|
||||
// readForPlanning is what each module's build read, as a change is mapped onto it — for a merge's reach, a pull request's
|
||||
// check and the facts alike (novox/hq ADR 0238), so they cannot disagree. No plan overtakes it any more (ADR 0539 §17).
|
||||
func readForPlanning(ctx context.Context, inv *inventory.Inventory) (map[string][]inventory.ReadRepository, error) {
|
||||
@@ -936,3 +918,111 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string {
|
||||
|
||||
// edgeChangeOrder is the edge kind a change order draws.
|
||||
const edgeChangeOrder = "change-order"
|
||||
|
||||
// upgradeCommand says what should happen when a module's current version moves (ADR 0162 §3, ADR
|
||||
// 0235): every module's policy and where it comes from; one module's; or a person's choice for one.
|
||||
func upgradeCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("upgrade", flag.ContinueOnError)
|
||||
together := set.Bool("together", false,
|
||||
"send every machine running it at once, instead of one machine first")
|
||||
why := set.String("why", "", "why this choice — required for record; kept and said with the policy")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
if len(positionals) == 0 {
|
||||
all, err := inv.Upgrades(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, 0, len(all))
|
||||
for n := range all {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
u := all[n]
|
||||
line := fmt.Sprintf("%-28s %-9s %s", n, u.Policy(), u.From)
|
||||
if u.Why != "" {
|
||||
line += ": " + u.Why
|
||||
}
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
module := positionals[0]
|
||||
if len(positionals) == 1 {
|
||||
decision, err := inv.UpgradeOf(ctx, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sayUpgrade(module, decision))
|
||||
return nil
|
||||
}
|
||||
|
||||
decision := inventory.Upgrade{Why: strings.TrimSpace(*why), By: link.Caller()}
|
||||
switch positionals[1] {
|
||||
case "roll-out", "roll":
|
||||
decision.RollOut, decision.Together = true, *together
|
||||
case "record":
|
||||
if *together {
|
||||
// Refused rather than ignored: --together only means anything for a roll-out, and
|
||||
// accepting it here would store a preference that never applies and looks like it does.
|
||||
return errors.New("`--together` says how to roll out, so it cannot be given with " +
|
||||
"`record`, which is the choice not to")
|
||||
}
|
||||
if decision.Why == "" {
|
||||
return fmt.Errorf("holding %s back from every merge is a choice a person reads later: --why <text> "+
|
||||
"(novox/hq ADR 0236). Nothing was changed", module)
|
||||
}
|
||||
case "default":
|
||||
if err := inv.ClearUpgradeOf(ctx, module); err != nil {
|
||||
return err
|
||||
}
|
||||
decision, err := inv.UpgradeOf(ctx, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sayUpgrade(module, decision))
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("upgrade <module> roll-out|record|default — not %q", positionals[1])
|
||||
}
|
||||
if err := inv.SetUpgradeOf(ctx, module, decision); err != nil {
|
||||
return err
|
||||
}
|
||||
decision, err = inv.UpgradeOf(ctx, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sayUpgrade(module, decision))
|
||||
return nil
|
||||
}
|
||||
|
||||
func sayUpgrade(module string, u inventory.Upgrade) string {
|
||||
from := " (" + u.From
|
||||
if u.By != "" {
|
||||
from += ", " + u.By
|
||||
}
|
||||
if u.Why != "" {
|
||||
from += ": " + u.Why
|
||||
}
|
||||
from += ")"
|
||||
if !u.RollOut {
|
||||
return fmt.Sprintf("when %s moves, the mesh records it and the machines running it are "+
|
||||
"reported as behind until a person pushes them%s", module, from)
|
||||
}
|
||||
if u.Together {
|
||||
return fmt.Sprintf("when %s moves, every machine running it is sent the new version "+
|
||||
"together%s", module, from)
|
||||
}
|
||||
return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+
|
||||
"the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from)
|
||||
}
|
||||
|
||||
@@ -146,25 +146,6 @@ type machineFacts struct {
|
||||
// asleep says the machine said it would be away and has not said it is back (ADR 0211).
|
||||
func (m machineFacts) asleep() bool { return m.power.Away() }
|
||||
|
||||
type planFacts struct {
|
||||
id, repository, commit string
|
||||
tier, tiers int
|
||||
entered time.Time
|
||||
bound time.Duration
|
||||
waiting string
|
||||
bus bool
|
||||
}
|
||||
|
||||
// waitFacts is one walk waiting for its delivery's word: since its merge opened it.
|
||||
type waitFacts struct {
|
||||
id, repository, commit, awaits string
|
||||
since time.Time
|
||||
// modules are the modules the walk moves, as a person names the delivery.
|
||||
modules []string
|
||||
// merges are the merges it answers (novox/hq ADR 0276), as the deliveries to read are found.
|
||||
merges []inventory.PlanMerge
|
||||
}
|
||||
|
||||
type loopFacts struct {
|
||||
took time.Time
|
||||
pending uint64
|
||||
@@ -415,14 +396,7 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("where the node tools run cannot be read: %w", err)
|
||||
}
|
||||
applies, err := inv.Durations(ctx, inventory.DurationApply, now.Add(-7*24*time.Hour))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the measured applies cannot be read: %w", err)
|
||||
}
|
||||
lastApply := map[string]time.Duration{}
|
||||
for _, d := range applies { // oldest first: the last one read is the newest
|
||||
lastApply[d.Node] = d.Took
|
||||
}
|
||||
var out []machineFacts
|
||||
anyPast := false
|
||||
for _, n := range nodes {
|
||||
|
||||
@@ -49,23 +49,23 @@ func TestABuildIsKeptWholeAndReadBackWithItsReuseKeyAndWhatItPublished(t *testin
|
||||
func TestAWriteAgainstAMovedRevisionIsRefusedAndNothingIsLaidOver(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
first, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"})
|
||||
first, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes"}); !errors.Is(err, ErrExists) {
|
||||
t.Fatalf("a create of a key that holds a value answered %v", err)
|
||||
}
|
||||
second, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c2"}, first)
|
||||
second, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c2"}, first)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A writer that read the first revision writes now.
|
||||
if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "stale"}, first); !errors.Is(err, ErrMoved) {
|
||||
if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "stale"}, first); !errors.Is(err, ErrMoved) {
|
||||
t.Fatalf("a write at a moved revision answered %v", err)
|
||||
}
|
||||
got, _, _ := c.Assignments.Get(ctx, "anchor.notes")
|
||||
if got.Value.Commit != "c2" || got.Revision != second {
|
||||
if got.Value.RequiredBy != "c2" || got.Revision != second {
|
||||
t.Fatalf("the key that moved was laid over: %+v at %d", got.Value, got.Revision)
|
||||
}
|
||||
}
|
||||
@@ -91,8 +91,8 @@ func TestNoKeyOfTheChainIsEverRemoved(t *testing.T) {
|
||||
}
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
rev, _ := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"})
|
||||
if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}, rev); err != nil {
|
||||
rev, _ := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"})
|
||||
if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"}, rev); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
keys, err := c.Assignments.Keys(ctx)
|
||||
@@ -190,31 +190,36 @@ func TestARolloutNamesEveryAssignmentItChangedWithBuildBeforeAndAfter(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
// **An assignment names its build, artifact or package, commit and rollout**, and keeps the fields its readers read
|
||||
// under the names they read them by (ADR 0408 §3).
|
||||
func TestAnAssignmentNamesBuildArtifactCommitAndRollout(t *testing.T) {
|
||||
// **An assignment is placement only** (ADR 0539 §12): the node, the module, since when, what requires it and its
|
||||
// provisioning plan; no build, artifact, package, commit, rollout or time since is written in it, and a value of before
|
||||
// release 6 that names them is read with them ignored.
|
||||
func TestAnAssignmentIsPlacementOnly(t *testing.T) {
|
||||
a := Assignment{Node: "anchor", Module: "notes", Assigned: at(3), RequiredBy: "wiki",
|
||||
Provisioning: json.RawMessage(`{"plan":1}`), Build: "b2", Artifact: "sha256:bb", Commit: "c2", Rollout: "r1", Since: at(4)}
|
||||
Provisioning: json.RawMessage(`{"plan":1}`)}
|
||||
raw, _ := json.Marshal(a)
|
||||
for _, want := range []string{`"node":"anchor"`, `"module":"notes"`, `"assigned":"2026-10-18T03:00:00Z"`,
|
||||
`"required-by":"wiki"`, `"provisioning":{"plan":1}`, `"build":"b2"`, `"artifact":"sha256:bb"`, `"commit":"c2"`,
|
||||
`"rollout":"r1"`, `"since":"2026-10-18T04:00:00Z"`} {
|
||||
`"required-by":"wiki"`, `"provisioning":{"plan":1}`} {
|
||||
if !strings.Contains(string(raw), want) {
|
||||
t.Errorf("the assignment's value lacks %s: %s", want, raw)
|
||||
}
|
||||
}
|
||||
for _, not := range []string{`"build"`, `"artifact"`, `"package"`, `"commit"`, `"rollout"`, `"since"`, `"pulled-in"`, `"sent-unknown"`} {
|
||||
if strings.Contains(string(raw), not) {
|
||||
t.Errorf("the assignment's value names %s: %s", not, raw)
|
||||
}
|
||||
}
|
||||
if a.Key() != "anchor.notes" {
|
||||
t.Errorf("key %s", a.Key())
|
||||
}
|
||||
// An assignment of today carries none of the new fields, and says none: a reader of today reads the bytes it did.
|
||||
raw, _ = json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at(3)})
|
||||
if string(raw) != `{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z"}` {
|
||||
t.Errorf("an assignment without a chain is %s", raw)
|
||||
// A value written before release 6 is read, its chain fields ignored, and written back without them.
|
||||
var old Assignment
|
||||
if err := json.Unmarshal([]byte(`{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z","build":"b2","commit":"c2",`+
|
||||
`"rollout":"r1","since":"2026-10-18T04:00:00Z","pulled-in":true}`), &old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A package in place of an artifact.
|
||||
raw, _ = json.Marshal(Assignment{Node: "a", Module: "m", Package: &Package{Name: "n", Version: "1"}})
|
||||
if !strings.Contains(string(raw), `"package":{"name":"n","version":"1"}`) {
|
||||
t.Errorf("the package is %s", raw)
|
||||
raw, _ = json.Marshal(old)
|
||||
if string(raw) != `{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z"}` {
|
||||
t.Errorf("an old value written again is %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -341,7 +346,7 @@ func TestAnEmptyBucketIsReadAsEmpty(t *testing.T) {
|
||||
func TestAnEndedAssignmentIsNoAssignmentAndStaysInTheBucket(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
c := InMemory(nil)
|
||||
rev, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Assigned: at(1), Commit: "c1"})
|
||||
rev, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Assigned: at(1), RequiredBy: "c1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -355,7 +360,7 @@ func TestAnEndedAssignmentIsNoAssignmentAndStaysInTheBucket(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, found, _ := c.Assignments.Get(ctx, "anchor.notes")
|
||||
if !found || got.Value.Stands() || got.Value.Commit != "c1" || got.Value.Ended.IsZero() {
|
||||
if !found || got.Value.Stands() || got.Value.RequiredBy != "c1" || got.Value.Ended.IsZero() {
|
||||
t.Fatalf("the ended value is %+v (found %v)", got.Value, found)
|
||||
}
|
||||
if history, _ := c.Assignments.History(ctx, "anchor.notes"); len(history) != 2 || !history[0].Value.Stands() {
|
||||
|
||||
@@ -61,27 +61,3 @@ func TestTheTransitionsOfAPendingAssignment(t *testing.T) {
|
||||
t.Error("a claim moves a wait to applying and back")
|
||||
}
|
||||
}
|
||||
|
||||
// **A module only pulled in is a standing value marked pulled-in, never a state, and a hand send makes a key say its commit
|
||||
// unknown with a time** (issue 1109): every reader treats a non-empty state as no assignment, so a pulled-in key must have
|
||||
// none; and unknown is never written in `commit`.
|
||||
func TestAPulledInValueStandsAndAHandSendIsATimeNotACommit(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
pulled := Assignment{Node: "anchor", Module: "mesh-wireguard", PulledIn: true, Commit: "c1", Build: "b1", Since: at}
|
||||
if !pulled.Stands() || pulled.State != "" {
|
||||
t.Fatalf("a pulled-in value does not stand: %+v", pulled)
|
||||
}
|
||||
raw, _ := json.Marshal(pulled)
|
||||
if !strings.Contains(string(raw), `"pulled-in":true`) || strings.Contains(string(raw), `"state"`) || strings.Contains(string(raw), `"assigned"`) {
|
||||
t.Fatalf("the pulled-in value is %s", raw)
|
||||
}
|
||||
unknown := Assignment{Node: "anchor", Module: "notes", Assigned: at, SentUnknown: at}
|
||||
raw, _ = json.Marshal(unknown)
|
||||
if !strings.Contains(string(raw), `"sent-unknown":"2026-10-18T03:00:00Z"`) || strings.Contains(string(raw), `"commit"`) || !unknown.Stands() {
|
||||
t.Fatalf("the hand-sent value is %s", raw)
|
||||
}
|
||||
plain, _ := json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at})
|
||||
if strings.Contains(string(plain), "pulled-in") || strings.Contains(string(plain), "sent-unknown") {
|
||||
t.Fatalf("a value that is neither says so: %s", plain)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,9 +175,10 @@ type Rollout struct {
|
||||
Links
|
||||
}
|
||||
|
||||
// Assignment is one module on one node, kept under `<node>.<module>`. The first five fields are the ones its readers
|
||||
// have always read, with the names they have always had (novox/hq ADR 0408 §3); the rest name the chain: the build, the
|
||||
// artifact (by digest) or the package it runs, its commit, the rollout that put it there and since when.
|
||||
// Assignment is one module on one node, kept under `<node>.<module>`: placement only (novox/hq ADR 0539 §12). The first
|
||||
// five fields are the ones its readers have always read, with the names they have always had (novox/hq ADR 0408 §3);
|
||||
// no build, commit or rollout is written in it any more, and a value written before release 6 that names them is read
|
||||
// with them ignored.
|
||||
type Assignment struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
@@ -185,13 +186,6 @@ type Assignment struct {
|
||||
RequiredBy string `json:"required-by,omitempty"`
|
||||
Provisioning json.RawMessage `json:"provisioning,omitempty"`
|
||||
|
||||
Build string `json:"build,omitempty"`
|
||||
Artifact string `json:"artifact,omitempty"`
|
||||
Package *Package `json:"package,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Rollout string `json:"rollout,omitempty"`
|
||||
Since time.Time `json:"since,omitzero"`
|
||||
|
||||
// Ended is when the assignment ended (an unassignment): absent while it stands. **No key of the chain is removed**
|
||||
// (ADR 0408 §4), so an unassigned module keeps its key, saying so; every reader treats a value with Ended, or with a
|
||||
// State, as no assignment. State is StatePending while the key holds an assignment waiting for its build, or one that
|
||||
@@ -201,14 +195,6 @@ type Assignment struct {
|
||||
// Pending is the assignment's wait for its build (novox/hq ADR 0261), kept in the value of the key it waits for and, when
|
||||
// it is made, carried into the value that stands: the record of what became of the wait.
|
||||
Pending *Pending `json:"pending,omitempty"`
|
||||
|
||||
// PulledIn marks a standing value of a module that is on the machine only because another requires it, or because a
|
||||
// send carried it: no time assigned, but a key all the same, so what the machine was sent of it is kept and read
|
||||
// (issue 1109). Never a state: every reader treats a non-empty state as no assignment, and would take it off the machine.
|
||||
PulledIn bool `json:"pulled-in,omitempty"`
|
||||
// SentUnknown is when a send by hand made what the machine runs unknown (a declaration sent with no builds known): the
|
||||
// commit is cleared with it. Absent normally; a send that names its builds clears it.
|
||||
SentUnknown time.Time `json:"sent-unknown,omitzero"`
|
||||
}
|
||||
|
||||
// StatePending is the State of a value that holds no assignment but a pending one.
|
||||
|
||||
@@ -1,138 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The merges a batch or walk answers (novox/hq ADR 0276): every merge the controller heard into a branch a
|
||||
// module follows is kept here once, with the batch or walk that answers it. A merge is never dropped and never
|
||||
// walked twice: its key is its repository and commit, and it belongs to one plan at a time.
|
||||
|
||||
// BatchedMerge is one merge as the controller heard it.
|
||||
type BatchedMerge struct {
|
||||
Repository string
|
||||
Branch string
|
||||
Commit string
|
||||
// Merged is when the forge made it: the branch's order. Heard is when the controller heard it: what the
|
||||
// window is measured from.
|
||||
Merged time.Time
|
||||
Heard time.Time
|
||||
// Event is the forge's announcement, what the cut plans from.
|
||||
Event json.RawMessage
|
||||
// Plan is the batch or walk answering it; empty while it waits to be walked alone.
|
||||
Plan string
|
||||
// Alone says it is walked on its own commit, after a failed walk that carried it in a later one.
|
||||
Alone bool
|
||||
}
|
||||
|
||||
// AddMerge keeps a merge for a plan; false when the merge was kept already, for whatever plan.
|
||||
func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return i.chainAddMerge(ctx, c, m)
|
||||
}
|
||||
|
||||
// MergeOf is one kept merge; false when it was never heard.
|
||||
func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (BatchedMerge, bool, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return BatchedMerge{}, false, err
|
||||
}
|
||||
return i.chainMergeOf(ctx, c, repository, commit)
|
||||
}
|
||||
|
||||
// MergesOf is every merge a plan answers, oldest merge first.
|
||||
func (i *Inventory) MergesOf(ctx context.Context, plan string) ([]BatchedMerge, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return i.chainMergesOf(ctx, c, plan)
|
||||
}
|
||||
|
||||
// LaterMergesOf is every merge of a repository's branch made after a moment and answered by a plan, the
|
||||
// newest first.
|
||||
func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string, after time.Time) ([]BatchedMerge, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return i.chainLaterMergesOf(ctx, c, repository, branch, after)
|
||||
}
|
||||
|
||||
// KeptOnBranch says the controller keeps a merge of a repository's branch, and when it heard the branch's first: a merge of a branch it keeps merges of, made since then, is the record's to answer, whatever a
|
||||
// module's last look says (novox/hq ADR 0276).
|
||||
func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) (bool, time.Time, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return false, time.Time{}, err
|
||||
}
|
||||
return i.chainKeptOnBranch(ctx, c, repository, branch)
|
||||
}
|
||||
|
||||
// EarliestKept says the controller keeps a merge of some repository's branch, and when it heard the first of all: where its record of
|
||||
// merges begins on the whole mesh (novox/hq ADR 0463). A merge made before then is history on every trunk.
|
||||
func (i *Inventory) EarliestKept(ctx context.Context) (bool, time.Time, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return false, time.Time{}, err
|
||||
}
|
||||
return i.chainEarliestKept(ctx, c)
|
||||
}
|
||||
|
||||
// AloneMerges is every merge waiting to be walked on its own commit, the newest first (ADR 0276 decision 3).
|
||||
func (i *Inventory) AloneMerges(ctx context.Context) ([]BatchedMerge, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return i.chainAloneMerges(ctx, c)
|
||||
}
|
||||
|
||||
// WaitingMerges is every merge no plan answers that is not walked alone: kept, and waiting for something else to
|
||||
// happen to it (a first merge into a repository, waiting for mesh-delivery's word on it, issue 1146).
|
||||
func (i *Inventory) WaitingMerges(ctx context.Context) ([]BatchedMerge, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return i.chainWaitingMerges(ctx, c)
|
||||
}
|
||||
|
||||
// AnswerMerge moves a merge to the plan that answers it now; an empty plan with alone leaves it waiting to
|
||||
// be walked on its own commit.
|
||||
func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan string, alone bool) error {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}})
|
||||
}
|
||||
|
||||
// HoldMerge puts a merge back to waiting, no plan answering it, with the event it is kept under replaced (novox/hq issue 1173:
|
||||
// a first merge held for a build-agent that reads its manifest, the hold said in its event).
|
||||
func (i *Inventory) HoldMerge(ctx context.Context, repository, commit string, event json.RawMessage) error {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Event: event}})
|
||||
}
|
||||
|
||||
// Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more
|
||||
// are read so a fault is seen.
|
||||
func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated)
|
||||
}
|
||||
|
||||
// SameRepository says two spellings of a repository are one.
|
||||
func SameRepository(a, b string) bool { return strings.EqualFold(a, b) }
|
||||
@@ -0,0 +1,34 @@
|
||||
package inventory
|
||||
|
||||
import "context"
|
||||
|
||||
// Lookups of builds the delivery plans once held, which other code still reads.
|
||||
|
||||
// BuildOf is the id of the newest successful build of a module from a commit; empty when none is
|
||||
// recorded (a manifest handed over by hand).
|
||||
func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return i.chainBuildOf(ctx, c, module, commit)
|
||||
}
|
||||
|
||||
// BuildFingerprints is, per commit, what the newest successful build of a module from it would put on a
|
||||
// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told
|
||||
// from one that does (the bus's planned step, ADR 0236).
|
||||
func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return i.chainBuildFingerprints(ctx, c, module)
|
||||
}
|
||||
|
||||
// short is a commit as a person reads it.
|
||||
func short(commit string) string {
|
||||
if len(commit) > 8 {
|
||||
return commit[:8]
|
||||
}
|
||||
return commit
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC)
|
||||
waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting,
|
||||
Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}}
|
||||
keep := func(busy []BusyWait) bool {
|
||||
t.Helper()
|
||||
news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1,
|
||||
Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return news
|
||||
}
|
||||
read := func() []BusyWait {
|
||||
t.Helper()
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range reports {
|
||||
if r.Node == node {
|
||||
return r.Busy
|
||||
}
|
||||
}
|
||||
t.Fatal("the machine's report is not read back")
|
||||
return nil
|
||||
}
|
||||
keep(nil)
|
||||
if got := read(); len(got) != 0 {
|
||||
t.Fatalf("a report with nothing busy reads back %+v", got)
|
||||
}
|
||||
if !keep(waiting) {
|
||||
t.Error("a container starting to wait is not news")
|
||||
}
|
||||
got := read()
|
||||
if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" {
|
||||
t.Fatalf("the wait is not read back: %+v", got)
|
||||
}
|
||||
again := []BusyWait{waiting[0]}
|
||||
again[0].Said = "busy: a check, a minute on"
|
||||
if keep(again) {
|
||||
t.Error("a wait whose output changed is news")
|
||||
}
|
||||
if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle",
|
||||
Since: since, Until: since.Add(15 * time.Minute)}}) {
|
||||
t.Error("a container replaced after it waited is not news")
|
||||
}
|
||||
if _, ok := WaitingFor(read(), "build-agent"); ok {
|
||||
t.Error("a container replaced still reads as waiting")
|
||||
}
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A walk whose first machine waited for a busy container counts the wait as a phase of its own, between the
|
||||
// send to the first machine and the judging, and its delivery time holds it (novox/hq ADR 0335 §4): the
|
||||
// recorded walk again, its judging started only at 17:54:07, a minute after the send, when the machine said
|
||||
// the container replaced.
|
||||
func TestABusyWaitIsAPhaseOfTheDeliveryTime(t *testing.T) {
|
||||
raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",`,
|
||||
`"gate": {"machines": ["novox"], "since": "2026-10-10T17:54:07.287136827Z",
|
||||
"busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery",
|
||||
"since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z", "ended": "2026-10-10T17:54:07.287136827Z"},`, 1)
|
||||
w := walkOf(t, raw).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil)
|
||||
sumsUp(t, w)
|
||||
if w.TotalMS != 282894 {
|
||||
t.Fatalf("the wait changed the delivery's total: %dms", w.TotalMS)
|
||||
}
|
||||
busy := phase(w, PhaseBusy, 0)
|
||||
if busy.State != PhaseMeasured || busy.TookMS != 60000 {
|
||||
t.Fatalf("the busy wait is not a measured phase of a minute: %+v", busy)
|
||||
}
|
||||
if judged := phase(w, PhaseJudge, 0); judged.TookMS != 148607-60000 {
|
||||
t.Fatalf("the judging still counts from the send, not from the replacement: %+v", judged)
|
||||
}
|
||||
var names []string
|
||||
for _, ph := range w.Phases {
|
||||
names = append(names, ph.Name)
|
||||
}
|
||||
if !strings.Contains(strings.Join(names, " "), PhaseSend+" "+PhaseBusy+" "+PhaseJudge) {
|
||||
t.Fatalf("the busy wait is not between the send and the judging: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
// While the wait goes on, the walk says it waits for a busy container there; a walk where nothing waited has no
|
||||
// such phase.
|
||||
func TestAnOpenBusyWaitIsSaid(t *testing.T) {
|
||||
raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", "passes": 3,
|
||||
"verdict": "passed", "judged_at": "2026-10-10T17:55:35.894985053Z"}`,
|
||||
`"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",
|
||||
"busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery",
|
||||
"since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z"}}`, 1)
|
||||
raw = strings.Replace(raw, `"state": "done"`, `"state": "rolling"`, 1)
|
||||
p := walkOf(t, raw)
|
||||
if !p.Modules["mesh-delivery"].Gate.Waiting() {
|
||||
t.Fatal("a gate whose busy wait has not ended does not say it waits")
|
||||
}
|
||||
w := p.WalkPhases(time.Date(2026, 10, 10, 17, 58, 0, 0, time.UTC), nil)
|
||||
busy := phase(w, PhaseBusy, 0)
|
||||
if busy.State != PhaseUnknown || !strings.Contains(busy.Said, "waits for a busy container on novox") {
|
||||
t.Fatalf("an open busy wait is not said: %+v", busy)
|
||||
}
|
||||
if plain := walkOf(t, recordedWalk).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil); phase(plain, PhaseBusy, 0).Name != "" {
|
||||
t.Fatalf("a walk where nothing waited has a busy phase: %+v", plain.Phases)
|
||||
}
|
||||
}
|
||||
|
||||
// A container a report lists waiting is its module's by its id; its output alone is not a new account.
|
||||
func TestABusyContainerIsItsModules(t *testing.T) {
|
||||
list := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, Said: "a"},
|
||||
{ID: "web.server", State: "replaced-when-idle"}}
|
||||
if b, ok := WaitingFor(list, "build-agent"); !ok || b.Container != "mesh-build-agent" {
|
||||
t.Fatalf("build-agent's container is not found waiting: %+v", b)
|
||||
}
|
||||
if _, ok := WaitingFor(list, "web"); ok {
|
||||
t.Error("a container replaced reads as waiting")
|
||||
}
|
||||
if _, ok := WaitingFor(list, "build"); ok {
|
||||
t.Error("another module's container is found by a prefix of its name")
|
||||
}
|
||||
again := []BusyWait{{ID: "build-agent.server", State: BusyWaiting, Said: "b, a minute on"}, list[1]}
|
||||
if !sameBusy(list, again) {
|
||||
t.Error("a wait whose output changed is a new account")
|
||||
}
|
||||
if sameBusy(list, list[1:]) || !(Doing{Busy: list}).Waiting() || (Doing{Busy: list[1:]}).Waiting() {
|
||||
t.Error("a wait that ended is not a new account")
|
||||
}
|
||||
}
|
||||
|
||||
// A machine's report of a busy container is kept with its last account and read back beside it, and its
|
||||
// starting and its end are news; its output alone is not (novox/hq ADR 0335). Needs the store.
|
||||
|
||||
// A machine of the rest that answered and waits for a busy container is never read as silent, its wait is kept
|
||||
// out of its apply time, and once replaced its report is the end of the wait (novox/hq ADR 0335).
|
||||
func TestARestMachineWaitingForABusyContainerIsNeverSilent(t *testing.T) {
|
||||
c := walk582(t)
|
||||
shanks := c.walk.Modules["build-agent"].Rest["shanks"]
|
||||
ref := shanks.Digest + "@" + shanks.At.Format("2006-01-02T15:04:05.000000")
|
||||
answered := shanks.At.Add(15 * time.Second)
|
||||
// shanks answered, and said a container waits: its apply row is the answer, marked busy.
|
||||
c.rows["shanks"][0].Detail = ApplyBusy
|
||||
at := c.judged.Add(20 * time.Minute) // past ApplySilentAfter from its send
|
||||
w := c.walk.WalkPhases(at, c.lookup)
|
||||
var got RestReport
|
||||
for _, r := range w.Rest {
|
||||
if r.Node == "shanks" {
|
||||
got = r
|
||||
}
|
||||
}
|
||||
if got.State != RestBusy || slicesContains(w.Silent, "shanks") {
|
||||
t.Fatalf("a machine waiting for a busy container reads %q (silent: %v)", got.State, w.Silent)
|
||||
}
|
||||
if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "waits for a busy container on shanks") {
|
||||
t.Fatalf("the walk ended, or does not say it waits: %+v", phase(w, PhaseApply, -1))
|
||||
}
|
||||
|
||||
// 17 minutes on, shanks says it replaced the container: its wait is its own duration, and its report is then.
|
||||
c.rows["shanks"] = append(c.rows["shanks"], Duration{Kind: DurationBusy, Subject: "shanks", Node: "shanks", Ref: ref,
|
||||
Started: answered, Took: 17 * time.Minute})
|
||||
w = c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
|
||||
sumsUp(t, w)
|
||||
for _, r := range w.Rest {
|
||||
if r.Node == "shanks" {
|
||||
got = r
|
||||
}
|
||||
}
|
||||
if got.State != RestApplied || got.Busy != "17m0s" || !got.Reported.Equal(answered.Add(17*time.Minute)) {
|
||||
t.Fatalf("a machine replaced after its wait reads %+v", got)
|
||||
}
|
||||
if slicesContains(w.Silent, "shanks") || !w.End.Equal(answered.Add(17*time.Minute)) {
|
||||
t.Fatalf("the walk read shanks as silent, or ended elsewhere: %v %v", w.Silent, w.End)
|
||||
}
|
||||
// And the apply time itself is the answer's, never the wait's.
|
||||
if r, _ := FirstReportOf(c.rows["shanks"], shanks, c.walkSent); !r.Answered.Equal(answered) || r.Busy != 17*time.Minute {
|
||||
t.Fatalf("the busy wait is not kept apart from the apply: %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func slicesContains(list []string, s string) bool {
|
||||
for _, x := range list {
|
||||
if x == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -25,10 +25,6 @@ var ErrNoChain = errors.New("this kind of record lives on the bus, and this proc
|
||||
type chainState struct {
|
||||
mu sync.Mutex
|
||||
open ChainOpener
|
||||
// view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that
|
||||
// is not saved yet.
|
||||
view *rolloutView
|
||||
pending map[string]pendingMerge
|
||||
// requireLayout is set for a chain on the bus: its builds are read only once the controller's start has laid them out
|
||||
// under their modules (issue 1121), and laid is that seen.
|
||||
requireLayout, laid bool
|
||||
@@ -75,10 +71,6 @@ func (i *Inventory) UseChain(c *chain.Chain) {
|
||||
i.state.mu.Lock()
|
||||
defer i.state.mu.Unlock()
|
||||
i.chain = c
|
||||
if i.state.view != nil {
|
||||
i.state.view.stop()
|
||||
i.state.view = nil
|
||||
}
|
||||
}
|
||||
|
||||
// UseThrowawayChain gives the inventory a chain in memory that is its own, laid out already: for a mesh raised for a moment
|
||||
|
||||
@@ -124,13 +124,6 @@ func (i *Inventory) isAssigned(ctx context.Context, node, module string) (bool,
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// raiseGeneration raises the assignment generation, as the store's triggers do for a change of its table (migration
|
||||
// 0094): in the chain nothing in the store changes with an assignment, so the writer says it.
|
||||
func (i *Inventory) raiseGeneration(ctx context.Context) error {
|
||||
_, err := i.store.Pool().Exec(ctx, `update assignment_generation set generation = generation + 1`)
|
||||
return err
|
||||
}
|
||||
|
||||
// chainAssign puts a module on a machine, and says whether that is new.
|
||||
func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, module string) (bool, error) {
|
||||
var known bool
|
||||
@@ -154,7 +147,7 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node,
|
||||
switch {
|
||||
case found && cur.Value.Stands():
|
||||
// A module only pulled in by another becomes an assignment, keeping what it holds.
|
||||
cur.Value.Assigned, cur.Value.PulledIn = now, false
|
||||
cur.Value.Assigned = now
|
||||
_, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision)
|
||||
case found:
|
||||
// One that ended is assigned again, and one that was pending is made: a fresh value over it, the earlier one
|
||||
@@ -170,9 +163,6 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node,
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s is not assigned to %s on the bus: %w", module, node.Name, err)
|
||||
}
|
||||
if err := i.raiseGeneration(ctx); err != nil {
|
||||
return true, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name)
|
||||
@@ -195,7 +185,7 @@ func (i *Inventory) chainUnassign(ctx context.Context, c *chain.Chain, node Node
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("%s is not taken off %s on the bus: %w", module, node.Name, err)
|
||||
}
|
||||
return i.raiseGeneration(ctx)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name)
|
||||
}
|
||||
@@ -273,11 +263,11 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i
|
||||
case keepSkip:
|
||||
continue
|
||||
case keepCreate:
|
||||
next.Assigned, next.PulledIn = time.Time{}, true
|
||||
next.Assigned = time.Time{}
|
||||
_, err = c.Assignments.Create(ctx, key, next)
|
||||
case keepFresh:
|
||||
// A value that ended, pulled in again: a fresh one over it, the ended one staying in history.
|
||||
next.Assigned, next.PulledIn = time.Time{}, true
|
||||
next.Assigned = time.Time{}
|
||||
_, err = c.Assignments.Update(ctx, key, next, cur.Revision)
|
||||
default:
|
||||
cur.Value.RequiredBy, cur.Value.Provisioning = next.RequiredBy, next.Provisioning
|
||||
@@ -292,8 +282,8 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i
|
||||
written++
|
||||
}
|
||||
for key, cur := range have {
|
||||
if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() || cur.Value.Commit != "" {
|
||||
continue // one a send still carries is on the machine, and ends when a send does not carry it any more
|
||||
if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() {
|
||||
continue
|
||||
}
|
||||
// A value of a module no longer pulled in, which no assignment names: it ends, and stays.
|
||||
cur.Value.Ended = time.Now().UTC()
|
||||
|
||||
@@ -810,33 +810,11 @@ func fillManifests(ctx context.Context, c *chain.Chain, builds []Build) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// failedAtAGate is the builds that failed their gate. The gate's verdicts stay in their table (issue 1097).
|
||||
func (i *Inventory) failedAtAGate(ctx context.Context) (map[string]bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select build from build_gate where verdict = 'failed'`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]bool{}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[id] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module string) (map[string][]sourceRow, []string, error) {
|
||||
all, err := chainBuildsOf(ctx, c, module)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
failed, err := i.failedAtAGate(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ok := worked(all)
|
||||
byModule(ok)
|
||||
out := map[string][]sourceRow{}
|
||||
@@ -853,7 +831,7 @@ func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module
|
||||
order = append(order, b.Module)
|
||||
}
|
||||
out[b.Module] = append(out[b.Module], sourceRow{id: b.ID, commit: b.Commit, fingerprint: b.SourceFingerprint,
|
||||
made: made, manifest: b.Manifest, manifestDigest: b.ManifestDigest, failedGate: failed[b.ID]})
|
||||
made: made, manifest: b.Manifest, manifestDigest: b.ManifestDigest})
|
||||
}
|
||||
return out, order, nil
|
||||
}
|
||||
@@ -1104,8 +1082,7 @@ func (i *Inventory) chainEveryReferenceMade(ctx context.Context, c *chain.Chain)
|
||||
|
||||
// chainHeldMade is, per module, what the build standing for its newest successful build made. Read from the pointers: the
|
||||
// module's `worked` (its newest success) and `held` (the first build of the run of one source that it stands on), one record
|
||||
// of each. A build that failed its gate ends a run (issue 280), and the pointer cannot say where; then the module's builds
|
||||
// are read.
|
||||
// of each.
|
||||
func (i *Inventory) chainHeldMade(ctx context.Context, c *chain.Chain) (map[string][]Artifact, error) {
|
||||
worked, err := i.pointedBuilds(ctx, c, "*.worked")
|
||||
if err != nil {
|
||||
@@ -1115,29 +1092,12 @@ func (i *Inventory) chainHeldMade(ctx context.Context, c *chain.Chain) (map[stri
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
failed, err := i.failedAtAGate(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string][]Artifact{}
|
||||
for module, w := range worked {
|
||||
stand := w
|
||||
if h, ok := held[module]; ok && !failed[w.ID] {
|
||||
if h, ok := held[module]; ok {
|
||||
stand = h
|
||||
}
|
||||
if failed[stand.ID] && stand.ID != w.ID {
|
||||
rows, _, err := i.chainSourceRows(ctx, c, module)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if builds := rows[module]; len(builds) > 0 {
|
||||
var made []Artifact
|
||||
if err := json.Unmarshal(builds[standing(builds, 0)].made, &made); err == nil {
|
||||
out[module] = made
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
out[module] = fromChainBuild(stand).Made
|
||||
}
|
||||
return out, nil
|
||||
|
||||
@@ -123,7 +123,6 @@ func TestTheBusBeingAwayIsNotNowLikeTheStoreBeingAway(t *testing.T) {
|
||||
for name, err := range map[string]error{
|
||||
"a value too large": chain.ErrTooLarge,
|
||||
"a key not taken": chain.ErrBadKey,
|
||||
"a plan moved": ErrPlanMoved,
|
||||
"a key that exists": chain.ErrExists,
|
||||
} {
|
||||
if Unreachable(err) {
|
||||
|
||||
@@ -23,3 +23,13 @@ func TestTheFirstReadOfAChainNotLaidOutLaysItOut(t *testing.T) {
|
||||
t.Fatalf("the read did not lay the builds out: found %v, %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// anInventoryOnTheChain is an inventory with no store and a chain in memory: the chain's own paths, which never reach the
|
||||
// store.
|
||||
func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) {
|
||||
t.Helper()
|
||||
c := chain.InMemory(nil)
|
||||
inv := &Inventory{state: &chainState{}}
|
||||
inv.UseChain(c)
|
||||
return inv, c
|
||||
}
|
||||
|
||||
@@ -2,15 +2,7 @@ package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
@@ -21,44 +13,6 @@ import (
|
||||
// release_plan.go). One read of an assignment then names the commit, the build and the rollout that put it there, and one read
|
||||
// of the rollout lists its assignments.
|
||||
|
||||
type rolloutKey struct{}
|
||||
|
||||
// RolloutModule is what a rollout sends of one module: the commit and the build of it.
|
||||
type RolloutModule struct{ Commit, Build string }
|
||||
|
||||
type rolloutOf struct {
|
||||
id string
|
||||
modules map[string]RolloutModule
|
||||
}
|
||||
|
||||
// WithRollout is a context for a send a rollout makes: what RecordSent writes in the assignments of the modules it carries
|
||||
// names this rollout and the build it sent. A module the send carries beside these, that another walk is moving, is named
|
||||
// by the build the mesh holds of its commit and by no rollout.
|
||||
func WithRollout(ctx context.Context, id string, modules map[string]RolloutModule) context.Context {
|
||||
return context.WithValue(ctx, rolloutKey{}, rolloutOf{id: id, modules: modules})
|
||||
}
|
||||
|
||||
// AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the
|
||||
// commit. Empty when the assignments are not on the bus, and for a module not sent yet.
|
||||
func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := map[string]chain.Ref{}
|
||||
for _, k := range all {
|
||||
a := k.Value
|
||||
if a.Node == node && a.Stands() && (a.Build != "" || a.Commit != "") {
|
||||
out[a.Module] = chain.Ref{Build: a.Build, Commit: a.Commit}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links.
|
||||
func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
@@ -78,285 +32,3 @@ func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Ass
|
||||
sort.Slice(out, func(a, b int) bool { return out[a].Module < out[b].Module })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// recordRuns writes what a send put on a machine into the assignments, once the send is recorded, and it is how the mesh
|
||||
// knows what a machine was sent (SentBuilds). **A send that names its builds** writes, for each module it carried a build of
|
||||
// — by the commit — the build, what it published, the commit, the rollout and since when, on the module's key: filled where
|
||||
// the key names nothing, replaced when the commit moved, never blanked by a push by hand; a module nothing assigns, carried
|
||||
// because another requires it, gets a standing key marked pulled-in; and every other key of the machine loses what it named,
|
||||
// for the machine is not sent it any more. **A send that names none** (a declaration by hand) makes what the machine runs
|
||||
// unknown: every key of it says so (sent-unknown) and names no commit. **An error here fails the record of the send**, for
|
||||
// what a machine was sent is read from these keys and from nowhere else.
|
||||
func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[string]string, at time.Time) error {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("what the machine runs cannot be written: %w", err)
|
||||
}
|
||||
var node string
|
||||
if err := i.store.Pool().QueryRow(ctx, `select name from node where id = $1`, nodeID).Scan(&node); errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
rollout, _ := ctx.Value(rolloutKey{}).(rolloutOf)
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the assignments of %s cannot be read: %w", node, err)
|
||||
}
|
||||
var failed []error
|
||||
for _, k := range sortedValues(all) {
|
||||
a := k.Value
|
||||
if a.Node != node || !a.Stands() {
|
||||
continue
|
||||
}
|
||||
if builds[a.Module] != "" {
|
||||
continue // carried, and its commit is written below
|
||||
}
|
||||
if err := i.forgetRun(ctx, c, k, builds == nil, at); err != nil {
|
||||
failed = append(failed, fmt.Errorf("%s on %s: %w", a.Module, node, err))
|
||||
}
|
||||
}
|
||||
modules := make([]string, 0, len(builds))
|
||||
for m := range builds {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
for _, m := range modules {
|
||||
if builds[m] == "" {
|
||||
continue
|
||||
}
|
||||
if err := i.recordRun(ctx, c, node, m, builds[m], rollout, at); err != nil {
|
||||
failed = append(failed, fmt.Errorf("%s on %s: %w", m, node, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(failed...)
|
||||
}
|
||||
|
||||
func sortedValues(all map[string]chain.Kept[chain.Assignment]) []chain.Kept[chain.Assignment] {
|
||||
out := make([]chain.Kept[chain.Assignment], 0, len(all))
|
||||
for key, k := range all {
|
||||
k.Key = key
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key })
|
||||
return out
|
||||
}
|
||||
|
||||
// forgetRun writes that a machine is no longer sent what a key named: its links are cleared, and for a send by hand the key
|
||||
// says what the machine runs is unknown.
|
||||
func (i *Inventory) forgetRun(ctx context.Context, c *chain.Chain, k chain.Kept[chain.Assignment], hand bool, at time.Time) error {
|
||||
for try := 0; try < casTries; try++ {
|
||||
next := k.Value
|
||||
next.Build, next.Artifact, next.Package, next.Commit, next.Rollout, next.Since = "", "", nil, "", "", time.Time{}
|
||||
next.SentUnknown = time.Time{}
|
||||
if hand {
|
||||
next.SentUnknown = at.UTC()
|
||||
}
|
||||
if next.Build == k.Value.Build && next.Commit == k.Value.Commit && next.Rollout == k.Value.Rollout &&
|
||||
next.Artifact == k.Value.Artifact && next.Package == k.Value.Package && next.Since.Equal(k.Value.Since) &&
|
||||
next.SentUnknown.Equal(k.Value.SentUnknown) {
|
||||
return nil
|
||||
}
|
||||
if _, err := c.Assignments.Update(ctx, k.Key, next, k.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
cur, found, gerr := c.Assignments.Get(ctx, k.Key)
|
||||
if gerr != nil {
|
||||
return gerr
|
||||
}
|
||||
if !found || !cur.Value.Stands() {
|
||||
return nil
|
||||
}
|
||||
k = chain.Kept[chain.Assignment]{Key: k.Key, Value: cur.Value, Revision: cur.Revision}
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the assignment kept moving under the write")
|
||||
}
|
||||
|
||||
func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, commit string, rollout rolloutOf, at time.Time) error {
|
||||
key := assignmentKey(node, module)
|
||||
var build, rolloutID string
|
||||
if sent, ours := rollout.modules[module]; ours && sent.Commit == commit {
|
||||
build, rolloutID = sent.Build, rollout.id
|
||||
}
|
||||
var published *chain.Published
|
||||
resolved := false
|
||||
for try := 0; try < casTries; try++ {
|
||||
cur, found, err := c.Assignments.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found && cur.Value.State != "" {
|
||||
return nil // a wait for a build: the module is not on the machine by it
|
||||
}
|
||||
standing := found && cur.Value.Stands()
|
||||
same := standing && cur.Value.Commit == commit
|
||||
if same && cur.Value.Build != "" && cur.Value.SentUnknown.IsZero() && (rolloutID == "" || cur.Value.Rollout != "") {
|
||||
return nil // a send that carries what the machine runs already says nothing new
|
||||
}
|
||||
if !resolved {
|
||||
// The build of the commit is looked up once, and only for a module that moved or names no build.
|
||||
if build == "" {
|
||||
if build, err = i.BuildOf(ctx, module, commit); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if build != "" {
|
||||
if b, found, err := i.BuildByID(ctx, build); err != nil {
|
||||
return err
|
||||
} else if found {
|
||||
published = publishedOf(b.Made)
|
||||
}
|
||||
}
|
||||
resolved = true
|
||||
}
|
||||
next := cur.Value
|
||||
switch {
|
||||
case !standing:
|
||||
// Carried and assigned to nobody here: a standing key, no time assigned, marked pulled in. An ended one is
|
||||
// written over, the ended one staying in history.
|
||||
next = chain.Assignment{Node: node, Module: module, PulledIn: true, Commit: commit, Build: build, Rollout: rolloutID,
|
||||
Since: at.UTC()}
|
||||
case same:
|
||||
// Already running it: fill what is missing, blank nothing.
|
||||
if next.Build == "" {
|
||||
next.Build = build
|
||||
}
|
||||
if next.Rollout == "" {
|
||||
next.Rollout = rolloutID
|
||||
}
|
||||
if next.Since.IsZero() {
|
||||
next.Since = at.UTC()
|
||||
}
|
||||
default:
|
||||
next.Commit, next.Build, next.Rollout, next.Since = commit, build, rolloutID, at.UTC()
|
||||
next.Artifact, next.Package = "", nil
|
||||
}
|
||||
next.SentUnknown = time.Time{}
|
||||
if published != nil && next.Artifact == "" && next.Package == nil && next.Build == build {
|
||||
next.Artifact, next.Package = published.Artifact, published.Package
|
||||
}
|
||||
if standing && next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout &&
|
||||
next.Artifact == cur.Value.Artifact && (next.Package == nil) == (cur.Value.Package == nil) &&
|
||||
next.Since.Equal(cur.Value.Since) && next.SentUnknown.Equal(cur.Value.SentUnknown) {
|
||||
return nil
|
||||
}
|
||||
if !found {
|
||||
if _, err := c.Assignments.Create(ctx, key, next); errors.Is(err, chain.ErrExists) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if _, err := c.Assignments.Update(ctx, key, next, cur.Revision); errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("the assignment kept moving under the write")
|
||||
}
|
||||
|
||||
// CopySentBuilds copies what delivery 1 recorded in node.sent_builds onto the keys of the assignments, once, at the start of
|
||||
// the controller that holds the lease and has attached the chain (issue 1109, review round 1): delivery 1 wrote no commit onto
|
||||
// the key of a module an assignment pulled in, so without it those modules would be missing from SentBuilds although the
|
||||
// machine is known to have been sent. A key that names a commit is never overwritten, so it is idempotent; once a node's keys
|
||||
// are written its column is emptied, so a later restart cannot lay a stale commit over a key a later send cleared or ended.
|
||||
// It answers how many keys it wrote; an error is raised by the caller as a condition.
|
||||
func (i *Inventory) CopySentBuilds(ctx context.Context) (int, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows, err := i.store.Pool().Query(ctx, `select name, sent_builds from node where sent_builds is not null`)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
type kept struct {
|
||||
name string
|
||||
sent map[string]string
|
||||
}
|
||||
var all []kept
|
||||
for rows.Next() {
|
||||
var name string
|
||||
var raw []byte
|
||||
if err := rows.Scan(&name, &raw); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
sent := map[string]string{}
|
||||
if err := json.Unmarshal(raw, &sent); err != nil {
|
||||
rows.Close()
|
||||
return 0, fmt.Errorf("what %s was sent, in the store, cannot be read: %w", name, err)
|
||||
}
|
||||
all = append(all, kept{name, sent})
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, k := range all {
|
||||
if err := i.copySentBuildsOf(ctx, c, k.name, k.sent, now); err != nil {
|
||||
return 0, fmt.Errorf("what %s was sent cannot be copied onto its assignments: %w", k.name, err)
|
||||
}
|
||||
if _, err := i.store.Pool().Exec(ctx, `update node set sent_builds = null where name = $1`, k.name); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(all), nil
|
||||
}
|
||||
|
||||
// copySentBuildsOf is CopySentBuilds for one node, over a chain: for each module the node was sent, a standing key that
|
||||
// names no commit gets it (and the build, when one is known; no rollout), and where no key exists a standing pulled-in key is
|
||||
// made. A key with a commit, an ended key, a hand send's unknown and a wait for a build are left as they are.
|
||||
func (i *Inventory) copySentBuildsOf(ctx context.Context, c *chain.Chain, node string, sent map[string]string, at time.Time) error {
|
||||
for _, module := range slices.Sorted(maps.Keys(sent)) {
|
||||
commit := sent[module]
|
||||
if commit == "" {
|
||||
continue
|
||||
}
|
||||
key := assignmentKey(node, module)
|
||||
build, err := i.chainBuildOf(ctx, c, module, commit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for try := 0; ; try++ {
|
||||
if try == casTries {
|
||||
return fmt.Errorf("the assignment %s kept moving under the write", key)
|
||||
}
|
||||
cur, found, err := c.Assignments.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found && (cur.Value.State != "" || !cur.Value.Ended.IsZero() || cur.Value.Commit != "" || !cur.Value.SentUnknown.IsZero()) {
|
||||
break // a wait, an ended key (never revived), one that names a commit, or a hand send's unknown
|
||||
}
|
||||
next := cur.Value
|
||||
if !found {
|
||||
next = chain.Assignment{Node: node, Module: module, PulledIn: true, Since: at}
|
||||
}
|
||||
next.Commit, next.Build = commit, build
|
||||
if next.Since.IsZero() {
|
||||
next.Since = at
|
||||
}
|
||||
if !found {
|
||||
_, err = c.Assignments.Create(ctx, key, next)
|
||||
if errors.Is(err, chain.ErrExists) {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
_, err = c.Assignments.Update(ctx, key, next, cur.Revision)
|
||||
if errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -77,9 +77,6 @@ func TestARecordedPendingIsSkippedByAssignedAndTheJoins(t *testing.T) {
|
||||
if running, _ := inv.Running(ctx, "web"); len(running) != 0 {
|
||||
t.Fatalf("running %v", running)
|
||||
}
|
||||
if runs, _ := inv.AssignmentRuns(ctx, node); len(runs) != 0 {
|
||||
t.Fatalf("runs %v", runs)
|
||||
}
|
||||
entries, _ := inv.Catalogued(ctx)
|
||||
for _, e := range entries {
|
||||
if e.Manifest.Module == "web" && len(e.On) != 0 {
|
||||
|
||||
@@ -1,986 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// Rollouts on the bus (novox/hq ADR 0408, ADR 0276, issue 1092): once the kind `rollouts` is moved (chain.go) every plan
|
||||
// and batch is a key of the rollouts bucket under its id, written by compare-and-set, and the merges a batch or walk
|
||||
// answers are held in its record. The tables are read when the kind is not moved, by the methods these stand beside.
|
||||
//
|
||||
// **Two counters.** The bucket's revision is the compare-and-set of every write. The plan's own Revision, kept in its value,
|
||||
// counts the saves of the plan: a save refuses (ErrPlanMoved) a plan another save moved since it was read, and takes no
|
||||
// notice of a merge moving in or out of the record, which a copy read before could not know of. Held is written by the
|
||||
// merge operations alone; a save keeps what the record holds.
|
||||
|
||||
// rolloutView is the rollouts bucket in memory, kept current by one watch (issue 1092): the survey of the plans, the
|
||||
// batches and the merges is read from here, never from the bucket one key at a time. A write is still made by
|
||||
// compare-and-set on the bucket, so a stale view costs a refused write and a read again, never a wrong one.
|
||||
type rolloutView struct {
|
||||
chain *chain.Chain
|
||||
mu sync.RWMutex
|
||||
entries map[string]viewEntry
|
||||
ready chan struct{}
|
||||
once sync.Once
|
||||
err error
|
||||
dead atomic.Bool
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
// viewEntry is a record as the bucket holds it, with what the surveys ask of it read out.
|
||||
type viewEntry struct {
|
||||
raw []byte
|
||||
rev uint64
|
||||
sum planSummary
|
||||
}
|
||||
|
||||
// planSummary is the part of a plan the surveys filter and sort by.
|
||||
type planSummary struct {
|
||||
ID string `json:"id"`
|
||||
State string `json:"state"`
|
||||
Created time.Time `json:"created"`
|
||||
Repository string `json:"repository"`
|
||||
Branch string `json:"branch"`
|
||||
Merged time.Time `json:"merged"`
|
||||
Release json.RawMessage `json:"release"`
|
||||
Held []chain.HeldMerge `json:"held"`
|
||||
}
|
||||
|
||||
func (s planSummary) open() bool { return s.State == PlanBuilding || s.State == PlanRolling }
|
||||
|
||||
func (s planSummary) isRelease() bool {
|
||||
r := strings.TrimSpace(string(s.Release))
|
||||
return r != "" && r != "null"
|
||||
}
|
||||
|
||||
func newRolloutView(c *chain.Chain) *rolloutView {
|
||||
return &rolloutView{chain: c, entries: map[string]viewEntry{}, ready: make(chan struct{})}
|
||||
}
|
||||
|
||||
func (v *rolloutView) start() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
v.cancel = cancel
|
||||
go func() {
|
||||
w, err := chain.RolloutsOf[Plan](v.chain).Watch(ctx)
|
||||
if err != nil {
|
||||
v.fail(fmt.Errorf("the rollouts cannot be watched: %w", err))
|
||||
return
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
v.fail(ctx.Err())
|
||||
return
|
||||
case e, open := <-w:
|
||||
if !open {
|
||||
v.fail(errors.New("the rollouts bucket stopped answering"))
|
||||
return
|
||||
}
|
||||
if e.Caught {
|
||||
v.once.Do(func() { close(v.ready) })
|
||||
continue
|
||||
}
|
||||
v.apply(e.Key, e.Value, e.Revision)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// fail ends the view: it answers an error to whoever waits for it to fill, and is made again by the next read.
|
||||
func (v *rolloutView) fail(err error) {
|
||||
v.mu.Lock()
|
||||
if v.err == nil {
|
||||
v.err = err
|
||||
}
|
||||
v.mu.Unlock()
|
||||
v.dead.Store(true)
|
||||
v.once.Do(func() { close(v.ready) })
|
||||
}
|
||||
|
||||
func (v *rolloutView) stop() {
|
||||
if v.cancel != nil {
|
||||
v.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
// apply keeps a record the watch handed over or a write of this process made, unless the view holds a newer one.
|
||||
func (v *rolloutView) apply(key string, raw []byte, rev uint64) {
|
||||
var sum planSummary
|
||||
if err := json.Unmarshal(raw, &sum); err != nil || sum.ID == "" {
|
||||
sum = planSummary{ID: key}
|
||||
}
|
||||
v.mu.Lock()
|
||||
defer v.mu.Unlock()
|
||||
if old, has := v.entries[key]; has && old.rev >= rev {
|
||||
return
|
||||
}
|
||||
v.entries[key] = viewEntry{raw: append([]byte(nil), raw...), rev: rev, sum: sum}
|
||||
}
|
||||
|
||||
func (v *rolloutView) list() []viewEntry {
|
||||
v.mu.RLock()
|
||||
defer v.mu.RUnlock()
|
||||
out := make([]viewEntry, 0, len(v.entries))
|
||||
for _, e := range v.entries {
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole.
|
||||
func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) {
|
||||
i.state.mu.Lock()
|
||||
v := i.state.view
|
||||
if v == nil || v.chain != c || v.dead.Load() {
|
||||
if v != nil {
|
||||
v.stop()
|
||||
}
|
||||
v = newRolloutView(c)
|
||||
i.state.view = v
|
||||
v.start()
|
||||
}
|
||||
i.state.mu.Unlock()
|
||||
select {
|
||||
case <-v.ready:
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
v.mu.RLock()
|
||||
defer v.mu.RUnlock()
|
||||
if v.err != nil && len(v.entries) == 0 {
|
||||
return nil, fmt.Errorf("the rollouts cannot be read from the bus: %w", v.err)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// stopViews ends the view's watch: the inventory is closing.
|
||||
func (i *Inventory) stopViews() {
|
||||
i.state.mu.Lock()
|
||||
defer i.state.mu.Unlock()
|
||||
if i.state.view != nil {
|
||||
i.state.view.stop()
|
||||
i.state.view = nil
|
||||
}
|
||||
}
|
||||
|
||||
func normalizePlan(p *Plan) {
|
||||
if p.Modules == nil {
|
||||
p.Modules = map[string]*PlanModule{}
|
||||
}
|
||||
if p.TierEntered.IsZero() {
|
||||
p.TierEntered = p.Created
|
||||
}
|
||||
if !p.Merged.IsZero() {
|
||||
p.Merged = p.Merged.UTC()
|
||||
}
|
||||
}
|
||||
|
||||
func planOf(e viewEntry) (Plan, error) {
|
||||
var p Plan
|
||||
if err := json.Unmarshal(e.raw, &p); err != nil {
|
||||
return Plan{}, fmt.Errorf("the plan %s on the bus cannot be read: %w", e.sum.ID, err)
|
||||
}
|
||||
normalizePlan(&p)
|
||||
return p, nil
|
||||
}
|
||||
|
||||
func sortCreated(entries []viewEntry) {
|
||||
sort.SliceStable(entries, func(a, b int) bool {
|
||||
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
|
||||
return entries[a].sum.Created.Before(entries[b].sum.Created)
|
||||
}
|
||||
return entries[a].sum.ID < entries[b].sum.ID
|
||||
})
|
||||
}
|
||||
|
||||
func sortCreatedNewest(entries []viewEntry) {
|
||||
sort.SliceStable(entries, func(a, b int) bool {
|
||||
if !entries[a].sum.Created.Equal(entries[b].sum.Created) {
|
||||
return entries[a].sum.Created.After(entries[b].sum.Created)
|
||||
}
|
||||
return entries[a].sum.ID > entries[b].sum.ID
|
||||
})
|
||||
}
|
||||
|
||||
// chainPlans is the plans that pass keep, in an order, read from the view.
|
||||
func (i *Inventory) chainPlans(ctx context.Context, c *chain.Chain, keep func(viewEntry) bool, order func([]viewEntry)) ([]Plan, error) {
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var chosen []viewEntry
|
||||
for _, e := range v.list() {
|
||||
if e.sum.ID != WaitingRecord && keep(e) {
|
||||
chosen = append(chosen, e)
|
||||
}
|
||||
}
|
||||
order(chosen)
|
||||
out := make([]Plan, 0, len(chosen))
|
||||
for _, e := range chosen {
|
||||
p, err := planOf(e)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainNewestMergeOf(ctx context.Context, c *chain.Chain, repository, branch string) (Plan, bool, error) {
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return Plan{}, false, err
|
||||
}
|
||||
var chosen []viewEntry
|
||||
for _, e := range v.list() {
|
||||
if strings.EqualFold(e.sum.Repository, repository) && e.sum.Branch == branch && !e.sum.Merged.IsZero() && !e.sum.isRelease() {
|
||||
chosen = append(chosen, e)
|
||||
}
|
||||
}
|
||||
sort.SliceStable(chosen, func(a, b int) bool {
|
||||
if !chosen[a].sum.Merged.Equal(chosen[b].sum.Merged) {
|
||||
return chosen[a].sum.Merged.After(chosen[b].sum.Merged)
|
||||
}
|
||||
return chosen[a].sum.Created.After(chosen[b].sum.Created)
|
||||
})
|
||||
if len(chosen) == 0 {
|
||||
return Plan{}, false, nil
|
||||
}
|
||||
p, err := planOf(chosen[0])
|
||||
return p, err == nil, err
|
||||
}
|
||||
|
||||
// chainPlanByID is one plan, read from the bucket itself: the caller of the verb wants the plan as it is, and the view
|
||||
// learns of it.
|
||||
func (i *Inventory) chainPlanByID(ctx context.Context, c *chain.Chain, id string) (Plan, error) {
|
||||
if !chain.ValidKey(id) {
|
||||
// No plan has an id the bus takes no spelling of; it is a plain id no plan has.
|
||||
return Plan{}, fmt.Errorf("no plan %s", id)
|
||||
}
|
||||
e, found, err := chain.RolloutsOf[Plan](c).GetRaw(ctx, id)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
if !found {
|
||||
return Plan{}, fmt.Errorf("no plan %s", id)
|
||||
}
|
||||
if v, err := i.rolloutView(ctx, c); err == nil {
|
||||
v.apply(id, e.Value, e.Revision)
|
||||
}
|
||||
return planOf(viewEntry{raw: e.Value, rev: e.Revision, sum: planSummary{ID: id}})
|
||||
}
|
||||
|
||||
// ---- the merges
|
||||
|
||||
// mergeHolder is a record's copy of a merge.
|
||||
type mergeHolder struct {
|
||||
plan string
|
||||
m chain.HeldMerge
|
||||
rev uint64
|
||||
}
|
||||
|
||||
func mergeKey(repository, commit string) string { return strings.ToLower(repository) + "|" + commit }
|
||||
|
||||
// mergeHolders is every record's copy of each merge, by the merge's key.
|
||||
func mergeHolders(entries []viewEntry) map[string][]mergeHolder {
|
||||
out := map[string][]mergeHolder{}
|
||||
for _, e := range entries {
|
||||
for _, m := range e.sum.Held {
|
||||
k := mergeKey(m.Repository, m.Commit)
|
||||
out[k] = append(out[k], mergeHolder{plan: e.sum.ID, m: m, rev: e.rev})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// winner is the copy of a merge that counts: the one that entered its record last (ADR 0408, issue 1092 — a merge in two
|
||||
// records is the half-done move of one, and the record it entered last is the one it moved to), a merge a plan answers
|
||||
// before one no plan does, then the record written last.
|
||||
func winner(copies []mergeHolder) mergeHolder {
|
||||
best := copies[0]
|
||||
for _, c := range copies[1:] {
|
||||
switch {
|
||||
case c.m.Placed.After(best.m.Placed):
|
||||
best = c
|
||||
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting && !c.m.Waiting:
|
||||
best = c
|
||||
case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting == c.m.Waiting && c.rev > best.rev:
|
||||
best = c
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func winners(entries []viewEntry) map[string]mergeHolder {
|
||||
out := map[string]mergeHolder{}
|
||||
for k, copies := range mergeHolders(entries) {
|
||||
out[k] = winner(copies)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func batchedOf(h mergeHolder) BatchedMerge {
|
||||
b := BatchedMerge{Repository: strings.ToLower(h.m.Repository), Branch: h.m.Branch, Commit: h.m.Commit,
|
||||
Heard: h.m.Heard.UTC(), Event: h.m.Event, Plan: h.plan, Alone: h.m.Alone}
|
||||
if !h.m.Merged.IsZero() {
|
||||
b.Merged = h.m.Merged.UTC()
|
||||
}
|
||||
if h.m.Waiting {
|
||||
b.Plan = ""
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func heldOf(m BatchedMerge, now time.Time) chain.HeldMerge {
|
||||
h := chain.HeldMerge{Repository: strings.ToLower(m.Repository), Branch: m.Branch, Commit: m.Commit, Heard: m.Heard.UTC(),
|
||||
Event: m.Event, Alone: m.Alone, Placed: now.UTC()}
|
||||
if !m.Merged.IsZero() {
|
||||
h.Merged = m.Merged.UTC()
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func (i *Inventory) chainWinners(ctx context.Context, c *chain.Chain) (map[string]mergeHolder, error) {
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return winners(v.list()), nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainMergeOf(ctx context.Context, c *chain.Chain, repository, commit string) (BatchedMerge, bool, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return BatchedMerge{}, false, err
|
||||
}
|
||||
h, ok := w[mergeKey(repository, commit)]
|
||||
if !ok {
|
||||
return BatchedMerge{}, false, nil
|
||||
}
|
||||
return batchedOf(h), true, nil
|
||||
}
|
||||
|
||||
// merged orders merges by the time they were made, those whose time is not known last, then by when they were heard.
|
||||
func sortMerges(ms []BatchedMerge, newest bool) {
|
||||
sort.SliceStable(ms, func(a, b int) bool {
|
||||
x, y := ms[a], ms[b]
|
||||
switch {
|
||||
case x.Merged.IsZero() != y.Merged.IsZero():
|
||||
return !x.Merged.IsZero()
|
||||
case !x.Merged.Equal(y.Merged):
|
||||
if newest {
|
||||
return x.Merged.After(y.Merged)
|
||||
}
|
||||
return x.Merged.Before(y.Merged)
|
||||
}
|
||||
if newest {
|
||||
return x.Heard.After(y.Heard)
|
||||
}
|
||||
return x.Heard.Before(y.Heard)
|
||||
})
|
||||
}
|
||||
|
||||
func (i *Inventory) chainMergesOf(ctx context.Context, c *chain.Chain, plan string) ([]BatchedMerge, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []BatchedMerge
|
||||
for _, h := range w {
|
||||
if h.plan == plan && !h.m.Waiting {
|
||||
out = append(out, batchedOf(h))
|
||||
}
|
||||
}
|
||||
sortMerges(out, false)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainLaterMergesOf(ctx context.Context, c *chain.Chain, repository, branch string, after time.Time) ([]BatchedMerge, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []BatchedMerge
|
||||
for _, h := range w {
|
||||
if !h.m.Waiting && strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && h.m.Merged.After(after) {
|
||||
out = append(out, batchedOf(h))
|
||||
}
|
||||
}
|
||||
sort.SliceStable(out, func(a, b int) bool { return out[a].Merged.After(out[b].Merged) })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainKeptOnBranch(ctx context.Context, c *chain.Chain, repository, branch string) (bool, time.Time, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return false, time.Time{}, err
|
||||
}
|
||||
var first time.Time
|
||||
for _, h := range w {
|
||||
if strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && (first.IsZero() || h.m.Heard.Before(first)) {
|
||||
first = h.m.Heard
|
||||
}
|
||||
}
|
||||
if first.IsZero() {
|
||||
return false, time.Time{}, nil
|
||||
}
|
||||
return true, first.UTC(), nil
|
||||
}
|
||||
|
||||
// chainEarliestKept is when the controller heard the first merge it holds a record of, on any repository and branch.
|
||||
func (i *Inventory) chainEarliestKept(ctx context.Context, c *chain.Chain) (bool, time.Time, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return false, time.Time{}, err
|
||||
}
|
||||
var first time.Time
|
||||
for _, h := range w {
|
||||
if first.IsZero() || h.m.Heard.Before(first) {
|
||||
first = h.m.Heard
|
||||
}
|
||||
}
|
||||
if first.IsZero() {
|
||||
return false, time.Time{}, nil
|
||||
}
|
||||
return true, first.UTC(), nil
|
||||
}
|
||||
|
||||
func (i *Inventory) chainAloneMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []BatchedMerge
|
||||
for _, h := range w {
|
||||
if h.m.Waiting && h.m.Alone {
|
||||
out = append(out, batchedOf(h))
|
||||
}
|
||||
}
|
||||
sortMerges(out, true)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// chainWaitingMerges is every merge no plan answers that waits and is not walked alone, the newest first.
|
||||
func (i *Inventory) chainWaitingMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) {
|
||||
w, err := i.chainWinners(ctx, c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []BatchedMerge
|
||||
for _, h := range w {
|
||||
if h.m.Waiting && !h.m.Alone {
|
||||
out = append(out, batchedOf(h))
|
||||
}
|
||||
}
|
||||
sortMerges(out, true)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// pendingMerge is a merge added to a plan that is not saved yet: carried until that plan's save writes it.
|
||||
type pendingMerge struct {
|
||||
plan string
|
||||
m chain.HeldMerge
|
||||
}
|
||||
|
||||
// chainAddMerge keeps a merge for a plan. **When the plan has a record the merge is written into it at once**; when it has
|
||||
// none — a new batch, saved by the caller next — the merge is carried and written by that plan's save. The forge's event is
|
||||
// acknowledged only after the caller's write succeeded, and handed over again if it did not (ADR 0407 §3).
|
||||
func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m BatchedMerge) (bool, error) {
|
||||
waiting := m.Plan == ""
|
||||
if waiting {
|
||||
// A merge no plan answers is held, flagged, by the record kept for those.
|
||||
m.Plan = WaitingRecord
|
||||
if err := i.ensureWaitingRecord(ctx, c); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
key := mergeKey(m.Repository, m.Commit)
|
||||
if _, known := winners(v.list())[key]; known {
|
||||
return false, nil
|
||||
}
|
||||
held := heldOf(m, time.Now())
|
||||
held.Waiting = waiting
|
||||
found, err := i.editRecord(ctx, c, m.Plan, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
||||
for _, x := range h {
|
||||
if mergeKey(x.Repository, x.Commit) == key {
|
||||
return h, false
|
||||
}
|
||||
}
|
||||
return append(h, held), true
|
||||
})
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !found {
|
||||
i.state.mu.Lock()
|
||||
if i.state.pending == nil {
|
||||
i.state.pending = map[string]pendingMerge{}
|
||||
}
|
||||
i.state.pending[key] = pendingMerge{plan: m.Plan, m: held}
|
||||
i.state.mu.Unlock()
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// WaitingRecord is the record that holds the merges no plan has ever answered: a merge kept with no plan, which a store
|
||||
// kept as a row with a null plan. It is no plan: the surveys of plans leave it out.
|
||||
const WaitingRecord = "waiting-merges"
|
||||
|
||||
// ensureWaitingRecord makes the record of the merges no plan answers, once.
|
||||
func (i *Inventory) ensureWaitingRecord(ctx context.Context, c *chain.Chain) error {
|
||||
b := chain.RolloutsOf[Plan](c)
|
||||
if _, found, err := b.GetRaw(ctx, WaitingRecord); err != nil || found {
|
||||
return err
|
||||
}
|
||||
body, err := json.Marshal(Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1,
|
||||
Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rev, err := b.CreateRaw(ctx, WaitingRecord, body)
|
||||
if errors.Is(err, chain.ErrExists) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if v, verr := i.rolloutView(ctx, c); verr == nil {
|
||||
v.apply(WaitingRecord, body, rev)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrMergeUnsaved is a merge moved to a plan that has no record to hold it.
|
||||
var ErrMergeUnsaved = errors.New("a merge was moved to a plan that has not been saved")
|
||||
|
||||
// editRecord changes the merges a plan's record holds and nothing else of it, by compare-and-set on the bucket, read again
|
||||
// when it moved. It answers false when the plan has no record.
|
||||
func (i *Inventory) editRecord(ctx context.Context, c *chain.Chain, id string, edit func([]chain.HeldMerge) ([]chain.HeldMerge, bool)) (bool, error) {
|
||||
b := chain.RolloutsOf[Plan](c)
|
||||
for try := 0; try < casTries; try++ {
|
||||
e, found, err := b.GetRaw(ctx, id)
|
||||
if err != nil || !found {
|
||||
return false, err
|
||||
}
|
||||
var record map[string]json.RawMessage
|
||||
if err := json.Unmarshal(e.Value, &record); err != nil {
|
||||
return true, fmt.Errorf("the plan %s on the bus cannot be read: %w", id, err)
|
||||
}
|
||||
var held []chain.HeldMerge
|
||||
if raw, has := record["held"]; has {
|
||||
if err := json.Unmarshal(raw, &held); err != nil {
|
||||
return true, fmt.Errorf("the merges of plan %s on the bus cannot be read: %w", id, err)
|
||||
}
|
||||
}
|
||||
next, changed := edit(held)
|
||||
if !changed {
|
||||
return true, nil
|
||||
}
|
||||
if len(next) == 0 {
|
||||
delete(record, "held")
|
||||
} else {
|
||||
raw, err := json.Marshal(next)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
record["held"] = raw
|
||||
}
|
||||
body, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
return true, err
|
||||
}
|
||||
rev, err := b.UpdateRaw(ctx, id, body, e.Revision)
|
||||
if errors.Is(err, chain.ErrMoved) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return true, fmt.Errorf("the merges of plan %s are not written: %w", id, err)
|
||||
}
|
||||
if v, verr := i.rolloutView(ctx, c); verr == nil {
|
||||
v.apply(id, body, rev)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return true, fmt.Errorf("the merges of plan %s kept moving under the write", id)
|
||||
}
|
||||
|
||||
// recordEdit is what the merges of one record gain, lose and have flagged in one act.
|
||||
type recordEdit struct {
|
||||
add []chain.HeldMerge
|
||||
drop map[string]bool
|
||||
flag map[string]chain.HeldMerge
|
||||
}
|
||||
|
||||
func (e *recordEdit) apply(held []chain.HeldMerge) []chain.HeldMerge {
|
||||
out := make([]chain.HeldMerge, 0, len(held)+len(e.add))
|
||||
for _, h := range held {
|
||||
k := mergeKey(h.Repository, h.Commit)
|
||||
if e.drop[k] {
|
||||
continue
|
||||
}
|
||||
if f, has := e.flag[k]; has {
|
||||
h.Waiting, h.Alone = f.Waiting, f.Alone
|
||||
// A merge put back to waiting may carry a new announcement (HoldMerge); every other flag copies its own.
|
||||
if len(f.Event) > 0 {
|
||||
h.Event = f.Event
|
||||
}
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
for _, a := range e.add {
|
||||
k := mergeKey(a.Repository, a.Commit)
|
||||
dup := false
|
||||
for _, h := range out {
|
||||
if mergeKey(h.Repository, h.Commit) == k {
|
||||
dup = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !dup {
|
||||
out = append(out, a)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (e *recordEdit) empty() bool { return len(e.add) == 0 && len(e.drop) == 0 && len(e.flag) == 0 }
|
||||
|
||||
// ---- the save
|
||||
|
||||
// tierLeftOf is the measurement a plan's save makes when it leaves a tier (planTierLeft, over the record the save
|
||||
// replaces): when the plan entered the tier it is now at, and the duration of the tier it left, if it left one.
|
||||
func tierLeftOf(old *Plan, p Plan, now time.Time) (time.Time, *Duration) {
|
||||
if old == nil {
|
||||
return now, nil // a new plan enters its first tier now
|
||||
}
|
||||
if old.State == PlanAssembling || old.State == PlanQueued {
|
||||
return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276)
|
||||
}
|
||||
since := old.TierEntered
|
||||
if since.IsZero() {
|
||||
since = old.Created
|
||||
}
|
||||
wasOpen := old.State == PlanBuilding || old.State == PlanRolling
|
||||
if !wasOpen && p.Open() {
|
||||
// **A walk set open again enters its tier anew** (issue 1886): a resume or a retry at the tier it ended at. Kept,
|
||||
// the entry of before its stop measured the tier from hours before the resume — R4e read "waiting … for 5h48m53s"
|
||||
// seven minutes after it was resumed — and the watchdog of its progress (S3) found it late at once.
|
||||
return now, nil
|
||||
}
|
||||
if !wasOpen || (old.Tier == p.Tier && p.Open()) {
|
||||
return since, nil // still in the tier, or already ended
|
||||
}
|
||||
var modules []string
|
||||
if old.Tier >= 0 && old.Tier < len(p.Tiers) {
|
||||
modules = p.Tiers[old.Tier]
|
||||
}
|
||||
detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, old.Tier, len(p.Tiers), modules, p.State)
|
||||
if p.Open() {
|
||||
detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, old.Tier, len(p.Tiers), modules, p.Tier)
|
||||
}
|
||||
return now, &Duration{Kind: DurationPlanTier, Subject: p.Repository, Ref: fmt.Sprintf("%s/tier-%d", p.ID, old.Tier),
|
||||
Started: since, Took: now.Sub(since), Detail: detail}
|
||||
}
|
||||
|
||||
// chainSavePlans writes plans, as SavePlan does each, and gives merges to the plans that answer them. **Each plan by
|
||||
// compare-and-set on its own Revision**, a copy read before another save refused with ErrPlanMoved before anything is
|
||||
// written; **a merge moved to a plan is written into the receiving record first, then taken from the giving one**, so a
|
||||
// crash between leaves it in both and never in neither, and the one it entered last counts (ADR 0408, issue 1092).
|
||||
func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []*Plan, moves []MergeMove) (err error) {
|
||||
epoch, err := i.actingEpoch(ctx)
|
||||
if err != nil {
|
||||
if len(plans) > 0 {
|
||||
return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b := chain.RolloutsOf[Plan](c)
|
||||
marked := time.Now()
|
||||
undo := make([]func(), 0, len(plans))
|
||||
defer func() {
|
||||
if err != nil {
|
||||
for _, u := range undo {
|
||||
u()
|
||||
}
|
||||
}
|
||||
}()
|
||||
for _, p := range plans {
|
||||
undo = append(undo, p.markWalk(marked))
|
||||
}
|
||||
|
||||
// Every plan still at the revision it was read at, before anything is written.
|
||||
cur := make(map[string]chain.Kept[Plan], len(plans))
|
||||
for _, p := range plans {
|
||||
k, found, err := b.Get(ctx, p.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
if k.Value.Revision != p.Revision {
|
||||
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
||||
}
|
||||
cur[p.ID] = k
|
||||
}
|
||||
}
|
||||
|
||||
// What each record gains, loses and has flagged.
|
||||
now := time.Now().UTC()
|
||||
edits := map[string]*recordEdit{}
|
||||
editOf := func(id string) *recordEdit {
|
||||
if edits[id] == nil {
|
||||
edits[id] = &recordEdit{drop: map[string]bool{}, flag: map[string]chain.HeldMerge{}}
|
||||
}
|
||||
return edits[id]
|
||||
}
|
||||
copies := mergeHolders(v.list())
|
||||
for _, m := range moves {
|
||||
key := mergeKey(m.Repository, m.Commit)
|
||||
held, has := copies[key]
|
||||
if !has {
|
||||
// A merge carried for a plan not saved yet goes where it is moved.
|
||||
i.state.mu.Lock()
|
||||
if pm, pending := i.state.pending[key]; pending && m.Plan != "" {
|
||||
pm.plan, pm.m.Alone = m.Plan, m.Alone
|
||||
i.state.pending[key] = pm
|
||||
}
|
||||
i.state.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
w := winner(held)
|
||||
if m.Plan == "" {
|
||||
f := w.m
|
||||
f.Waiting, f.Alone = true, m.Alone
|
||||
if m.Event != nil {
|
||||
f.Event = m.Event
|
||||
}
|
||||
editOf(w.plan).flag[key] = f
|
||||
continue
|
||||
}
|
||||
if w.plan == m.Plan {
|
||||
f := w.m
|
||||
f.Waiting, f.Alone = false, m.Alone
|
||||
editOf(w.plan).flag[key] = f
|
||||
continue
|
||||
}
|
||||
moved := w.m
|
||||
moved.Waiting, moved.Alone, moved.Placed = false, m.Alone, now
|
||||
editOf(m.Plan).add = append(editOf(m.Plan).add, moved)
|
||||
for _, h := range held {
|
||||
editOf(h.plan).drop[key] = true
|
||||
}
|
||||
}
|
||||
|
||||
// The records not among the plans: receivers now, givers after.
|
||||
for id, e := range edits {
|
||||
if _, written := cur[id]; written || planNamed(plans, id) || len(e.add) == 0 {
|
||||
continue
|
||||
}
|
||||
e := e
|
||||
found, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
||||
return (&recordEdit{add: e.add}).apply(h), true
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !found {
|
||||
// Nothing is taken from the record that holds a merge to give it to a plan that has none.
|
||||
return fmt.Errorf("%w: %s has no record to hold %d merge(s) moved to it", ErrMergeUnsaved, id, len(e.add))
|
||||
}
|
||||
}
|
||||
|
||||
// The plans, those that gain a merge first.
|
||||
order := append([]*Plan(nil), plans...)
|
||||
sort.SliceStable(order, func(a, b int) bool {
|
||||
x, y := edits[order[a].ID], edits[order[b].ID]
|
||||
return x != nil && len(x.add) > 0 && (y == nil || len(y.add) == 0)
|
||||
})
|
||||
type written struct {
|
||||
revision int64
|
||||
entered time.Time
|
||||
held []chain.HeldMerge
|
||||
duration *Duration
|
||||
}
|
||||
done := map[string]written{}
|
||||
for _, p := range order {
|
||||
var w written
|
||||
for try := 0; ; try++ {
|
||||
if try == casTries {
|
||||
return fmt.Errorf("the plan for %s %s (%s) kept moving under the write", p.Repository, short(p.Commit), p.ID)
|
||||
}
|
||||
k, found := cur[p.ID]
|
||||
if try > 0 {
|
||||
fresh, still, gerr := b.Get(ctx, p.ID)
|
||||
if gerr != nil {
|
||||
return gerr
|
||||
}
|
||||
if still && fresh.Value.Revision != p.Revision {
|
||||
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
||||
}
|
||||
k, found = fresh, still
|
||||
cur[p.ID] = k
|
||||
}
|
||||
record := *p
|
||||
var old *Plan
|
||||
var held []chain.HeldMerge
|
||||
if found {
|
||||
o := k.Value
|
||||
old, held = &o, append([]chain.HeldMerge(nil), k.Value.Held...)
|
||||
record.Revision = k.Value.Revision + 1
|
||||
} else {
|
||||
record.Revision = 1
|
||||
}
|
||||
if e := edits[p.ID]; e != nil {
|
||||
held = e.apply(held)
|
||||
}
|
||||
i.state.mu.Lock()
|
||||
for key, pm := range i.state.pending {
|
||||
if pm.plan == p.ID {
|
||||
held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held)
|
||||
_ = key
|
||||
}
|
||||
}
|
||||
i.state.mu.Unlock()
|
||||
record.Held = held
|
||||
entered, duration := tierLeftOf(old, *p, time.Now())
|
||||
record.TierEntered, record.Updated = entered, time.Now().UTC()
|
||||
if epoch != nil {
|
||||
record.Epoch = uint64(*epoch)
|
||||
} else {
|
||||
record.Epoch = 0
|
||||
}
|
||||
body, merr := json.Marshal(record)
|
||||
if merr != nil {
|
||||
return merr
|
||||
}
|
||||
var rev uint64
|
||||
var werr error
|
||||
if found {
|
||||
rev, werr = b.UpdateRaw(ctx, p.ID, body, k.Revision)
|
||||
} else {
|
||||
rev, werr = createRaw(ctx, c, p.ID, body)
|
||||
}
|
||||
if errors.Is(werr, chain.ErrMoved) || (!found && errors.Is(werr, chain.ErrExists)) {
|
||||
if !found {
|
||||
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if werr != nil {
|
||||
return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, werr)
|
||||
}
|
||||
v.apply(p.ID, body, rev)
|
||||
w = written{revision: record.Revision, entered: entered, held: held, duration: duration}
|
||||
break
|
||||
}
|
||||
done[p.ID] = w
|
||||
}
|
||||
|
||||
// The records that gave a merge up, now it is held by the one it went to.
|
||||
for id, e := range edits {
|
||||
if planNamed(plans, id) || (len(e.drop) == 0 && len(e.flag) == 0) {
|
||||
continue
|
||||
}
|
||||
e := e
|
||||
if _, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
||||
return (&recordEdit{drop: e.drop, flag: e.flag}).apply(h), true
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
i.state.mu.Lock()
|
||||
for key, pm := range i.state.pending {
|
||||
if _, saved := done[pm.plan]; saved {
|
||||
delete(i.state.pending, key)
|
||||
}
|
||||
}
|
||||
i.state.mu.Unlock()
|
||||
for _, p := range plans {
|
||||
w := done[p.ID]
|
||||
p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held
|
||||
if epoch != nil {
|
||||
p.Epoch = uint64(*epoch)
|
||||
} else {
|
||||
p.Epoch = 0
|
||||
}
|
||||
if w.duration != nil {
|
||||
// After the write: the key of the duration is the plan and its tier, so a measurement made twice is one.
|
||||
record := i.RecordDuration
|
||||
if i.recorded != nil {
|
||||
record = i.recorded
|
||||
}
|
||||
if derr := record(ctx, *w.duration); derr != nil {
|
||||
fmt.Printf("the time plan %s spent in a tier is not recorded: %v\n", p.ID, derr)
|
||||
}
|
||||
}
|
||||
if PlanSaved != nil {
|
||||
PlanSaved(i, *p)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func planNamed(plans []*Plan, id string) bool {
|
||||
for _, p := range plans {
|
||||
if p.ID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// createRaw writes a plan's first record.
|
||||
func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uint64, error) {
|
||||
return chain.RolloutsOf[json.RawMessage](c).CreateRaw(ctx, id, body)
|
||||
}
|
||||
|
||||
// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired
|
||||
// at the next cut (ADR 0408, issue 1092). It says how many it repaired.
|
||||
func (i *Inventory) RepairMerges(ctx context.Context) (int, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
v, err := i.rolloutView(ctx, c)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
repaired := 0
|
||||
for key, copies := range mergeHolders(v.list()) {
|
||||
if len(copies) < 2 {
|
||||
continue
|
||||
}
|
||||
w := winner(copies)
|
||||
for _, h := range copies {
|
||||
if h.plan == w.plan {
|
||||
continue
|
||||
}
|
||||
key := key
|
||||
if _, err := i.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
||||
return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true
|
||||
}); err != nil {
|
||||
return repaired, err
|
||||
}
|
||||
repaired++
|
||||
}
|
||||
}
|
||||
return repaired, nil
|
||||
}
|
||||
@@ -1,374 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// anInventoryOnTheChain is an inventory with no store and a chain in memory: the chain's own paths, which never reach the
|
||||
// store but for the durations a save measures, taken here by nothing.
|
||||
func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) {
|
||||
t.Helper()
|
||||
c := chain.InMemory(nil)
|
||||
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv.UseChain(c)
|
||||
t.Cleanup(inv.stopViews)
|
||||
return inv, c
|
||||
}
|
||||
|
||||
var plansAt = time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
|
||||
func aBatch(id string, at time.Time) *Plan {
|
||||
return &Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c-" + id, Created: at, State: PlanAssembling,
|
||||
Tiers: [][]string{}, Modules: map[string]*PlanModule{}}
|
||||
}
|
||||
|
||||
func aMerge(commit, plan string) BatchedMerge {
|
||||
return BatchedMerge{Repository: "Novox/Mesh-Catalog", Branch: "main", Commit: commit, Merged: plansAt, Heard: plansAt.Add(time.Second),
|
||||
Event: []byte(`{"commit":"` + commit + `"}`), Plan: plan}
|
||||
}
|
||||
|
||||
// **A plan is kept under its id; a save of a copy another save has moved since is refused and writes nothing** (ADR 0408
|
||||
// §2, issue 1092): the plan's own Revision counts its saves, 1 for the first.
|
||||
func TestAPlanIsKeptUnderItsIdAndASaveOfAStaleCopyIsRefused(t *testing.T) {
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
ctx := context.Background()
|
||||
p := aBatch("plan-1", plansAt)
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Revision != 1 {
|
||||
t.Fatalf("a new plan is at revision %d", p.Revision)
|
||||
}
|
||||
stale := *p
|
||||
p.Note = "moved on"
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil || p.Revision != 2 {
|
||||
t.Fatalf("the second save: revision %d, %v", p.Revision, err)
|
||||
}
|
||||
stale.Note = "laid over"
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{&stale}, nil); !errors.Is(err, ErrPlanMoved) {
|
||||
t.Fatalf("a stale copy was saved: %v", err)
|
||||
}
|
||||
got, err := inv.chainPlanByID(ctx, c, "plan-1")
|
||||
if err != nil || got.Note != "moved on" || got.Revision != 2 {
|
||||
t.Fatalf("the plan is %+v: %v", got, err)
|
||||
}
|
||||
// A plan new to the bucket that is already there is refused too.
|
||||
again := aBatch("plan-1", plansAt)
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{again}, nil); !errors.Is(err, ErrPlanMoved) {
|
||||
t.Fatalf("a new plan over a kept one: %v", err)
|
||||
}
|
||||
if _, err := inv.chainPlanByID(ctx, c, "plan-nobody"); err == nil || err.Error() != "no plan plan-nobody" {
|
||||
t.Fatalf("a plan nobody kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A save keeps the merges the record holds, not the ones its copy read** (issue 1092): a merge that moved into the record
|
||||
// after the copy was read is not laid back by the copy's save.
|
||||
func TestASaveKeepsTheMergesTheRecordHoldsAndNotTheCopys(t *testing.T) {
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
ctx := context.Background()
|
||||
batch := aBatch("plan-1", plansAt)
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || !added {
|
||||
t.Fatalf("added %v: %v", added, err)
|
||||
}
|
||||
// The copy was read before the merge came in; saving it neither fails nor loses the merge.
|
||||
batch.Note = "window closed"
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil {
|
||||
t.Fatalf("a copy read before a merge came in: %v", err)
|
||||
}
|
||||
merges, err := inv.chainMergesOf(ctx, c, "plan-1")
|
||||
if err != nil || len(merges) != 1 || merges[0].Commit != "m1" || merges[0].Repository != "novox/mesh-catalog" {
|
||||
t.Fatalf("the merges of the plan: %+v %v", merges, err)
|
||||
}
|
||||
if string(merges[0].Event) != `{"commit":"m1"}` || merges[0].Heard.IsZero() || merges[0].Plan != "plan-1" {
|
||||
t.Fatalf("the merge came back as %+v", merges[0])
|
||||
}
|
||||
// A merge heard is not heard twice, whatever the spelling of its repository.
|
||||
if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || added {
|
||||
t.Fatalf("added again %v: %v", added, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A merge added to a plan that is not saved is written by that plan's save, and by nothing else** (issue 1092): until
|
||||
// then it is in no record — so the event that brought it is handed over again if the save fails.
|
||||
func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) {
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
ctx := context.Background()
|
||||
if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-new")); err != nil || !added {
|
||||
t.Fatalf("added %v: %v", added, err)
|
||||
}
|
||||
if _, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); known {
|
||||
t.Fatal("a merge for a plan not saved is in a record")
|
||||
}
|
||||
// The save is refused: nothing is written, the merge is still carried.
|
||||
held := false
|
||||
away := chain.New(chain.NewMemory(), chain.NewMemory(), chain.NewMemory(), func(context.Context) error {
|
||||
if held {
|
||||
return nil
|
||||
}
|
||||
return errors.New("the lease is not held")
|
||||
})
|
||||
other := &Inventory{state: &chainState{}, recorded: inv.recorded}
|
||||
other.UseChain(away)
|
||||
t.Cleanup(other.stopViews)
|
||||
other.state.pending = inv.state.pending
|
||||
p := aBatch("plan-new", plansAt)
|
||||
if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) {
|
||||
t.Fatalf("the save answered %v", err)
|
||||
}
|
||||
if _, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); known {
|
||||
t.Fatal("a refused save left the merge in a record")
|
||||
}
|
||||
held = true
|
||||
if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); !known || m.Plan != "plan-new" {
|
||||
t.Fatalf("the save did not write the merge: %+v %v", m, known)
|
||||
}
|
||||
}
|
||||
|
||||
// failingOn is a bucket that refuses to write one key: the second half of a move, interrupted.
|
||||
type failingOn struct {
|
||||
chain.Backend
|
||||
key string
|
||||
fail *atomic.Bool
|
||||
}
|
||||
|
||||
func (f failingOn) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
|
||||
if key == f.key && f.fail.Load() {
|
||||
return 0, errors.New("the bus went away")
|
||||
}
|
||||
return f.Backend.Update(ctx, key, value, revision)
|
||||
}
|
||||
|
||||
// **A merge moved to a plan is written to the receiving record first, then taken from the giving one; one found in both is
|
||||
// read from the one it entered last and repaired** (issue 1092, ADR 0299 §7).
|
||||
func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *testing.T) {
|
||||
c := chain.InMemory(nil)
|
||||
var fail atomic.Bool
|
||||
rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail}
|
||||
c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil })
|
||||
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv.UseChain(c)
|
||||
t.Cleanup(inv.stopViews)
|
||||
ctx := context.Background()
|
||||
a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute))
|
||||
for _, p := range []*Plan{a, b} {
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The move stops before the giving record is written: the merge is in both.
|
||||
fail.Store(true)
|
||||
err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b"}})
|
||||
if err == nil {
|
||||
t.Fatal("the move was said to be made with the giving record unwritten")
|
||||
}
|
||||
fail.Store(false)
|
||||
if m, _, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); m.Plan != "plan-b" {
|
||||
t.Fatalf("a merge in two records is read from %q, not the one it moved to", m.Plan)
|
||||
}
|
||||
if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 {
|
||||
t.Fatalf("the giving plan still answers %+v", got)
|
||||
}
|
||||
// The next cut repairs it.
|
||||
v, _ := inv.rolloutView(ctx, c)
|
||||
if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 {
|
||||
t.Fatalf("the merge is held by %d records, want the two of a stopped move", n)
|
||||
}
|
||||
repaired := 0
|
||||
for k, copies := range mergeHolders(v.list()) {
|
||||
if len(copies) < 2 {
|
||||
continue
|
||||
}
|
||||
w := winner(copies)
|
||||
for _, h := range copies {
|
||||
if h.plan != w.plan {
|
||||
key := k
|
||||
if _, err := inv.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) {
|
||||
return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
repaired++
|
||||
}
|
||||
}
|
||||
}
|
||||
if repaired != 1 {
|
||||
t.Fatalf("repaired %d", repaired)
|
||||
}
|
||||
waitFor(t, func() bool {
|
||||
v, _ := inv.rolloutView(ctx, c)
|
||||
return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1
|
||||
})
|
||||
}
|
||||
|
||||
// waitFor polls a condition that a watch makes true, for a moment.
|
||||
func waitFor(t *testing.T, ok func() bool) {
|
||||
t.Helper()
|
||||
for end := time.Now().Add(3 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) {
|
||||
if ok() {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("the condition did not come true")
|
||||
}
|
||||
|
||||
// **A merge no plan answers stays, flagged, in the record that held it last, and is walked alone** (issue 1092): not in
|
||||
// the merges of the plan, listed among the merges waiting, and answered again by the plan it is moved to.
|
||||
func TestAWaitingMergeStaysInTheRecordThatHeldItLast(t *testing.T) {
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
ctx := context.Background()
|
||||
a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute))
|
||||
for _, p := range []*Plan{a, b} {
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "", Alone: true}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 {
|
||||
t.Fatalf("a waiting merge is answered by %+v", got)
|
||||
}
|
||||
alone, err := inv.chainAloneMerges(ctx, c)
|
||||
if err != nil || len(alone) != 1 || alone[0].Commit != "m1" || alone[0].Plan != "" || !alone[0].Alone {
|
||||
t.Fatalf("waiting: %+v %v", alone, err)
|
||||
}
|
||||
if m, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); !known || m.Plan != "" {
|
||||
t.Fatalf("a waiting merge reads as %+v %v", m, known)
|
||||
}
|
||||
if later, _ := inv.chainLaterMergesOf(ctx, c, "novox/mesh-catalog", "main", plansAt.Add(-time.Hour)); len(later) != 0 {
|
||||
t.Fatalf("a waiting merge is among those a plan answers: %+v", later)
|
||||
}
|
||||
if kept, first, _ := inv.chainKeptOnBranch(ctx, c, "novox/mesh-catalog", "main"); !kept || first.IsZero() {
|
||||
t.Fatalf("a waiting merge is not kept on its branch: %v %v", kept, first)
|
||||
}
|
||||
// Walked alone: the plan it goes to answers it, and the record that held it lets it go.
|
||||
if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b", Alone: true}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 || !got[0].Alone {
|
||||
t.Fatalf("plan-b answers %+v", got)
|
||||
}
|
||||
if left, _ := inv.chainAloneMerges(ctx, c); len(left) != 0 {
|
||||
t.Fatalf("still waiting: %+v", left)
|
||||
}
|
||||
waitFor(t, func() bool {
|
||||
v, _ := inv.rolloutView(ctx, c)
|
||||
return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1
|
||||
})
|
||||
// A move to a plan with no record is refused and takes nothing from the one that holds the merge.
|
||||
err = inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-nowhere"}})
|
||||
if !errors.Is(err, ErrMergeUnsaved) {
|
||||
t.Fatalf("a move to a plan with no record: %v", err)
|
||||
}
|
||||
if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 {
|
||||
t.Fatalf("the merge was taken from plan-b: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// countingBackend counts the reads a bucket answers.
|
||||
type countingBackend struct {
|
||||
chain.Backend
|
||||
gets, keys atomic.Int32
|
||||
}
|
||||
|
||||
func (c *countingBackend) Get(ctx context.Context, key string) (chain.Entry, bool, error) {
|
||||
c.gets.Add(1)
|
||||
return c.Backend.Get(ctx, key)
|
||||
}
|
||||
|
||||
func (c *countingBackend) Keys(ctx context.Context) ([]string, error) {
|
||||
c.keys.Add(1)
|
||||
return c.Backend.Keys(ctx)
|
||||
}
|
||||
|
||||
// **The surveys are read from a view kept by one watch, never from the bucket key by key** (issue 1092): the plans and the
|
||||
// merges are asked for many times and the bucket is read for none of them; a write by another process reaches the view
|
||||
// through its watch.
|
||||
func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) {
|
||||
counting := &countingBackend{Backend: chain.NewMemory()}
|
||||
c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil })
|
||||
inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }}
|
||||
inv.UseChain(c)
|
||||
t.Cleanup(inv.stopViews)
|
||||
ctx := context.Background()
|
||||
for k, id := range []string{"plan-1", "plan-2", "plan-3"} {
|
||||
p := aBatch(id, plansAt.Add(time.Duration(k)*time.Minute))
|
||||
if k > 0 {
|
||||
p.State = PlanBuilding
|
||||
}
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
before := counting.gets.Load()
|
||||
for n := 0; n < 20; n++ {
|
||||
open, err := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
|
||||
if err != nil || len(open) != 2 || open[0].ID != "plan-2" {
|
||||
t.Fatalf("open plans %+v: %v", open, err)
|
||||
}
|
||||
if _, err := inv.chainAloneMerges(ctx, c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if counting.gets.Load() != before || counting.keys.Load() != 0 {
|
||||
t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load())
|
||||
}
|
||||
// Another process writes; the view follows.
|
||||
other := &Inventory{state: &chainState{}, recorded: inv.recorded}
|
||||
other.UseChain(c)
|
||||
t.Cleanup(other.stopViews)
|
||||
late := aBatch("plan-4", plansAt.Add(time.Hour))
|
||||
late.State = PlanRolling
|
||||
if err := other.chainSavePlans(ctx, c, []*Plan{late}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
waitFor(t, func() bool {
|
||||
open, _ := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
|
||||
return len(open) == 3
|
||||
})
|
||||
}
|
||||
|
||||
// Where the record of merges begins is the earliest merge kept on any repository and branch (novox/hq ADR 0463 §2).
|
||||
func TestTheRecordOfMergesBeginsAtTheEarliestMergeKeptOnAnyTrunk(t *testing.T) {
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
ctx := context.Background()
|
||||
if kept, _, err := inv.chainEarliestKept(ctx, c); err != nil || kept {
|
||||
t.Fatalf("an inventory with no merge kept says it keeps one: %v %v", kept, err)
|
||||
}
|
||||
if err := inv.chainSavePlans(ctx, c, []*Plan{aBatch("plan-1", plansAt), aBatch("plan-2", plansAt)}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
later := aMerge("m-late", "plan-1")
|
||||
later.Repository, later.Branch, later.Heard = "novox/mesh-sdk", "main", plansAt.Add(48*time.Hour)
|
||||
earlier := aMerge("m-early", "plan-2")
|
||||
earlier.Repository, earlier.Branch, earlier.Heard = "novox/photos", "nox-mesh", plansAt.Add(time.Hour)
|
||||
for _, m := range []BatchedMerge{later, earlier} {
|
||||
if added, err := inv.chainAddMerge(ctx, c, m); err != nil || !added {
|
||||
t.Fatalf("%s: %v %v", m.Commit, added, err)
|
||||
}
|
||||
}
|
||||
kept, first, err := inv.chainEarliestKept(ctx, c)
|
||||
if err != nil || !kept || !first.Equal(plansAt.Add(time.Hour)) {
|
||||
t.Fatalf("the record begins at %v (%v, %v), want %v", first, kept, err, plansAt.Add(time.Hour))
|
||||
}
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **What delivery 1 recorded in node.sent_builds is copied onto the keys of the assignments, once** (issue 1109, review
|
||||
// round 1): a module pulled in by an assignment had no commit on its key, so it would fall out of SentBuilds. The copy
|
||||
// creates a standing pulled-in key where none stands, fills a standing key that names no commit (commit, and the build
|
||||
// when one is known, no rollout), and overwrites nothing: a key with a commit, a hand send's unknown, a wait for a build.
|
||||
func TestWhatTheColumnRecordedIsCopiedOntoTheKeysAndOverwritesNothing(t *testing.T) {
|
||||
inv, c := anInventoryOnTheChain(t)
|
||||
ctx := t.Context()
|
||||
at := time.Date(2026, 10, 18, 4, 0, 0, 0, time.UTC)
|
||||
if err := inv.chainRecordBuild(ctx, c, Build{ID: "b-wg", Module: "mesh-wireguard", Commit: "c-wg"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put := func(module string, a chain.Assignment) {
|
||||
a.Node, a.Module = "laptop", module
|
||||
if _, err := c.Assignments.Create(ctx, assignmentKey("laptop", module), a); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
put("notes", chain.Assignment{Assigned: at}) // assigned, no commit yet
|
||||
put("web", chain.Assignment{Assigned: at, Commit: "c-web-new", Build: "b1"}) // already has one
|
||||
put("hand", chain.Assignment{Assigned: at, SentUnknown: at}) // a hand send since
|
||||
put("waits", chain.Assignment{Assigned: at, State: chain.StatePending}) // a wait for a build
|
||||
put("ended", chain.Assignment{Assigned: at, Ended: at}) // unassigned
|
||||
sent := map[string]string{"notes": "c-notes", "web": "c-web-old", "mesh-wireguard": "c-wg", "hand": "c-hand",
|
||||
"waits": "c-waits", "ended": "c-ended"}
|
||||
|
||||
for pass := 0; pass < 2; pass++ { // and the second pass changes nothing
|
||||
if err := inv.copySentBuildsOf(ctx, c, "laptop", sent, at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
get := func(module string) chain.Assignment {
|
||||
t.Helper()
|
||||
k, found, err := c.Assignments.Get(ctx, assignmentKey("laptop", module))
|
||||
if err != nil || !found {
|
||||
t.Fatalf("%s: %v %v", module, found, err)
|
||||
}
|
||||
return k.Value
|
||||
}
|
||||
if a := get("notes"); a.Commit != "c-notes" || a.Rollout != "" || !a.Stands() || a.PulledIn {
|
||||
t.Errorf("notes: %+v", a)
|
||||
}
|
||||
if a := get("web"); a.Commit != "c-web-new" || a.Build != "b1" {
|
||||
t.Errorf("a key with a commit was overwritten: %+v", a)
|
||||
}
|
||||
if a := get("hand"); a.Commit != "" || a.SentUnknown.IsZero() {
|
||||
t.Errorf("a hand send's unknown was overwritten: %+v", a)
|
||||
}
|
||||
if a := get("waits"); a.Commit != "" || a.State != chain.StatePending {
|
||||
t.Errorf("a wait was overwritten: %+v", a)
|
||||
}
|
||||
if a := get("mesh-wireguard"); a.Commit != "c-wg" || a.Build != "b-wg" || !a.PulledIn || !a.Stands() || a.Rollout != "" {
|
||||
t.Errorf("the pulled-in module has no standing key of its own: %+v", a)
|
||||
}
|
||||
if a := get("ended"); a.Commit != "" || a.Stands() {
|
||||
t.Errorf("an ended key was revived or given a commit: %+v", a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The copy reads the column of the store, writes the keys, and empties the column it has copied: SentBuilds then answers the
|
||||
// pulled-in module, and a second copy finds nothing to do.
|
||||
func TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
if _, err := inv.AddNode(ctx, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.ExecForTest(t, `update node set sent_builds = '{"mesh-wireguard": "c-wg"}'::jsonb, sent_at = now() where name = 'laptop'`)
|
||||
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 1 {
|
||||
t.Fatalf("the copy did %d nodes (%v)", n, err)
|
||||
}
|
||||
got, known, err := inv.SentBuilds(ctx, "laptop")
|
||||
if err != nil || !known || got["mesh-wireguard"] != "c-wg" {
|
||||
t.Fatalf("SentBuilds answers %v, known %v (%v)", got, known, err)
|
||||
}
|
||||
if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" {
|
||||
t.Fatalf("the column was not emptied: %v", rows)
|
||||
}
|
||||
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 {
|
||||
t.Fatalf("a second copy did %d nodes (%v)", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine delivery 2 has sent has its column emptied by the send itself, so a copy after it leaves a key the send cleared
|
||||
// without a commit.
|
||||
func TestASendEmptiesTheColumnSoALaterCopyLeavesAClearedKeyBare(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.AddNode(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.ExecForTest(t, `update node set sent_builds = '{"notes": "c-old"}'::jsonb where name = 'laptop'`)
|
||||
if err := inv.RecordSentUnder(ctx, record.ID, "d1", map[string]string{}, 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" {
|
||||
t.Fatalf("a send left the column: %v", rows)
|
||||
}
|
||||
if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 {
|
||||
t.Fatalf("the copy did %d nodes (%v)", n, err)
|
||||
}
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **What a machine was sent is read from the keys of its assignments that name a commit** (ADR 0408, issue 1109): an assigned
|
||||
// module's, and a module only pulled in, whose key is a standing value marked pulled-in and never a state.
|
||||
func TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "web", "cache")
|
||||
ctx := t.Context()
|
||||
record, _ := inv.NodeByName(ctx, node)
|
||||
if _, err := inv.Assign(ctx, node, "web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, node); err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"web": "c1", "cache": "c2"}) {
|
||||
t.Fatalf("sent %v %v %v", builds, known, err)
|
||||
}
|
||||
pulled, found, _ := inv.AssignmentValue(ctx, node, "cache")
|
||||
if !found || !pulled.Stands() || pulled.State != "" || !pulled.PulledIn || !pulled.Assigned.IsZero() || pulled.Commit != "c2" {
|
||||
t.Fatalf("the module only carried has the key %+v", pulled)
|
||||
}
|
||||
if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"web"}) {
|
||||
t.Fatalf("a module only pulled in is assigned: %v", on)
|
||||
}
|
||||
// Assigned later, it keeps what it was sent and is pulled in no longer.
|
||||
if _, err := inv.Assign(ctx, node, "cache"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
made, _, _ := inv.AssignmentValue(ctx, node, "cache")
|
||||
if made.PulledIn || made.Assigned.IsZero() || made.Commit != "c2" {
|
||||
t.Fatalf("assigned over a pulled-in key: %+v", made)
|
||||
}
|
||||
}
|
||||
|
||||
// **A send replaces the whole map of the node, and a hand send makes it unknown until the next send that names its builds**
|
||||
// (issue 1109): a module no longer carried loses its commit, the node reads unknown when any key says so.
|
||||
func TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "web", "cache")
|
||||
ctx := t.Context()
|
||||
record, _ := inv.NodeByName(ctx, node)
|
||||
if _, err := inv.Assign(ctx, node, "web"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "d2", map[string]string{"web": "c3"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c3"}) {
|
||||
t.Fatalf("the second send did not replace the map: %v %v", builds, known)
|
||||
}
|
||||
if old, _, _ := inv.AssignmentValue(ctx, node, "cache"); old.Commit != "" || old.Build != "" {
|
||||
t.Fatalf("a module no longer carried keeps %+v", old)
|
||||
}
|
||||
// By hand: unknown, with a time and no commit anywhere.
|
||||
if err := inv.RecordSent(ctx, record.ID, "d3", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, _ := inv.SentBuilds(ctx, node); known || builds != nil {
|
||||
t.Fatalf("a hand send reads as %v %v", builds, known)
|
||||
}
|
||||
web, _, _ := inv.AssignmentValue(ctx, node, "web")
|
||||
if web.SentUnknown.IsZero() || web.Commit != "" || !web.Stands() {
|
||||
t.Fatalf("web after a hand send: %+v", web)
|
||||
}
|
||||
if sent, _ := inv.Outstanding(ctx, node); sent != "d3" {
|
||||
t.Fatalf("the digest of the hand send: %q", sent)
|
||||
}
|
||||
// A send that names its builds makes it known again.
|
||||
if err := inv.RecordSent(ctx, record.ID, "d4", map[string]string{"web": "c4"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c4"}) {
|
||||
t.Fatalf("after a send that names its builds: %v %v", builds, known)
|
||||
}
|
||||
if again, _, _ := inv.AssignmentValue(ctx, node, "web"); !again.SentUnknown.IsZero() {
|
||||
t.Fatalf("web still says unknown: %+v", again)
|
||||
}
|
||||
}
|
||||
|
||||
// failingToWrite is an assignments bucket that fails every write.
|
||||
type failingToWrite struct{ chain.Backend }
|
||||
|
||||
func (f failingToWrite) Create(context.Context, string, []byte) (uint64, error) {
|
||||
return 0, errors.New("the bus went away")
|
||||
}
|
||||
|
||||
func (f failingToWrite) Update(context.Context, string, []byte, uint64) (uint64, error) {
|
||||
return 0, errors.New("the bus went away")
|
||||
}
|
||||
|
||||
// **A links write that fails fails the record of the send, loudly** (issue 1109): what a machine was sent is read from the
|
||||
// keys, so a send recorded without them would read as one that never happened.
|
||||
func TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "web")
|
||||
ctx := t.Context()
|
||||
record, _ := inv.NodeByName(ctx, node)
|
||||
inv.UseChain(chain.New(chain.NewMemory(), chain.NewMemory(), failingToWrite{chain.NewMemory()},
|
||||
func(context.Context) error { return nil }))
|
||||
err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1"})
|
||||
if err == nil {
|
||||
t.Fatal("a send whose links could not be written was recorded")
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"reflect"
|
||||
"slices"
|
||||
@@ -72,10 +73,6 @@ func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) {
|
||||
if err != nil || len(announced) != 3 {
|
||||
t.Fatalf("announceable %d: %v", len(announced), err)
|
||||
}
|
||||
prints, err := inv.Fingerprints(ctx)
|
||||
if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" {
|
||||
t.Fatalf("fingerprints %v: %v", prints, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A build request row as the store held it before the move.
|
||||
@@ -166,7 +163,7 @@ func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T)
|
||||
|
||||
// **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a
|
||||
// repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module
|
||||
// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the
|
||||
// may be forgotten, what is behind — read the bucket, and a seat held by the
|
||||
// assignment goes with it.
|
||||
func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "postgres", "web")
|
||||
@@ -174,7 +171,6 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
|
||||
if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
start := generationNow(t, inv)
|
||||
if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added {
|
||||
t.Fatalf("assigned %v: %v", added, err)
|
||||
}
|
||||
@@ -185,8 +181,24 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
|
||||
if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() {
|
||||
t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err)
|
||||
}
|
||||
if generationNow(t, inv) <= start {
|
||||
t.Fatal("an assignment left the generation where it was")
|
||||
// **Placement only** (ADR 0539 §12): the value on the bus names the node, the module and since when, as mesh-config
|
||||
// reads them (unknown fields ignored there), and no build, commit or rollout.
|
||||
kv, err := inv.KeptAssignments(ctx)
|
||||
if err != nil || len(kv["anchor.postgres"].Value) == 0 {
|
||||
t.Fatalf("the kept assignments: %v %v", kv, err)
|
||||
}
|
||||
var read struct {
|
||||
Node string `json:"node"`
|
||||
Module string `json:"module"`
|
||||
Assigned time.Time `json:"assigned"`
|
||||
}
|
||||
if err := json.Unmarshal(kv["anchor.postgres"].Value, &read); err != nil || read.Node != "anchor" || read.Module != "postgres" || read.Assigned.IsZero() {
|
||||
t.Fatalf("a reader of placement reads %+v from %s: %v", read, kv["anchor.postgres"].Value, err)
|
||||
}
|
||||
for _, not := range []string{`"build"`, `"commit"`, `"rollout"`, `"artifact"`, `"package"`, `"since"`} {
|
||||
if strings.Contains(string(kv["anchor.postgres"].Value), not) {
|
||||
t.Errorf("an assignment names %s: %s", not, kv["anchor.postgres"].Value)
|
||||
}
|
||||
}
|
||||
if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) {
|
||||
t.Fatalf("a module the mesh does not know: %v", err)
|
||||
@@ -224,13 +236,9 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
|
||||
t.Fatalf("a seat handed to an assignment: %v", err)
|
||||
}
|
||||
|
||||
before := generationNow(t, inv)
|
||||
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if generationNow(t, inv) <= before {
|
||||
t.Fatal("an unassignment left the generation where it was")
|
||||
}
|
||||
// No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history.
|
||||
gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres")
|
||||
if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() {
|
||||
@@ -278,12 +286,12 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) {
|
||||
}
|
||||
|
||||
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
|
||||
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
|
||||
// plan of a module the machine was sent is written into its assignment, another field of the value stays,
|
||||
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
|
||||
// ends an assignment.
|
||||
|
||||
// **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the
|
||||
// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay,
|
||||
// plan of a module the machine was sent is written into its assignment, another field of the value stays,
|
||||
// a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here
|
||||
// ends an assignment.
|
||||
func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) {
|
||||
@@ -300,23 +308,14 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi
|
||||
Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
|
||||
Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
|
||||
raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A build and a commit put in the value by another write, after the send: a send that carries no builds (a hand send, as this
|
||||
// one) forgets what each assignment of its machine runs (issue 1109), so they are written once it is recorded.
|
||||
cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
|
||||
cur.Value.Build, cur.Value.Commit = "build-1", "c1"
|
||||
if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sentPlans(t, inv, record.ID, raw)
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
written, removed, err := inv.KeepAssignmentPlans(ctx)
|
||||
if err != nil || written != 2 || removed != 0 {
|
||||
t.Fatalf("kept: %d written, %d ended: %v", written, removed, err)
|
||||
}
|
||||
dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst")
|
||||
if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() {
|
||||
if dunst.Value.Provisioning == nil || dunst.Value.Assigned.IsZero() {
|
||||
t.Fatalf("dunst's value is %+v", dunst.Value)
|
||||
}
|
||||
if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil {
|
||||
@@ -334,9 +333,7 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi
|
||||
}
|
||||
// Sent again without the pulled-in module: its value ends, the assignments stay.
|
||||
only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`)
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sentPlans(t, inv, record.ID, only)
|
||||
if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 {
|
||||
t.Fatalf("ended %d: %v", removed, err)
|
||||
}
|
||||
@@ -351,3 +348,20 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi
|
||||
// **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value**
|
||||
// (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build
|
||||
// already is left as it is.
|
||||
|
||||
// sentPlans gives a machine the plans it was last sent, as the column the mirror reads keeps them.
|
||||
func sentPlans(t *testing.T, inv *Inventory, id string, plans []byte) {
|
||||
t.Helper()
|
||||
if _, err := inv.store.Pool().Exec(t.Context(), `update node set sent_provisioning = $2 where id = $1`, id, plans); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) string {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
|
||||
@@ -458,7 +458,7 @@ func keptLately(builds []Build, planned map[string]bool, now time.Time, within t
|
||||
return out
|
||||
}
|
||||
|
||||
// keptLatelyOrPlanned is keptLately of every build the records hold, against the plans that have not ended.
|
||||
// keptLatelyOrPlanned is keptLately of every build the records hold, (no plan names a build any more, ADR 0539 §17).
|
||||
func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]string, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
@@ -468,27 +468,7 @@ func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]strin
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := i.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return keptLately(all, plannedBuilds(plans), time.Now().UTC(), i.lately()), nil
|
||||
}
|
||||
|
||||
// plannedBuilds is every build id these plans name: a module's build, and the builds a rollout carried.
|
||||
func plannedBuilds(plans []Plan) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, p := range plans {
|
||||
for _, m := range p.Modules {
|
||||
if m != nil && m.Build != "" {
|
||||
out[m.Build] = true
|
||||
}
|
||||
}
|
||||
for _, id := range p.Builds {
|
||||
out[id] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
return keptLately(all, nil, time.Now().UTC(), i.lately()), nil
|
||||
}
|
||||
|
||||
// digestIn is the `sha256:<hex>` a reference names, empty when it names none.
|
||||
|
||||
@@ -1,134 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066.
|
||||
|
||||
// The kinds of duration recorded.
|
||||
const (
|
||||
DurationApply = "apply"
|
||||
DurationPlanTier = "plan-tier"
|
||||
DurationBuild = "build"
|
||||
// DurationWalkPhase is one phase of an ended walk, per class (novox/hq ADR 0282 decision 6): subject
|
||||
// "<class>/<phase>", and "<class>/total" for its merge to every machine running it.
|
||||
DurationWalkPhase = "walk-phase"
|
||||
// DurationBusy is a machine's wait for a busy container (novox/hq ADR 0335): from its first report of a
|
||||
// send, which said a container waits because it is busy, to the report that says it replaced — the same
|
||||
// send's ref as its `apply` row, whose detail is ApplyBusy. Its own measure, never in the apply's.
|
||||
DurationBusy = "busy-wait"
|
||||
)
|
||||
|
||||
// ApplyBusy is the detail of an `apply` duration whose report said a container waits because it is busy: the
|
||||
// machine answered, and is not yet running what it was sent (novox/hq ADR 0335).
|
||||
const ApplyBusy = "busy"
|
||||
|
||||
// DurationKinds are every kind, in the order `durations` shows them.
|
||||
var DurationKinds = []string{DurationApply, DurationPlanTier, DurationBuild, DurationWalkPhase,
|
||||
DurationBusy}
|
||||
|
||||
// DurationsKeptFor is how long a duration is kept: long enough to set a bound from, and to correct it
|
||||
// in Phase 1's first live week.
|
||||
const DurationsKeptFor = 30 * 24 * time.Hour
|
||||
|
||||
// Duration is one measurement.
|
||||
type Duration struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Ref string `json:"ref"`
|
||||
Started time.Time `json:"started"`
|
||||
Took time.Duration `json:"took"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// RecordDuration keeps one measurement, once: the same thing measured again is not a second row.
|
||||
func (i *Inventory) RecordDuration(ctx context.Context, d Duration) error {
|
||||
if d.Took < 0 {
|
||||
return nil // a clock that went back measures nothing
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||
values ($1, $2, $3, $4, $5, $6, $7) on conflict do nothing`,
|
||||
d.Kind, d.Subject, d.Node, d.Ref, d.Started, d.Took.Milliseconds(), d.Detail)
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordApplyDuration measures a machine's report of the declaration it was last sent: from the send
|
||||
// to the first report of it. A report of anything else, or of a send already measured, measures
|
||||
// nothing — a machine reconciling reports the same declaration every few minutes.
|
||||
func (i *Inventory) RecordApplyDuration(ctx context.Context, node, declared, outcome string) error {
|
||||
if declared == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||
select $1, n.name, n.name, n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US'),
|
||||
n.sent_at, (extract(epoch from (now() - n.sent_at)) * 1000)::bigint, $4
|
||||
from node n
|
||||
where n.name = $2 and n.sent = $3 and n.sent_at is not null
|
||||
on conflict do nothing`,
|
||||
DurationApply, node, declared, outcome)
|
||||
return err
|
||||
}
|
||||
|
||||
// RecordBusyDuration measures a machine's wait for a busy container, at the report that says it replaced
|
||||
// (novox/hq ADR 0335): from the end of its `apply` duration of the send it was last sent, which said it waited, to
|
||||
// now. Nothing when that send's first report did not say a wait, and once per send.
|
||||
func (i *Inventory) RecordBusyDuration(ctx context.Context, node, declared string) error {
|
||||
if declared == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`insert into duration (kind, subject, node, ref, started, took_ms, detail)
|
||||
select $1, d.subject, d.node, d.ref, d.started + d.took_ms * interval '1 millisecond',
|
||||
greatest(0, (extract(epoch from (now() - (d.started + d.took_ms * interval '1 millisecond'))) * 1000)::bigint), ''
|
||||
from duration d join node n on n.name = d.node
|
||||
where d.kind = $2 and d.subject = $3 and d.detail = $5 and n.sent = $4 and n.sent_at is not null
|
||||
and d.ref = n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US')
|
||||
on conflict do nothing`,
|
||||
DurationBusy, DurationApply, node, declared, ApplyBusy)
|
||||
return err
|
||||
}
|
||||
|
||||
// Durations is every measurement of a kind since a moment, oldest first; every kind when kind is empty.
|
||||
func (i *Inventory) Durations(ctx context.Context, kind string, since time.Time) ([]Duration, error) {
|
||||
rows, err := i.store.Pool().Query(ctx,
|
||||
`select kind, subject, node, ref, started, took_ms, detail from duration
|
||||
where ($1 = '' or kind = $1) and recorded >= $2 order by recorded`, kind, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Duration
|
||||
for rows.Next() {
|
||||
var d Duration
|
||||
var ms int64
|
||||
if err := rows.Scan(&d.Kind, &d.Subject, &d.Node, &d.Ref, &d.Started, &ms, &d.Detail); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d.Took = time.Duration(ms) * time.Millisecond
|
||||
out = append(out, d)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// WalkPhasesKept says whether any phase of a walk is kept as a walk-phase duration, and whether its total is.
|
||||
func (i *Inventory) WalkPhasesKept(ctx context.Context, plan string) (anyKept, totalKept bool, err error) {
|
||||
err = i.store.Pool().QueryRow(ctx,
|
||||
`select count(*) > 0, count(*) filter (where ref = $2) > 0 from duration where kind = $1 and ref like $3`,
|
||||
DurationWalkPhase, plan+"/total", plan+"/%").Scan(&anyKept, &totalKept)
|
||||
return anyKept, totalKept, err
|
||||
}
|
||||
|
||||
// ForgetOldDurations removes what is older than DurationsKeptFor, and says how many.
|
||||
func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) {
|
||||
tag, err := i.store.Pool().Exec(ctx, `delete from duration where recorded < $1`,
|
||||
time.Now().Add(-DurationsKeptFor))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// **The durations the bounds are set from are recorded once each** (novox/hq to-be 45 Phase 0): a
|
||||
// send measured at its first report and not again at every reconcile that repeats it; a send of
|
||||
// something else measures nothing; a new send is a new measurement.
|
||||
func TestAnApplyIsMeasuredOncePerSend(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
node, err := inv.AddNode(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, node.ID, "d1", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 3 { // the report, then two reconciles saying the same
|
||||
if err := inv.RecordApplyDuration(ctx, "anchor", "d1", OutcomeApplied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := inv.RecordApplyDuration(ctx, "anchor", "d0", OutcomeApplied); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ds, err := inv.Durations(ctx, DurationApply, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(ds) != 1 || ds[0].Subject != "anchor" || ds[0].Detail != OutcomeApplied || ds[0].Took < 0 {
|
||||
t.Fatalf("%v %+v", err, ds)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordApplyDuration(ctx, "anchor", "d2", OutcomeFailed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ds, _ = inv.Durations(ctx, "", time.Now().Add(-time.Hour)); len(ds) != 2 {
|
||||
t.Fatalf("a second send was not measured: %+v", ds)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan's tier is measured when the plan leaves it — moving on, or ending — and only then.
|
||||
func TestAPlansTierIsMeasuredWhenItIsLeft(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
|
||||
State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}},
|
||||
Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.State = PlanRolling // the same tier, saved again
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ds, _ := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)); len(ds) != 0 {
|
||||
t.Fatalf("a tier not left was measured: %+v", ds)
|
||||
}
|
||||
p.Tier = 1
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.State = PlanDone
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil { // saved again once ended: nothing more to measure
|
||||
t.Fatal(err)
|
||||
}
|
||||
ds, err := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(ds) != 2 || ds[0].Subject != "novox/mesh-tools" || ds[0].Ref != "plan-1/tier-0" ||
|
||||
ds[1].Ref != "plan-1/tier-1" {
|
||||
t.Fatalf("%v %+v", err, ds)
|
||||
}
|
||||
}
|
||||
@@ -1,335 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// The gate's verdicts (novox/hq ADR 0236, to-be 45 §8): what a build did on its first machine, and,
|
||||
// for one that failed there, how it was put back. One row per build, written by the plan that rolled it
|
||||
// out, under the lease.
|
||||
|
||||
// The gate's verdicts and a failed build's rollback.
|
||||
const (
|
||||
GatePassed = "passed"
|
||||
GateFailed = "failed"
|
||||
// GateSuperseded is a judging ended by a later send to the judged machine that moved the module to
|
||||
// another build (novox/hq issue 352): no verdict on the build, and nothing put back.
|
||||
GateSuperseded = "superseded"
|
||||
|
||||
RollingBack = "rolling-back"
|
||||
RolledBack = "rolled-back"
|
||||
NotRolledBack = "not-rolled-back"
|
||||
)
|
||||
|
||||
// GateVerdict is one build's verdict at its gate.
|
||||
type GateVerdict struct {
|
||||
Build string
|
||||
Module string
|
||||
Commit string
|
||||
Previous string
|
||||
Plan string
|
||||
Machines []string
|
||||
Verdict string
|
||||
Rollback string
|
||||
Why string
|
||||
Component string
|
||||
// JudgingFrom is when the first machine reported the build applied and the judging began.
|
||||
JudgingFrom *time.Time
|
||||
JudgedAt time.Time
|
||||
Epoch uint64
|
||||
}
|
||||
|
||||
// RecordGate writes a build's verdict. A build that passed on one machine may still fail on the next one
|
||||
// judged; **a failed build's row is written once**: any later verdict for it is refused with ErrGateKept, which is what keeps a rollback to one per build — the row
|
||||
// is written before the rollback's send, and a controller replaced in between finds it.
|
||||
func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error {
|
||||
epoch, err := i.actingEpoch(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the gate's verdict on %s is not written: %w", v.Build, err)
|
||||
}
|
||||
if v.Machines == nil {
|
||||
v.Machines = []string{}
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`insert into build_gate (build, module, commit_hash, previous, plan, machines, verdict, rollback, why,
|
||||
component, judging_from, judged_at, epoch)
|
||||
values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12)
|
||||
on conflict (build) do update set verdict = excluded.verdict, rollback = excluded.rollback,
|
||||
why = excluded.why, previous = excluded.previous, machines = excluded.machines,
|
||||
judged_at = now(), epoch = excluded.epoch
|
||||
where build_gate.verdict = 'passed'`,
|
||||
v.Build, v.Module, v.Commit, v.Previous, v.Plan, v.Machines, v.Verdict, v.Rollback, v.Why, v.Component,
|
||||
v.JudgingFrom, epoch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%w: %s", ErrGateKept, v.Build)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ErrGateKept is a verdict already kept for the build, which is not written over.
|
||||
var ErrGateKept = errors.New("this build's verdict at its gate is already kept")
|
||||
|
||||
// SetRollback records how a failed build's rollback went.
|
||||
func (i *Inventory) SetRollback(ctx context.Context, build, rollback, why string) error {
|
||||
if _, err := i.actingEpoch(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := i.store.Pool().Exec(ctx,
|
||||
`update build_gate set rollback = $2, why = $3, judged_at = now() where build = $1 and verdict = 'failed'`,
|
||||
build, rollback, why)
|
||||
return err
|
||||
}
|
||||
|
||||
// GateOf is a build's verdict, and whether it has one.
|
||||
func (i *Inventory) GateOf(ctx context.Context, build string) (GateVerdict, bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, gateSelect+` where build = $1`, build)
|
||||
if err != nil {
|
||||
return GateVerdict{}, false, err
|
||||
}
|
||||
list, err := scanGates(rows)
|
||||
if err != nil || len(list) == 0 {
|
||||
return GateVerdict{}, false, err
|
||||
}
|
||||
return list[0], true, nil
|
||||
}
|
||||
|
||||
// GateFailed is whether a build failed its gate: one the mesh never registers or sends again on its own.
|
||||
func (i *Inventory) GateFailed(ctx context.Context, build string) (bool, error) {
|
||||
v, found, err := i.GateOf(ctx, build)
|
||||
return found && v.Verdict == GateFailed, err
|
||||
}
|
||||
|
||||
// LatestGates is the newest verdict of every module that has one: what the gate probe (DG) reads.
|
||||
func (i *Inventory) LatestGates(ctx context.Context) ([]GateVerdict, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select distinct on (module) build, module, commit_hash, previous, plan,
|
||||
machines, verdict, rollback, why, component, judging_from, judged_at, coalesce(epoch, 0)
|
||||
from build_gate order by module, judged_at desc`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanGates(rows)
|
||||
}
|
||||
|
||||
// Gates is the newest verdicts, newest first: what `plans gates` lists.
|
||||
func (i *Inventory) Gates(ctx context.Context, limit int) ([]GateVerdict, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, gateSelect+` order by judged_at desc limit $1`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanGates(rows)
|
||||
}
|
||||
|
||||
const gateSelect = `select build, module, commit_hash, previous, plan, machines, verdict, rollback, why, component,
|
||||
judging_from, judged_at, coalesce(epoch, 0) from build_gate`
|
||||
|
||||
func scanGates(rows pgx.Rows) ([]GateVerdict, error) {
|
||||
defer rows.Close()
|
||||
var out []GateVerdict
|
||||
for rows.Next() {
|
||||
var v GateVerdict
|
||||
var epoch int64
|
||||
if err := rows.Scan(&v.Build, &v.Module, &v.Commit, &v.Previous, &v.Plan, &v.Machines, &v.Verdict,
|
||||
&v.Rollback, &v.Why, &v.Component, &v.JudgingFrom, &v.JudgedAt, &epoch); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
v.Epoch = uint64(epoch)
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// PreviousBuild is the build a module goes back to when a build of it fails its gate: the newest build
|
||||
// that worked, made from the commit the first machine ran before, asked before the failed one, and not
|
||||
// itself failed at a gate. Among the builds whose artifacts the mesh keeps (KeptBuilds): an older one
|
||||
// may already be gone from the artifact store. False when there is none to go back to.
|
||||
func (i *Inventory) PreviousBuild(ctx context.Context, module, commit string, failed Build) (Build, bool, error) {
|
||||
builds, err := i.Builds(ctx, module, 50)
|
||||
if err != nil {
|
||||
return Build{}, false, err
|
||||
}
|
||||
kept := 0
|
||||
for _, b := range builds {
|
||||
if !b.Worked() {
|
||||
continue
|
||||
}
|
||||
kept++
|
||||
if kept > KeptBuilds {
|
||||
break
|
||||
}
|
||||
if b.ID == failed.ID || (commit != "" && b.Commit != commit) {
|
||||
continue
|
||||
}
|
||||
// Only a build of the repository the failed build was made from — the module's, since its take-in
|
||||
// registered it (novox/hq ADR 0266): a build of the module's name from another repository is recorded
|
||||
// and was never registered, and putting it back would register it now.
|
||||
if failed.Repository != "" && !sameRepositoryAs(b.Repository, failed.Repository) {
|
||||
continue
|
||||
}
|
||||
if !failed.AskedOrAt().IsZero() && !b.AskedOrAt().Before(failed.AskedOrAt()) {
|
||||
continue
|
||||
}
|
||||
if bad, err := i.GateFailed(ctx, b.ID); err != nil {
|
||||
return Build{}, false, err
|
||||
} else if bad {
|
||||
continue
|
||||
}
|
||||
manifest, err := i.manifestOf(ctx, b.ID)
|
||||
if err != nil {
|
||||
return Build{}, false, err
|
||||
}
|
||||
if len(manifest) == 0 || string(manifest) == "null" {
|
||||
continue
|
||||
}
|
||||
b.Manifest = manifest
|
||||
return b, true, nil
|
||||
}
|
||||
return Build{}, false, nil
|
||||
}
|
||||
|
||||
// sameRepositoryAs says two recorded repositories are one, however their case or `.git` is spelled.
|
||||
func sameRepositoryAs(a, b string) bool {
|
||||
trim := func(s string) string {
|
||||
return strings.TrimSuffix(strings.TrimRight(strings.ToLower(strings.TrimSpace(s)), "/"), ".git")
|
||||
}
|
||||
return trim(a) == trim(b)
|
||||
}
|
||||
|
||||
// RestoreModule puts a module's registered build back to an earlier one: its manifest, the commit it
|
||||
// was built from, and when it was asked — as now, so the build that failed its gate, asked before, can
|
||||
// never register over it again (issue 219's order). The source's head is left where the merge moved it:
|
||||
// the module IS behind its source, and `status` says so.
|
||||
func (i *Inventory) RestoreModule(ctx context.Context, b Build) error {
|
||||
if _, err := i.actingEpoch(ctx); err != nil {
|
||||
return fmt.Errorf("%s is not put back: %w", b.Module, err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(b.Manifest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s's build %s is not a manifest the mesh can register again: %w", b.Module, b.ID, err)
|
||||
}
|
||||
raw, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := i.store.Pool().Exec(ctx,
|
||||
`update module set manifest = $2, version = nullif($3, ''), built_from = nullif($4, ''),
|
||||
built_asked = now(), registered = now()
|
||||
where name = $1`, b.Module, raw, m.Version, b.Commit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("%w: %s", ErrNoSuchModule, b.Module)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BuildFingerprints is, per commit, what the newest successful build of a module from it would put on a
|
||||
// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told
|
||||
// from one that does (the bus's planned step, ADR 0236).
|
||||
func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return i.chainBuildFingerprints(ctx, c, module)
|
||||
}
|
||||
|
||||
// Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint.
|
||||
func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return i.chainFingerprints(ctx, c)
|
||||
}
|
||||
|
||||
// PassedCommits is, per module, the commits a build of which passed its gate on some machine.
|
||||
func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select module, commit_hash from build_gate where verdict = 'passed'`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]map[string]bool{}
|
||||
for rows.Next() {
|
||||
var module, commit string
|
||||
if err := rows.Scan(&module, &commit); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out[module] == nil {
|
||||
out[module] = map[string]bool{}
|
||||
}
|
||||
out[module][commit] = true
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// BuildOf is the id of the newest successful build of a module from a commit; empty when none is
|
||||
// recorded (a manifest handed over by hand).
|
||||
func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return i.chainBuildOf(ctx, c, module, commit)
|
||||
}
|
||||
|
||||
// ManifestAt is the manifest a module was registered with at a commit (or a commit it abbreviates): the
|
||||
// newest successful build from it, with the artifacts of the build standing for it when its source was
|
||||
// unchanged (issue 280) — what a machine last sent that build runs. False when no such build, or none
|
||||
// with a manifest, is kept (novox/hq issue 295: a recorded module is composed at the build its machine
|
||||
// runs until a person's push moves it).
|
||||
func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (catalogue.Manifest, bool, error) {
|
||||
if commit == "" {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
id, raw, found, err := i.newestManifestAt(ctx, module, commit)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, false, err
|
||||
}
|
||||
if !found {
|
||||
return catalogue.Manifest{}, false, nil
|
||||
}
|
||||
if stands, same, err := i.StandingBuild(ctx, module, id); err != nil {
|
||||
return catalogue.Manifest{}, false, err
|
||||
} else if stands != "" && stands != id && len(same) > 0 {
|
||||
raw = same
|
||||
}
|
||||
m, err := catalogue.ParseManifest(raw)
|
||||
if err != nil {
|
||||
return catalogue.Manifest{}, false, fmt.Errorf("%s's build from %s is not a manifest the mesh can compose: %w",
|
||||
module, commit, err)
|
||||
}
|
||||
return m, true, nil
|
||||
}
|
||||
|
||||
// manifestOf is the manifest a build was recorded with; nil when it kept none.
|
||||
func (i *Inventory) manifestOf(ctx context.Context, id string) ([]byte, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return i.chainManifestOf(ctx, c, id)
|
||||
}
|
||||
|
||||
// newestManifestAt is the newest successful build of a module from a commit, or one it abbreviates, that kept a manifest:
|
||||
// its id and manifest; false when there is none.
|
||||
func (i *Inventory) newestManifestAt(ctx context.Context, module, commit string) (string, []byte, bool, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return "", nil, false, err
|
||||
}
|
||||
return i.chainManifestBuild(ctx, c, module, commit)
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import "testing"
|
||||
|
||||
// 1183/AC3: the heartbeat-gap duration is no kind the mesh measures.
|
||||
func TestNoHeartbeatGapIsMeasured(t *testing.T) { // 1183/AC3
|
||||
for _, k := range DurationKinds {
|
||||
if k == "heartbeat-gap" {
|
||||
t.Fatalf("%s is still a kind of duration: %v", k, DurationKinds)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// novox/hq issue 218: a seat held once for the mesh is granted and issued only to the holder on record;
|
||||
// a node seat to every machine's claimant; a mesh seat with no holder on record as derived.
|
||||
func TestOnlyTheRecordedHolderHoldsAMeshSeat(t *testing.T) {
|
||||
claimed := []broker.Seat{
|
||||
{Name: "mesh-store", Scope: catalogue.ScopeMesh},
|
||||
{Name: "node-packet-filter", Scope: catalogue.ScopeNode},
|
||||
{Name: "unrecorded", Scope: catalogue.ScopeMesh},
|
||||
}
|
||||
holdings := []catalogue.Held{{Claim: "mesh-store", Scope: catalogue.ScopeMesh, Node: "control", Module: "postgres"}}
|
||||
names := func(ss []broker.Seat) (out []string) {
|
||||
for _, s := range ss {
|
||||
out = append(out, s.Name)
|
||||
}
|
||||
return
|
||||
}
|
||||
if got := names(heldHere(claimed, holdings, "control", "postgres")); len(got) != 3 {
|
||||
t.Errorf("the holder lost a seat: %v", got)
|
||||
}
|
||||
got := names(heldHere(claimed, holdings, "other", "postgres"))
|
||||
if len(got) != 2 || got[0] != "node-packet-filter" || got[1] != "unrecorded" {
|
||||
t.Errorf("a claimant on another machine holds %v; want the node seat and the unrecorded one, not the store", got)
|
||||
}
|
||||
}
|
||||
@@ -1,149 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrNodeBusy is a node another act kept holding for longer than a caller waits.
|
||||
var ErrNodeBusy = errors.New("another act is composing or sending that node's declaration")
|
||||
|
||||
// HoldWaitFor is how long HoldNodes waits for a node another act holds, and HoldPoll how often it
|
||||
// looks again. Variables so a test need not wait minutes.
|
||||
var (
|
||||
HoldWaitFor = 2 * time.Minute
|
||||
HoldPoll = 250 * time.Millisecond
|
||||
)
|
||||
|
||||
// HoldNodes serialises composing and sending a declaration per node (novox/hq ADR 0100): while
|
||||
// one caller holds a node, another asking for it waits. Without it a push that composed a node as
|
||||
// adopted could send that declaration after `converge --yes` sent the converged one, and the node
|
||||
// would return to adopted with nobody having asked.
|
||||
//
|
||||
// Session-level advisory locks on one connection, all or none: a set not wholly free is given back
|
||||
// at once, so two callers holding overlapping sets never each wait on the other. **A waiter pins
|
||||
// no connection.** It looks again every HoldPoll with a connection borrowed for the look, and gives
|
||||
// up after HoldWaitFor with ErrNodeBusy naming the node — so a stuck holder costs the pool one
|
||||
// connection, never one per caller queued behind it. Release gives every one back, and may be
|
||||
// called more than once.
|
||||
func (i *Inventory) HoldNodes(ctx context.Context, names []string) (func(), error) {
|
||||
sorted := slices.Clone(names)
|
||||
slices.Sort(sorted)
|
||||
sorted = slices.Compact(sorted)
|
||||
deadline := time.Now().Add(HoldWaitFor)
|
||||
for {
|
||||
release, busy, err := i.tryHold(ctx, sorted)
|
||||
if err != nil || busy == "" {
|
||||
return release, err
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return nil, fmt.Errorf("%w: %s has been held for over %s — try again once it is done",
|
||||
ErrNodeBusy, busy, HoldWaitFor)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(HoldPoll):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tryHold takes every named node's lock or none, and says which node was busy when it took none.
|
||||
func (i *Inventory) tryHold(ctx context.Context, sorted []string) (func(), string, error) {
|
||||
conn, err := i.store.Pool().Acquire(ctx)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
var once sync.Once
|
||||
release := func() {
|
||||
once.Do(func() {
|
||||
// Unlocking all of this session's advisory locks, then handing the connection back: a
|
||||
// connection returned still holding one would hold it for whoever borrows it next.
|
||||
_, err := conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock_all()`)
|
||||
if err != nil {
|
||||
// The session's locks die with the session: close it rather than return it.
|
||||
_ = conn.Conn().Close(context.WithoutCancel(ctx))
|
||||
}
|
||||
conn.Release()
|
||||
})
|
||||
}
|
||||
for _, name := range sorted {
|
||||
var took bool
|
||||
if err := conn.QueryRow(ctx,
|
||||
`select pg_try_advisory_lock(hashtext('mesh-node-declaration:' || $1)::bigint)`,
|
||||
name).Scan(&took); err != nil {
|
||||
release()
|
||||
return nil, "", err
|
||||
}
|
||||
if !took {
|
||||
release()
|
||||
return nil, strings.TrimSpace(name), nil
|
||||
}
|
||||
}
|
||||
return release, "", nil
|
||||
}
|
||||
|
||||
// ErrPlansBusy is the plans held by another act — on a machine replacing its controller, the other
|
||||
// controller — for longer than a caller waits, or at all for one that does not wait.
|
||||
var ErrPlansBusy = errors.New("another controller is working the plans")
|
||||
|
||||
// HoldPlans makes working the plans one act at a time, across every controller on the store
|
||||
// (novox/hq issue 213). A plan is read, changed and written whole; two controllers doing that at
|
||||
// once — the old and the new for the moment a machine hands its controller over, or a controller
|
||||
// and a person's `plans stop` — each act on what the other has not saved yet: a tier asked twice,
|
||||
// an outcome written over. A session-level advisory lock on one connection, released by the
|
||||
// returned function and by the session ending, so a controller that dies holding it holds nothing.
|
||||
//
|
||||
// wait false gives ErrPlansBusy at once when another holds them — the timer's way: the holder is
|
||||
// moving the plans already; and a build outcome's on the event loop, which the build records keep
|
||||
// for the look to settle (novox/hq issue 1708). wait true looks again every HoldPoll for up to
|
||||
// HoldWaitFor — a merge's way, which must be written.
|
||||
func (i *Inventory) HoldPlans(ctx context.Context, wait bool) (func(), error) {
|
||||
deadline := time.Now().Add(HoldWaitFor)
|
||||
for {
|
||||
release, took, err := i.tryLock(ctx, "mesh-plans")
|
||||
if err != nil || took {
|
||||
return release, err
|
||||
}
|
||||
if !wait || time.Now().After(deadline) {
|
||||
return nil, ErrPlansBusy
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(HoldPoll):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tryLock takes one named advisory lock on a connection of its own, or gives the connection back.
|
||||
func (i *Inventory) tryLock(ctx context.Context, key string) (func(), bool, error) {
|
||||
conn, err := i.store.Pool().Acquire(ctx)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
var once sync.Once
|
||||
release := func() {
|
||||
once.Do(func() {
|
||||
if _, err := conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock_all()`); err != nil {
|
||||
_ = conn.Conn().Close(context.WithoutCancel(ctx))
|
||||
}
|
||||
conn.Release()
|
||||
})
|
||||
}
|
||||
var took bool
|
||||
if err := conn.QueryRow(ctx, `select pg_try_advisory_lock(hashtext($1)::bigint)`, key).Scan(&took); err != nil {
|
||||
release()
|
||||
return nil, false, err
|
||||
}
|
||||
if !took {
|
||||
release()
|
||||
return nil, false, nil
|
||||
}
|
||||
return release, true, nil
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// novox/hq issue 213: while a machine hands its controller over from the container to the process,
|
||||
// two controllers run on one store for a moment. Working the plans is one act at a time across them.
|
||||
func TestThePlansAreWorkedByOneControllerAtATime(t *testing.T) {
|
||||
first := ForTest(t)
|
||||
// A second controller: its own connections to the same store.
|
||||
second, err := Open(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(second.Close)
|
||||
|
||||
release, err := first.HoldPlans(t.Context(), false)
|
||||
if err != nil {
|
||||
t.Fatalf("the plans could not be held when nobody held them: %v", err)
|
||||
}
|
||||
t.Cleanup(release) // a pool closing waits for a connection still held; release is idempotent
|
||||
if _, err := second.HoldPlans(t.Context(), false); !errors.Is(err, ErrPlansBusy) {
|
||||
t.Fatalf("a second controller held the plans while the first did: %v", err)
|
||||
}
|
||||
// A waiter gets them once they are let go.
|
||||
was := HoldPoll
|
||||
HoldPoll = 10 * time.Millisecond
|
||||
defer func() { HoldPoll = was }()
|
||||
go func() { time.Sleep(50 * time.Millisecond); release() }()
|
||||
again, err := second.HoldPlans(t.Context(), true)
|
||||
if err != nil {
|
||||
t.Fatalf("a waiting controller never got the plans once they were let go: %v", err)
|
||||
}
|
||||
again()
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Composing and sending one node's declaration is serialised: a second holder waits for the first.
|
||||
func TestHoldingANodeMakesTheNextHolderWait(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := t.Context()
|
||||
release, err := inv.HoldNodes(ctx, []string{"anchor", "laptop"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := make(chan func(), 1)
|
||||
go func() {
|
||||
second, err := inv.HoldNodes(ctx, []string{"laptop"})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
got <- func() {}
|
||||
return
|
||||
}
|
||||
got <- second
|
||||
}()
|
||||
select {
|
||||
case <-got:
|
||||
t.Fatal("a node held by one caller was held by another at the same time")
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
}
|
||||
// Another node is not held up.
|
||||
other, err := inv.HoldNodes(ctx, []string{"joiner"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other()
|
||||
release()
|
||||
select {
|
||||
case second := <-got:
|
||||
second()
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("releasing the node did not let the next holder in")
|
||||
}
|
||||
}
|
||||
|
||||
// A waiter pins no pool connection while it waits, and gives up after a bounded wait saying which
|
||||
// node is busy.
|
||||
func TestAWaiterPinsNoConnectionAndGivesUp(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := t.Context()
|
||||
release, err := inv.HoldNodes(ctx, []string{"anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release()
|
||||
savedWait, savedPoll := HoldWaitFor, HoldPoll
|
||||
HoldWaitFor, HoldPoll = 1500*time.Millisecond, 50*time.Millisecond
|
||||
defer func() { HoldWaitFor, HoldPoll = savedWait, savedPoll }()
|
||||
|
||||
pool := inv.store.Pool()
|
||||
base := pool.Stat().AcquiredConns()
|
||||
const waiters = 3
|
||||
done := make(chan error, waiters)
|
||||
for range waiters {
|
||||
go func() {
|
||||
_, err := inv.HoldNodes(ctx, []string{"anchor"})
|
||||
done <- err
|
||||
}()
|
||||
}
|
||||
// While they wait, the pool lends nothing to them for longer than a look: **pinned is held throughout**,
|
||||
// so at some moment of the wait fewer than all of them hold one. Asked of the least seen over many
|
||||
// looks, not of the most: a look that lands while every waiter is mid-query is a busy machine, not a
|
||||
// pinned connection, and on a loaded one (the suite's packages at once, the race detector) it did land.
|
||||
least := waiters
|
||||
for range 100 {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if n := int(pool.Stat().AcquiredConns() - base); n < least {
|
||||
least = n
|
||||
}
|
||||
}
|
||||
if least >= waiters {
|
||||
t.Fatalf("%d waiters held %d connections at every look across their wait", waiters, least)
|
||||
}
|
||||
for range waiters {
|
||||
if err := <-done; !errors.Is(err, ErrNodeBusy) || !strings.Contains(err.Error(), "anchor") {
|
||||
t.Fatalf("a waiter did not give up naming the busy node: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
-- The tables of the delivery plans' machinery leave the store (novox/hq ADR 0539 §17, design 47 release 6, issue 1945).
|
||||
--
|
||||
-- A machine follows its wants and applies a module from the composition the controller writes for it (ADR 0539 §1, §2), so
|
||||
-- the controller keeps no record of what it sent, no counter of the assignments a send was composed from, no verdict of a
|
||||
-- gate that judged a build on its first machine and no measure of how long a plan's tier or a machine's apply took. This
|
||||
-- drops the four tables that kept them: declaration_send (0094), assignment_generation (0094), build_gate (0073) and duration
|
||||
-- (0066, 0075).
|
||||
--
|
||||
-- **It is one-way**: a controller older than this schema is refused by the put-back gate (the highest migration it carries is
|
||||
-- recorded by its version, issue 352). What stays: the node row and its columns (the digest of the last send, which the old
|
||||
-- reads of `status` still name until they go with their verbs), the bus step, the core box and the refused registrations.
|
||||
-- The function raise_assignment_generation and its triggers went with 0105.
|
||||
drop table if exists declaration_send;
|
||||
drop table if exists assignment_generation;
|
||||
drop table if exists build_gate;
|
||||
drop table if exists duration;
|
||||
@@ -37,8 +37,6 @@ type Inventory struct {
|
||||
// chain is the controller's chain on the bus (novox/hq ADR 0408), once attached (UseChain).
|
||||
chain *chain.Chain
|
||||
state *chainState
|
||||
// recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store.
|
||||
recorded func(context.Context, Duration) error
|
||||
// throwaway is set on an inventory raised for a moment beside the real mesh (the merge gate's): nothing it
|
||||
// composes is to be written to the store or the bus.
|
||||
throwaway bool
|
||||
@@ -73,7 +71,6 @@ func Open(ctx context.Context) (*Inventory, error) {
|
||||
}
|
||||
|
||||
func (i *Inventory) Close() {
|
||||
i.stopViews()
|
||||
i.store.Close()
|
||||
}
|
||||
|
||||
@@ -1283,8 +1280,9 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu
|
||||
`update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3 where id = $1`, node, digest, sentEpoch); err != nil {
|
||||
return err
|
||||
}
|
||||
// What it was sent of each module is the assignments' (SentBuilds); a write of it that fails fails this record.
|
||||
return i.recordRuns(ctx, node, builds, time.Now())
|
||||
// What it was sent of each module is no longer kept in the assignments: they hold placement only (ADR 0539 §12).
|
||||
_ = builds
|
||||
return nil
|
||||
}
|
||||
|
||||
// AwaitingSince is since when a machine has had something waiting for its next send (novox/hq issue
|
||||
@@ -1301,36 +1299,8 @@ func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time,
|
||||
// assigned module and one only pulled in (ADR 0408, issue 1109). Known is false when that was not kept: a machine never sent
|
||||
// anything, or last sent a declaration by hand, which makes every key of it say its commit unknown.
|
||||
func (i *Inventory) SentBuilds(ctx context.Context, name string) (builds map[string]string, known bool, err error) {
|
||||
var sentAt *time.Time
|
||||
err = i.store.Pool().QueryRow(ctx, `select sent_at from node where name = $1`, name).Scan(&sentAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil || sentAt == nil {
|
||||
return nil, false, err
|
||||
}
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
all, err := c.Assignments.All(ctx)
|
||||
if err != nil {
|
||||
return nil, false, fmt.Errorf("what %s was sent cannot be read from the bus: %w", name, err)
|
||||
}
|
||||
builds = map[string]string{}
|
||||
for _, k := range all {
|
||||
a := k.Value
|
||||
if a.Node != name || !a.Stands() {
|
||||
continue
|
||||
}
|
||||
if !a.SentUnknown.IsZero() {
|
||||
return nil, false, nil
|
||||
}
|
||||
if a.Commit != "" {
|
||||
builds[a.Module] = a.Commit
|
||||
}
|
||||
}
|
||||
return builds, true, nil
|
||||
// Never known: the assignments hold placement only (ADR 0539 §12), and what a machine runs is its have.
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
// RecordSentBusUsers keeps a digest of the bus's user list a machine was just sent, by its name
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq 04-ISSUES/219: two builds of one module in flight together, the one asked first heard
|
||||
// last. The newer request stood on the newer base; the older one's late outcome is recorded and is
|
||||
// not what the module is.
|
||||
|
||||
func postgresBuild(id string, asked time.Time, image string) Build {
|
||||
b := aBuild(id, "postgres", "")
|
||||
b.Asked = asked
|
||||
b.Against = []string{"mesh-tools/runtime@sha256:" + id}
|
||||
b.Made = []Artifact{{Name: "server", Kind: "image", Reference: "postgres@sha256:" + image}}
|
||||
return b
|
||||
}
|
||||
|
||||
func TestAnOlderRequestFinishingLaterIsNotWhatTheModuleHolds(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
||||
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
||||
|
||||
// The newer request finishes first, the older one last — recorded in that order.
|
||||
if err := inv.RecordBuild(ctx, postgresBuild("newer", newer, "4bcd5f73")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordBuild(ctx, postgresBuild("older", older, "0ab07fa9")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
held, err := inv.Held(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := held["postgres/server"]; got != "postgres@sha256:4bcd5f73" {
|
||||
t.Errorf("postgres holds %q; want the newer request's image 4bcd5f73", got)
|
||||
}
|
||||
against, err := inv.BuiltAgainst(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := against["postgres"]; len(got) != 1 || got[0] != "mesh-tools/runtime@sha256:newer" {
|
||||
t.Errorf("postgres stands on %v; want what the newer request stood on", got)
|
||||
}
|
||||
|
||||
// Both are still recorded, the late one first as what happened lately.
|
||||
builds, err := inv.Builds(ctx, "postgres", 5)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(builds) != 2 || builds[0].ID != "older" || !builds[0].Asked.Equal(older) {
|
||||
t.Fatalf("both builds, newest heard first, with when they were asked: %+v", builds)
|
||||
}
|
||||
}
|
||||
|
||||
func TestABuildWithNoKnownRequestTimeIsOrderedByWhenItWasRecorded(t *testing.T) {
|
||||
// What the mesh did before it kept the request time, so a row from before still answers.
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
for _, id := range []string{"first", "second"} {
|
||||
b := aBuild(id, "shell", "")
|
||||
b.Made = []Artifact{{Name: "config", Kind: "archive", Reference: "…/" + id}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
held, err := inv.Held(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := held["shell/config"]; got != "…/second" {
|
||||
t.Errorf("shell holds %q; want the one recorded last", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestARegistrationFromAnOlderRequestDoesNotReplaceANewerOne(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := context.Background()
|
||||
older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC)
|
||||
newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC)
|
||||
from := func(asked time.Time) Source {
|
||||
return Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/postgres",
|
||||
BuiltFrom: "efff5415", Asked: asked}
|
||||
}
|
||||
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "fixed"}, from(newer)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "stale"}, from(older))
|
||||
if !errors.Is(err, ErrSuperseded) {
|
||||
t.Fatalf("an older request's registration was not refused as superseded: %v", err)
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := shelf["postgres"].Version; got != "fixed" {
|
||||
t.Fatalf("postgres is %q; want the newer request's manifest", got)
|
||||
}
|
||||
|
||||
// A later request, and a manifest handed over by hand — asked when it is handed over — both
|
||||
// replace it as before.
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "later"},
|
||||
from(newer.Add(time.Minute))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "by-hand"}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if shelf, _ := inv.Catalogue(ctx); shelf["postgres"].Version != "by-hand" {
|
||||
t.Fatalf("postgres is %q; want the manifest handed over by hand", shelf["postgres"].Version)
|
||||
}
|
||||
src, err := inv.SourceOf(ctx, "postgres")
|
||||
if err != nil || src.Repository != "novox/mesh-catalog" {
|
||||
t.Fatalf("a hand registration erased the provenance: %+v %v", src, err)
|
||||
}
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Order and one writer, as the store keeps them (novox/hq to-be 45 §6).
|
||||
|
||||
// **A plan is written by compare-and-set on its revision, carrying the epoch**: a write against a plan
|
||||
// another writer moved since is refused, and nothing is written by a process that may not act.
|
||||
func TestAPlanIsWrittenByCompareAndSetUnderTheLease(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := context.Background()
|
||||
epoch := uint64(57)
|
||||
inv.ActsUnder(func(context.Context) (uint64, error) { return epoch, nil })
|
||||
|
||||
p := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), State: PlanBuilding,
|
||||
Tiers: [][]string{{"app"}}, Modules: map[string]*PlanModule{"app": {}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if p.Revision != 1 || p.Epoch != 57 {
|
||||
t.Fatalf("a new plan was written at revision %d, epoch %d", p.Revision, p.Epoch)
|
||||
}
|
||||
// Two readers of revision 1: the first write wins, the second is refused and writes nothing.
|
||||
first, err := inv.PlanByID(ctx, "plan-cas")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second := first
|
||||
first.Note = "the newer word"
|
||||
if err := inv.SavePlan(ctx, &first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second.State = PlanFailed
|
||||
if err := inv.SavePlan(ctx, &second); !errors.Is(err, ErrPlanMoved) {
|
||||
t.Fatalf("a write against a plan moved since it was read was not refused: %v", err)
|
||||
}
|
||||
kept, _ := inv.PlanByID(ctx, "plan-cas")
|
||||
if kept.State != PlanBuilding || kept.Note != "the newer word" || kept.Revision != 2 {
|
||||
t.Fatalf("the refused write reached the plan: %+v", kept)
|
||||
}
|
||||
// The writer saves its own plan again without reading it: its revision moved with its write.
|
||||
first.Tier = 0
|
||||
if err := inv.SavePlan(ctx, &first); err != nil {
|
||||
t.Fatalf("a writer could not save its own plan twice: %v", err)
|
||||
}
|
||||
// A new plan under an id already written is refused, not laid over it.
|
||||
again := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "deadbeef", Created: time.Now(), State: PlanBuilding}
|
||||
if err := inv.SavePlan(ctx, &again); !errors.Is(err, ErrPlanMoved) {
|
||||
t.Fatalf("a second plan under one id was written: %v", err)
|
||||
}
|
||||
// And a process that does not hold the lease writes nothing.
|
||||
inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") })
|
||||
first.Note = "from a controller that lost the lease"
|
||||
if err := inv.SavePlan(ctx, &first); err == nil {
|
||||
t.Fatal("a plan was written by a controller that does not hold the lease")
|
||||
}
|
||||
if kept, _ := inv.PlanByID(ctx, "plan-cas"); kept.Note != "the newer word" || kept.Epoch != 57 {
|
||||
t.Fatalf("a controller without the lease wrote the plan: %+v", kept)
|
||||
}
|
||||
}
|
||||
|
||||
// The epochs the mesh issued: the highest is the floor; taking one ends every earlier one nobody gave
|
||||
// back as found expired, and says which; one given back says so and is not said again.
|
||||
func TestTheEpochsIssuedAreKept(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := context.Background()
|
||||
if highest, err := inv.HighestEpoch(ctx); err != nil || highest != 0 {
|
||||
t.Fatalf("a mesh that issued none has %d (%v)", highest, err)
|
||||
}
|
||||
if _, err := inv.TookEpoch(ctx, Epoch{Epoch: 41, Instance: "a", Taken: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// a stopped renewing and said nothing; b takes the lease.
|
||||
expired, err := inv.TookEpoch(ctx, Epoch{Epoch: 57, Instance: "b", Taken: time.Now()})
|
||||
if err != nil || len(expired) != 1 || expired[0].Epoch != 41 || expired[0].How != EpochExpired {
|
||||
t.Fatalf("taking 57 ended %+v (%v), want 41 found expired", expired, err)
|
||||
}
|
||||
if err := inv.EndEpoch(ctx, 57, EpochReleased); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expired, err = inv.TookEpoch(ctx, Epoch{Epoch: 60, Instance: "c", Taken: time.Now()})
|
||||
if err != nil || len(expired) != 0 {
|
||||
t.Fatalf("a lease given back was found expired: %+v (%v)", expired, err)
|
||||
}
|
||||
if highest, _ := inv.HighestEpoch(ctx); highest != 60 {
|
||||
t.Fatalf("the highest epoch issued is %d, want 60", highest)
|
||||
}
|
||||
e, found, err := inv.EpochOf(ctx, 57)
|
||||
if err != nil || !found || e.Instance != "b" || e.How != EpochReleased || e.Ended == nil {
|
||||
t.Fatalf("epoch 57 reads %+v (%v, %v)", e, found, err)
|
||||
}
|
||||
recent, err := inv.EpochsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(recent) != 3 || recent[0].Epoch != 60 || recent[0].Ended != nil {
|
||||
t.Fatalf("the epochs of the last hour read %+v (%v)", recent, err)
|
||||
}
|
||||
}
|
||||
|
||||
// What a machine was sent under, and whether it reads an epoch.
|
||||
func TestTheEpochAMachineWasSentIsKept(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := context.Background()
|
||||
node, err := inv.AddNode(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSentUnder(ctx, node.ID, "d1", nil, 57); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, err := inv.SentEpoch(ctx, node.ID); err != nil || e != 57 {
|
||||
t.Fatalf("sent under %d (%v)", e, err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e, _ := inv.SentEpoch(ctx, node.ID); e != 0 {
|
||||
t.Fatalf("a declaration sent without an epoch left %d kept", e)
|
||||
}
|
||||
if reads, _ := inv.ReadsEpoch(ctx, node.ID); reads {
|
||||
t.Fatal("a machine that never said so reads an epoch")
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The plan's backups are the contract mesh-delivery reads from `plan-moved` (issue 1065): their JSON field names
|
||||
// are fixed here, and a plan keeps them through a round trip.
|
||||
func TestAPlanKeepsItsBackupsThroughItsJSONUnderTheNamesTheDeliveryReads(t *testing.T) {
|
||||
at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC)
|
||||
p := Plan{ID: "p", Backups: &PlanBackups{State: BackupFailed, NoHolder: []string{"bare"},
|
||||
Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupFailed, AskedAt: &at, Words: "no space left"}}}}
|
||||
raw, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{`"backups":{"state":"failed","asked":[{"machine":"ace","module":"mail","state":"failed",` +
|
||||
`"asked_at":"2026-10-17T12:00:00Z","words":"no space left"}],"no_holder":["bare"]}`} {
|
||||
if !strings.Contains(string(raw), name) {
|
||||
t.Fatalf("plan JSON %s lacks %s", raw, name)
|
||||
}
|
||||
}
|
||||
var back Plan
|
||||
if err := json.Unmarshal(raw, &back); err != nil || !reflect.DeepEqual(back.Backups, p.Backups) {
|
||||
t.Fatalf("round trip: %v %+v", err, back.Backups)
|
||||
}
|
||||
raw, _ = json.Marshal(Plan{ID: "q"})
|
||||
if strings.Contains(string(raw), "backups") {
|
||||
t.Fatalf("a plan with no backups says none: %s", raw)
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// **A plan's id is a value, never a query** (novox/hq issue 561): the id reaches PlanByID from a caller of the
|
||||
// controller's plans verb, so an id holding a quote is read as a plain id — no error from the store, and no
|
||||
// other plan answered for it. On the bus an id is a key, and one the bus takes no spelling of is refused to be kept.
|
||||
func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
p := Plan{ID: "plan-561", Repository: "novox/mesh-controller", Commit: "abc", Created: time.Now().UTC(),
|
||||
State: PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*PlanModule{"a": {}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, id := range []string{
|
||||
"x' or '1'='1", // every plan, when pasted into the query
|
||||
"plan-561' or id like '%", // the plan above, by a pattern
|
||||
"it's", // a quote alone: a syntax error, when pasted
|
||||
} {
|
||||
got, err := inv.PlanByID(ctx, id)
|
||||
if err == nil {
|
||||
t.Fatalf("PlanByID(%q) answered %s, a plan of another id", id, got.ID)
|
||||
}
|
||||
if want := "no plan " + id; err.Error() != want {
|
||||
t.Fatalf("PlanByID(%q) failed with %q, not as a plain id no plan has (%q)", id, err, want)
|
||||
}
|
||||
}
|
||||
// A plan whose id holds a quote cannot be a key of the bus: refused by name, never rewritten into another id.
|
||||
quoted := p
|
||||
quoted.ID, quoted.Revision = "plan-'561'", 0
|
||||
if err := inv.SavePlan(ctx, "ed); !errors.Is(err, chain.ErrBadKey) {
|
||||
t.Fatalf("a plan with a quote in its id was kept, or refused for another reason: %v", err)
|
||||
}
|
||||
if recent, err := inv.RecentPlans(ctx, 1); err != nil || len(recent) != 1 || !strings.HasPrefix(recent[0].ID, "plan-") {
|
||||
t.Fatalf("RecentPlans with a parameter: %v %+v", err, recent)
|
||||
}
|
||||
}
|
||||
@@ -1,765 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
)
|
||||
|
||||
// A Plan is what a merge produces (novox/hq ADR 0162): the modules it changed and everything
|
||||
// standing on them, sorted into tiers, each module's state, and the tier the plan is at. Kept in
|
||||
// the store so a controller replaced mid-plan resumes it, and so `status` can say what a merge
|
||||
// still waits for.
|
||||
type Plan struct {
|
||||
ID string `json:"id"`
|
||||
Repository string `json:"repository"`
|
||||
// Branch is the branch the merge went into (novox/hq issue 254): a newer plan supersedes the open
|
||||
// ones of the same repository and branch. Empty for a plan from before it was kept.
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Commit string `json:"commit"`
|
||||
// Merged is when the forge made the merge this plan answers (novox/hq issue 349): the order of a
|
||||
// branch's merges, which is not the order their plans were made in when one was acted on late. Zero
|
||||
// for a release, and for a plan kept before it was.
|
||||
Merged time.Time `json:"merged,omitzero"`
|
||||
Created time.Time `json:"created"`
|
||||
Updated time.Time `json:"updated"`
|
||||
State string `json:"state"`
|
||||
Tier int `json:"tier"`
|
||||
Tiers [][]string `json:"tiers"`
|
||||
Modules map[string]*PlanModule `json:"modules"`
|
||||
Note string `json:"note,omitempty"`
|
||||
// Away is the machines the walk passed because they were out of touch as a tier was sent (novox/hq ADR 0435 §1,
|
||||
// issue 1161): not sent to and not waited for. Their want of the walk's builds stays for the send they get when
|
||||
// they are heard again (ADR 0360 §5). Sorted.
|
||||
Away []string `json:"away,omitempty"`
|
||||
// TierEntered is when the plan entered the tier it is at (novox/hq to-be 45 Phase 0), read and
|
||||
// never written from here: a save measures the tier it leaves and stamps the next. What the
|
||||
// watchdog of a plan's progress (S3) reads.
|
||||
TierEntered time.Time `json:"tier_entered,omitempty"`
|
||||
// Revision is the plan's as it was read, and the one a save must find (novox/hq to-be 45 §6): a
|
||||
// plan is written by compare-and-set, so a write against a plan another writer moved since is
|
||||
// refused rather than laid over it. Zero is a plan never saved. SavePlan moves it.
|
||||
Revision int64 `json:"revision"`
|
||||
// Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none.
|
||||
Epoch uint64 `json:"epoch,omitempty"`
|
||||
// Release is set on a release plan (novox/hq ADR 0236): not a merge's, but the builds waiting for a
|
||||
// gate, walked through the machines one at a time.
|
||||
Release *PlanRelease `json:"release,omitempty"`
|
||||
// Delivery is set on a walk that waits for its delivery's word (novox/hq ADR 0239), and on every walk and
|
||||
// batch that names the merges it answers (ADR 0276): Awaits empty for a walk on the controller's own path,
|
||||
// which starts when it is cut.
|
||||
Delivery *PlanDelivery `json:"delivery,omitempty"`
|
||||
// Commits is every repository's commit the walk carries, one per repository (novox/hq ADR 0276): a batch's
|
||||
// walk can carry several. Repository and Commit above keep one of them, for a reader that knows one. Empty
|
||||
// for a walk kept before it was: Repository and Commit are then the whole of it.
|
||||
Commits []PlanCommit `json:"commits,omitempty"`
|
||||
// Times are the moments of a walk no other field keeps (novox/hq ADR 0282 decision 6): when its batch's
|
||||
// window closed, when it was cut, and its class. Nil for a walk kept before they were, whose phases before
|
||||
// its build are said unknown.
|
||||
Times *PlanTimes `json:"times,omitempty"`
|
||||
// Phases is the walk's time from its merge, phase by phase (ADR 0282 decision 6): never kept, worked out
|
||||
// from the walk's own moments by whoever says the walk (`delivery walks`, `plan-moved`).
|
||||
Phases *WalkPhases `json:"phases,omitempty"`
|
||||
// Backups is the backups the walk takes before it asks its first tier (novox/hq ADR 0404 decisions 2 and 3):
|
||||
// per machine and module, what was asked and how it went, and the machines with no backup holder. Nil for a
|
||||
// walk that moves no module with an item protected by a backup, and for a walk kept before it was.
|
||||
Backups *PlanBackups `json:"backups,omitempty"`
|
||||
// Links are what the rollout names of the chain (novox/hq ADR 0408 §1): the builds it carried, the machine steps it has
|
||||
// taken and the assignments it changed, each with the build before and after. Kept in the rollout's record beside
|
||||
// the fields above; empty in the controller's store.
|
||||
chain.Links
|
||||
// Held are the merges this plan or batch holds (novox/hq ADR 0276, ADR 0408): what the store kept as batched merges
|
||||
// with this plan's id, and the merges no plan answers that it held last. **Written by the merge operations only**: a
|
||||
// save of the plan keeps what the record holds, never what this copy read, so a copy read before a merge moved cannot
|
||||
// lay it back (issue 1092).
|
||||
Held []chain.HeldMerge `json:"held,omitempty"`
|
||||
// WalkSaid is, on the save that started or ended the walk, which of the two it was (novox/hq ADR 0308
|
||||
// decision 1): set by the save, never kept, read by whoever says the save (`plan-moved`). Empty on every
|
||||
// other save.
|
||||
WalkSaid string `json:"-"`
|
||||
}
|
||||
|
||||
// The states of a plan's backups, as a whole and each.
|
||||
const (
|
||||
BackupPending = "pending"
|
||||
BackupDone = "done"
|
||||
BackupFailed = "failed"
|
||||
)
|
||||
|
||||
// PlanBackups is a walk's record of the backups it takes before it asks anything (novox/hq ADR 0404): read by
|
||||
// mesh-delivery from `plan-moved`, so its JSON field names are a contract between the two repositories.
|
||||
type PlanBackups struct {
|
||||
// State is the whole: pending while any backup is asked or unasked and none failed, failed as soon as one
|
||||
// failed, done when every one is.
|
||||
State string `json:"state"`
|
||||
// Asked is one entry per machine and module, sorted by machine then module.
|
||||
Asked []PlanBackup `json:"asked"`
|
||||
// NoHolder are the machines that run a module with an item protected by a backup and hold no node-backup:
|
||||
// nothing is asked of them, and the plan says so.
|
||||
NoHolder []string `json:"no_holder,omitempty"`
|
||||
}
|
||||
|
||||
// PlanBackup is one backup asked of one machine's backup holder for one module.
|
||||
type PlanBackup struct {
|
||||
Machine string `json:"machine"`
|
||||
Module string `json:"module"`
|
||||
// State is pending, done or failed.
|
||||
State string `json:"state"`
|
||||
// AskedAt is when the holder was asked; nil while it is not asked yet.
|
||||
AskedAt *time.Time `json:"asked_at,omitempty"`
|
||||
// TriedAt is when the controller first tried to ask, kept while the holder cannot be reached: the bound of a
|
||||
// backup that is never asked counts from it. Nil once asked.
|
||||
TriedAt *time.Time `json:"tried_at,omitempty"`
|
||||
// Words are the backup's own: the holder's error for a failed one, "did not answer in ten minutes" for a silent one.
|
||||
Words string `json:"words,omitempty"`
|
||||
}
|
||||
|
||||
// PlanTimes are a walk's own moments beside its modules' (novox/hq ADR 0282 decision 6).
|
||||
type PlanTimes struct {
|
||||
// WindowClosed is when its batch's merge window closed: no merge for the window's length, or its maximum.
|
||||
// Nil for a walk no window assembled (one merge walked alone).
|
||||
WindowClosed *time.Time `json:"window_closed,omitempty"`
|
||||
// Cut is when the batch became the walk.
|
||||
Cut *time.Time `json:"cut,omitempty"`
|
||||
// Class is the walk's module class, read at its cut: core when it moves a module on the controller's own
|
||||
// path or the mesh's resolver, leaf otherwise (ADR 0282 decision 1).
|
||||
Class string `json:"class,omitempty"`
|
||||
// Started is when the walk was said started, and Ended when it was said ended (novox/hq ADR 0308 decision
|
||||
// 1): each said once, on the save that marked it.
|
||||
Started *time.Time `json:"started,omitempty"`
|
||||
Ended *time.Time `json:"ended,omitempty"`
|
||||
}
|
||||
|
||||
// PlanCommit is one repository's commit a walk carries: the latest merge of its branch in the batch.
|
||||
type PlanCommit struct {
|
||||
Repository string `json:"repository"`
|
||||
Branch string `json:"branch,omitempty"`
|
||||
Commit string `json:"commit"`
|
||||
Merged time.Time `json:"merged,omitzero"`
|
||||
}
|
||||
|
||||
// PlanMerge is one merge a walk or batch answers (novox/hq ADR 0276), as mesh-delivery reads it: Carried is
|
||||
// the commit of its repository the walk builds and sends when that is a later merge containing it, and
|
||||
// empty when it is the merge itself.
|
||||
type PlanMerge struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
Carried string `json:"carried,omitempty"`
|
||||
// Number and Title are the merge's pull request as the forge announced it, and Moves the modules the merge
|
||||
// moved when it was heard: what `plans` names a walk by. Empty where the announcement said none.
|
||||
Number int `json:"number,omitempty"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Moves []string `json:"moves,omitempty"`
|
||||
// Head is the pull request's head commit the forge merged (novox/hq issue 703): the walk takes only a build a
|
||||
// check of that head made. Empty where the announcement said none, and then it takes none.
|
||||
Head string `json:"head,omitempty"`
|
||||
// Merged is when the forge made the merge and Heard when the controller heard it (novox/hq ADR 0282): where
|
||||
// its delivery time starts. Zero for a merge named before they were kept.
|
||||
Merged time.Time `json:"merged,omitzero"`
|
||||
Heard time.Time `json:"heard,omitzero"`
|
||||
}
|
||||
|
||||
// PlanBatch is a batch's window while it is one (novox/hq ADR 0276): when it closes unless another merge
|
||||
// comes, when it closes at the latest, and the walk it waits behind once closed.
|
||||
type PlanBatch struct {
|
||||
ClosesAt time.Time `json:"closes_at"`
|
||||
AtMost time.Time `json:"at_most"`
|
||||
Behind string `json:"behind,omitempty"`
|
||||
// BehindWhy says why the batch waits behind that walk: what the two share (issue 973).
|
||||
BehindWhy string `json:"behind_why,omitempty"`
|
||||
// Own says the batch holds merges on the controller's own path, whose walk waits for nobody's word: such
|
||||
// a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the
|
||||
// build, 2026-10-10).
|
||||
Own bool `json:"own,omitempty"`
|
||||
// Closed says the window was closed by a stop that walks a stopped walk's merges again in this batch: they
|
||||
// were heard before, and an older commit never waits behind a newer one (novox/hq ADR 0299). Withheld are
|
||||
// the modules a stopped merge moves: built at the branch, which holds that merge, they would deliver it, so
|
||||
// the batch's walk does not build them.
|
||||
Closed bool `json:"closed,omitempty"`
|
||||
Withheld []string `json:"withheld,omitempty"`
|
||||
// CutAtOnce says the batch's window closed when it was made: its merge shares nothing with a batch
|
||||
// assembling and has no delivery group member still to merge, so it waits for no window (novox/hq ADR 0304
|
||||
// decision 1, ADR 0306 decisions 2 and 3). A merge that joins it later opens its window again. It is not
|
||||
// Closed, which is a stop's.
|
||||
CutAtOnce bool `json:"cut_at_once,omitempty"`
|
||||
// Waits says why a batch made for one merge waits for the window: its delivery group has a member still to
|
||||
// merge, or mesh-delivery could not say whether it has (novox/hq ADR 0306 decision 2).
|
||||
Waits string `json:"waits,omitempty"`
|
||||
// Holds are the groups whose last member is still to merge, by head branch name, each with why in words
|
||||
// (novox/hq ADR 0418 clause 3): the batch does not close by the merge window while one is held, however long the
|
||||
// member takes. Cleared when every member is merged or closed; the window then runs again from that moment.
|
||||
Holds map[string]string `json:"holds,omitempty"`
|
||||
// HeldAt is when the last hold was cleared: the maximum is counted from it, not from the batch's making.
|
||||
HeldAt time.Time `json:"held_at,omitempty"`
|
||||
}
|
||||
|
||||
// OwnPath says the record is a batch of merges on the controller's own path.
|
||||
func (p Plan) OwnPath() bool {
|
||||
return p.Delivery != nil && p.Delivery.Batch != nil && p.Delivery.Batch.Own
|
||||
}
|
||||
|
||||
// CommitOf is the commit of a repository the walk carries; empty when it carries none of it.
|
||||
func (p Plan) CommitOf(repository string) string {
|
||||
for _, c := range p.Commits {
|
||||
if strings.EqualFold(c.Repository, repository) {
|
||||
return c.Commit
|
||||
}
|
||||
}
|
||||
if strings.EqualFold(p.Repository, repository) {
|
||||
return p.Commit
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// CommitOn is the commit of a repository's branch the walk carries; empty when it carries none of it. A
|
||||
// commit kept without its branch matches any branch.
|
||||
func (p Plan) CommitOn(repository, branch string) string {
|
||||
for _, c := range p.Commits {
|
||||
if strings.EqualFold(c.Repository, repository) && (branch == "" || c.Branch == "" || c.Branch == branch) {
|
||||
return c.Commit
|
||||
}
|
||||
}
|
||||
if len(p.Commits) == 0 && strings.EqualFold(p.Repository, repository) &&
|
||||
(branch == "" || p.Branch == "" || p.Branch == branch) {
|
||||
return p.Commit
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Carried is every repository's commit the walk carries: Commits, or its one repository and commit.
|
||||
func (p Plan) Carried() []PlanCommit {
|
||||
if len(p.Commits) > 0 {
|
||||
return p.Commits
|
||||
}
|
||||
if p.Repository == "" && p.Commit == "" {
|
||||
return nil
|
||||
}
|
||||
return []PlanCommit{{Repository: p.Repository, Branch: p.Branch, Commit: p.Commit, Merged: p.Merged}}
|
||||
}
|
||||
|
||||
// Named is the walk as a person reads it: each repository at its commit.
|
||||
func (p Plan) Named() string {
|
||||
var out []string
|
||||
for _, c := range p.Carried() {
|
||||
out = append(out, c.Repository+" "+short(c.Commit))
|
||||
}
|
||||
return strings.Join(out, ", ")
|
||||
}
|
||||
|
||||
// Batch says the record is a batch still being assembled or waiting to be cut, not a walk (ADR 0276).
|
||||
func (p Plan) Batch() bool { return p.State == PlanAssembling || p.State == PlanQueued }
|
||||
|
||||
// PlanDelivery is what a walk waits for and what came of the wait (novox/hq ADR 0239).
|
||||
type PlanDelivery struct {
|
||||
// Order are the edges the plan's own order was drawn with (novox/hq ADR 0419): verb before caller, version skew,
|
||||
// engine before controller. Kept on the plan because the catalogue's graph does not hold them, and the plan's gates
|
||||
// (a later tier waits for the earlier to be applied) are read from them.
|
||||
Order []Edge `json:"order,omitempty"`
|
||||
// Awaits is who must say the walk may start: the seat whose holder owns the delivery.
|
||||
Awaits string `json:"awaits"`
|
||||
// Go is when it was let go, By by whom (the seat's holder, or a person's name) and Why.
|
||||
Go *time.Time `json:"go,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// Stopped is who ended the walk, through the delivery's owner or by hand (`plans stop|close`), and StoppedWhy
|
||||
// why: the walk is failed, and the delivery reads it as stopped, not as a build that failed.
|
||||
Stopped string `json:"stopped,omitempty"`
|
||||
StoppedWhy string `json:"stopped_why,omitempty"`
|
||||
// Merges is every merge the walk or batch answers (novox/hq ADR 0276): its own commits, and the earlier
|
||||
// merges of a repository its later commit contains.
|
||||
Merges []PlanMerge `json:"merges,omitempty"`
|
||||
// TakenOverBy names the walk a walk folded before it started was taken over by: its merges are that
|
||||
// walk's now.
|
||||
TakenOverBy string `json:"taken_over_by,omitempty"`
|
||||
// WalkedAgainBy names, on a walk stopped without some of its merges, the batch that walks its other merges
|
||||
// again, and LeftOut the merges it left out (novox/hq issue 459, ADR 0299): a delivery of a merge left out
|
||||
// ends stopped with the walk, one of a merge walked again follows that batch's walk.
|
||||
WalkedAgainBy string `json:"walked_again_by,omitempty"`
|
||||
LeftOut []PlanLeftOut `json:"left_out,omitempty"`
|
||||
// Kept are the modules the stopped walk already sent and nothing walks again: the machines named keep running
|
||||
// the stopped walk's build (novox/hq ADR 0299).
|
||||
Kept []PlanKept `json:"kept,omitempty"`
|
||||
// Withheld are, on a walk cut from a batch a stop walked merges again in, the modules it did not build: a
|
||||
// stopped merge moves them, and built only as dependents at the branch they would deliver it (ADR 0299).
|
||||
Withheld []string `json:"withheld,omitempty"`
|
||||
// Unchanged are the modules a merge it answers changed files in the directory of, whose recorded build source
|
||||
// holds none of them, each with why (novox/hq issue 611, ADR 0267 §5): not built, because the build the mesh
|
||||
// holds is their build. A delivery plan that read them whole named them; this says they matched what runs,
|
||||
// rather than that the walk left them unbuilt. Modules keeps meaning what the walk builds.
|
||||
Unchanged map[string]string `json:"unchanged,omitempty"`
|
||||
// Batch is the window of a batch; nil once it is cut into a walk.
|
||||
Batch *PlanBatch `json:"batch,omitempty"`
|
||||
// Alone says the walk walks one merge on its own commit, after a failed walk carried it in a later one:
|
||||
// built at that commit, not at the branch.
|
||||
Alone bool `json:"alone,omitempty"`
|
||||
}
|
||||
|
||||
// PlanLeftOut is a merge a stop left out of the batch that walks a stopped walk's merges again (novox/hq ADR
|
||||
// 0299): a merge stopped, or a later merge of its repository's branch, which contains the stopped one and so
|
||||
// cannot be walked without delivering it.
|
||||
type PlanLeftOut struct {
|
||||
Repository string `json:"repository"`
|
||||
Commit string `json:"commit"`
|
||||
// Contains is the stopped merge a later merge contains, or whose modules it shares; empty for a merge
|
||||
// stopped itself.
|
||||
Contains string `json:"contains,omitempty"`
|
||||
// Shares are the modules it moves that a stopped merge moves too: built at the branch, which holds the stopped
|
||||
// merge, they would deliver it.
|
||||
Shares []string `json:"shares,omitempty"`
|
||||
}
|
||||
|
||||
// PlanKept is a module a stopped walk sent and nothing walks again, and the machines it reached.
|
||||
type PlanKept struct {
|
||||
Module string `json:"module"`
|
||||
Machines []string `json:"machines"`
|
||||
}
|
||||
|
||||
// Waiting is whether the walk waits for its delivery's word.
|
||||
func (p Plan) Waiting() bool {
|
||||
return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil
|
||||
}
|
||||
|
||||
// PlanSaved is told every plan this process kept, after it is kept, with the inventory that kept it (novox/hq
|
||||
// ADR 0239): the serving controller says it on the bus as `plan-moved`, and so does a command (the controller's
|
||||
// run installs it for every verb, novox/hq ADR 0308 decision 1), so the save that starts or ends a walk is said
|
||||
// whichever process made it. Nil says nothing; whoever follows a walk also asks for it, and finds an unsaid save
|
||||
// by comparison.
|
||||
var PlanSaved func(*Inventory, Plan)
|
||||
|
||||
// ErrPlanMoved is a save against a plan written by somebody else since it was read.
|
||||
var ErrPlanMoved = errors.New("the plan was written by somebody else since it was read")
|
||||
|
||||
// PlanModule is one module's state within a plan.
|
||||
type PlanModule struct {
|
||||
// State: asked, built, failed; empty for a module whose tier has not been asked yet.
|
||||
State string `json:"state,omitempty"`
|
||||
AskedAt *time.Time `json:"asked_at,omitempty"`
|
||||
BuiltAt *time.Time `json:"built_at,omitempty"`
|
||||
// BusWaitFrom is when the bus first did not answer an ask of this module (issue 1334): the module stays unasked and is
|
||||
// asked again at the next look, until the bus has been silent for the bound, when the plan fails as for any refusal.
|
||||
BusWaitFrom *time.Time `json:"bus_wait_from,omitempty"`
|
||||
// SentAt is when the plan sent the machines running this module its new build, because a
|
||||
// later tier is built by it (ADR 0163's gate): the reports that open the gate are the ones
|
||||
// after this.
|
||||
SentAt *time.Time `json:"sent_at,omitempty"`
|
||||
// First is the machines the plan sent the new build to first, and FirstAt when (novox/hq issue
|
||||
// 249, ADR 0218): unless the module's policy rolls it out together, one machine takes it before
|
||||
// the rest, and the rest are sent once that one reports it applied. Kept so a controller
|
||||
// replaced while the plan waits on that report resumes the wait rather than sending again. The
|
||||
// machine holding the bus is among them when its user list had to go first.
|
||||
First []string `json:"first,omitempty"`
|
||||
FirstAt *time.Time `json:"first_at,omitempty"`
|
||||
Commit string `json:"commit,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
// Placed is the edge that put the module after another, and the rule that drew it (novox/hq ADR 0419 clause 5):
|
||||
// "after <module> (verb before caller: <seat>.<verb>)". Empty for a module placed first by nothing.
|
||||
Placed string `json:"placed,omitempty"`
|
||||
// Made is what the walk's build of it did (novox/hq ADR 0350 §4): "reused from <check build>", naming the pull
|
||||
// request's head, the trunk commit and the digests, or "built: <why>". Said in `plan-moved` and kept here.
|
||||
Made string `json:"made,omitempty"`
|
||||
// Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan
|
||||
// matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt
|
||||
// its module's name included. Empty in a plan from before it was kept, which is matched by
|
||||
// module, or by repository and path, as before.
|
||||
Build string `json:"build,omitempty"`
|
||||
// Previous is the build the first machine ran of this module before the plan sent it the new one —
|
||||
// the commit its last send carried (ADR 0221) — kept at the first send: what a rollback puts back
|
||||
// (novox/hq ADR 0236). Empty when the machine had never been sent the module, or what it was sent
|
||||
// is not known.
|
||||
Previous string `json:"previous,omitempty"`
|
||||
// Gate is the new build's judging on its first machine (novox/hq ADR 0236, to-be 45 §8), kept so a
|
||||
// controller replaced mid-judging resumes it, and read back through `plans` as the rollout's record.
|
||||
Gate *PlanGate `json:"gate,omitempty"`
|
||||
// GatedBy names the module of the same tier whose gate judges this one on its first machine: they
|
||||
// went there in one send (novox/hq issue 281), and one gate judges what one send moved. Empty for
|
||||
// the module the gate is kept on, and for a plan from before tiers were sent whole.
|
||||
GatedBy string `json:"gated_by,omitempty"`
|
||||
// Rest is, per machine of the rest, the declaration the send after the first-node gate carried there (novox/hq ADR
|
||||
// 0282 decision 6): the machine's first report of it, applied, is when the build runs there.
|
||||
Rest map[string]SentDeclaration `json:"rest,omitempty"`
|
||||
// Recorded is, for a module whose policy records, per machine running it with a move, which way the walk took it
|
||||
// (novox/hq ADR 0377 §6): "went out: …" when the move recreates none of its containers and moves none of its
|
||||
// data, or "waits: …" with the recreate words, the data change named, the person's or the bus's hold, or why it
|
||||
// could not be judged.
|
||||
Recorded map[string]string `json:"recorded,omitempty"`
|
||||
// Key says the walk takes this module from the builder's done key and asked no build (novox/hq ADR 0501, ADR 0453
|
||||
// §5; issue 1851): from R4e every module but the builder's own. Its outcome is the key's, never a build-agent outcome.
|
||||
Key bool `json:"key,omitempty"`
|
||||
// KeyAt is the commit of its repository whose indexer's trunk value names the fingerprint taken (ADR 0501 §2); empty
|
||||
// for a module of a repository the walk carries no commit of (a dependent through a base), taken at the value now.
|
||||
KeyAt string `json:"key_at,omitempty"`
|
||||
// Fingerprint is the fingerprint of the new form (`src2:`) the walk takes the module at, read from that trunk value;
|
||||
// empty until the indexer has written it.
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
}
|
||||
|
||||
// PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine,
|
||||
// from and to which build, the verdict, how long it took to reach it, and whether it was rolled back.
|
||||
type PlanGate struct {
|
||||
// Component is the core component the module is — mesh-controller, mesh-host, node-tools — or empty
|
||||
// for any other module, judged by its own health.
|
||||
Component string `json:"component,omitempty"`
|
||||
Machines []string `json:"machines"`
|
||||
From string `json:"from,omitempty"`
|
||||
To string `json:"to,omitempty"`
|
||||
// Since is when the judging began: the first machine reported the new build applied.
|
||||
Since *time.Time `json:"since,omitempty"`
|
||||
// Sent is, per machine, the declaration the gate's send carried there (novox/hq issue 352): what a
|
||||
// machine's report is held against. Absent on a gate kept before it was, which reads the report
|
||||
// against the send made last, as before.
|
||||
Sent map[string]SentDeclaration `json:"sent,omitempty"`
|
||||
// Applied is, per machine, the time of the first report the gate read of its send applied: the
|
||||
// node-engine started the build before it, so a witness window counted from it ends no earlier than the
|
||||
// node-engine's own (novox/hq ADR 0351 decisions 1 and 4).
|
||||
Applied map[string]time.Time `json:"applied,omitempty"`
|
||||
// Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that
|
||||
// does not resets them.
|
||||
Passes int `json:"passes,omitempty"`
|
||||
// Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, or fewer at two
|
||||
// minutes when healthy throughout, one for any other module (novox/hq ADR 0351 and ADR 0352, changing ADR
|
||||
// 0282 decision 4 and ADR 0304 decision 3). Zero in a gate
|
||||
// judged before it was said: three.
|
||||
Wants int `json:"wants,omitempty"`
|
||||
LastPass *time.Time `json:"last_pass,omitempty"`
|
||||
// Faulted says a judging found the send broken, or not healthy after a healthy reading: a module under the three-reading
|
||||
// rule then passes on its readings alone, never at its two minutes (novox/hq ADR 0352).
|
||||
Faulted bool `json:"faulted,omitempty"`
|
||||
// HealthySeen says a judging found the send healthy, counted or not: one taken before a module's floor
|
||||
// counts too, so a dip after it is a fault (novox/hq ADR 0352 decision 3).
|
||||
HealthySeen bool `json:"healthy_seen,omitempty"`
|
||||
// Last is what the newest judging found wanting, while it still may pass.
|
||||
Last string `json:"last,omitempty"`
|
||||
// Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since.
|
||||
Verdict string `json:"verdict,omitempty"`
|
||||
Why string `json:"why,omitempty"`
|
||||
JudgedAt *time.Time `json:"judged_at,omitempty"`
|
||||
Took string `json:"took,omitempty"`
|
||||
// Rollback is how a failed build was put back: rolled-back, or not-rolled-back with why.
|
||||
Rollback string `json:"rollback,omitempty"`
|
||||
// Kept says a passing verdict was written to the gate's records.
|
||||
Kept bool `json:"kept,omitempty"`
|
||||
// Carried is every module whose build moved on the judged machines with the send — the plan's own
|
||||
// module and whatever else was waiting there for a gate (novox/hq ADR 0236): each is judged here, a
|
||||
// pass is its verdict too, and one that fails is put back.
|
||||
Carried []CarriedMove `json:"carried,omitempty"`
|
||||
// Failing names the modules the last judging found wanting.
|
||||
Failing []string `json:"failing,omitempty"`
|
||||
// Healthy counts, per module, the consecutive judgings that found it healthy on every machine judged,
|
||||
// or waiting for a person (novox/hq ADR 0254): each module's passes, apart from the send's.
|
||||
Healthy map[string]int `json:"healthy,omitempty"`
|
||||
// Waits is, per module, the wait for a person the newest judging read (ADR 0254): carried along, never
|
||||
// a reason to fail.
|
||||
Waits map[string]string `json:"waits,omitempty"`
|
||||
// Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its
|
||||
// pass, and is not put back.
|
||||
Passing []string `json:"passing,omitempty"`
|
||||
// HealthyAt is, per module, when its newest healthy judging was counted: a pass is counted only gateEvery
|
||||
// after the one before (issue 318 review).
|
||||
HealthyAt map[string]time.Time `json:"healthy_at,omitempty"`
|
||||
// Broken names the modules a judging found broken, kept for the rest of the judging, and BrokenWhy the
|
||||
// first reason: the send fails, and the modules beside them are judged to their own verdict first.
|
||||
Broken []string `json:"broken,omitempty"`
|
||||
BrokenWhy string `json:"broken_why,omitempty"`
|
||||
// Returned names the broken modules already put back, at once, while the rest of the send is judged.
|
||||
Returned []string `json:"returned,omitempty"`
|
||||
// Readings are the judging's readings with their times (novox/hq ADR 0282 decision 6): every reading that
|
||||
// counted a pass, and the first that did not after one that did. At most maxReadings, the newest kept.
|
||||
Readings []GateReading `json:"readings,omitempty"`
|
||||
// Grants is the grants step taken before this gate's send (novox/hq issue 490): the bus's user list sent
|
||||
// alone to the machine holding the bus, every build there kept, so what the send newly grants is on the
|
||||
// bus before the gate judges it. Nil when the list did not change, or the gate's machine holds the bus.
|
||||
Grants *GrantsStep `json:"grants,omitempty"`
|
||||
// Busy is the wait for a busy container on a judged machine (novox/hq ADR 0335 §4): while the machine
|
||||
// reports its module's container left running because it said busy, the judging waits and has not
|
||||
// started; once it reports the container replaced, Since is moved to then. Nil when nothing waited.
|
||||
Busy *GateBusy `json:"busy,omitempty"`
|
||||
// Pending says the send this gate judges is kept and not yet made (novox/hq ADR 0353 §8): a controller that
|
||||
// takes over reads the machine's last send before making it, and never sends it twice.
|
||||
Pending bool `json:"pending,omitempty"`
|
||||
// Silent says the machine reported nothing on its send within ApplySilentAfter (novox/hq ADR 0353 §4): left,
|
||||
// neither failed nor put back.
|
||||
Silent bool `json:"silent,omitempty"`
|
||||
// TookOver names the gate whose readings this one took over (novox/hq ADR 0353 §7): another walk's judging of
|
||||
// the same build on the same machine, which this gate goes on from instead of starting from zero.
|
||||
TookOver string `json:"took_over,omitempty"`
|
||||
}
|
||||
|
||||
// TakeReadings makes this gate go on from another's readings of the same build on the same machine (novox/hq ADR
|
||||
// 0353 §7): when the judging began, when each machine applied, the passes counted and when, and what each found.
|
||||
// A module's own count is taken only for the modules this gate judges.
|
||||
func (g *PlanGate) TakeReadings(from *PlanGate, modules []string, id string) {
|
||||
if from == nil {
|
||||
return
|
||||
}
|
||||
if from.Since != nil {
|
||||
since := *from.Since
|
||||
g.Since = &since
|
||||
}
|
||||
for n, s := range from.Sent {
|
||||
if slices.Contains(g.Machines, n) {
|
||||
if g.Sent == nil {
|
||||
g.Sent = map[string]SentDeclaration{}
|
||||
}
|
||||
g.Sent[n] = s
|
||||
}
|
||||
}
|
||||
for n, at := range from.Applied {
|
||||
if slices.Contains(g.Machines, n) {
|
||||
if g.Applied == nil {
|
||||
g.Applied = map[string]time.Time{}
|
||||
}
|
||||
g.Applied[n] = at
|
||||
}
|
||||
}
|
||||
g.Passes, g.Faulted, g.HealthySeen, g.Last = from.Passes, from.Faulted, from.HealthySeen, from.Last
|
||||
if from.LastPass != nil {
|
||||
at := *from.LastPass
|
||||
g.LastPass = &at
|
||||
}
|
||||
for _, m := range modules {
|
||||
if n, counted := from.Healthy[m]; counted {
|
||||
if g.Healthy == nil {
|
||||
g.Healthy = map[string]int{}
|
||||
}
|
||||
g.Healthy[m] = n
|
||||
}
|
||||
if at, counted := from.HealthyAt[m]; counted {
|
||||
if g.HealthyAt == nil {
|
||||
g.HealthyAt = map[string]time.Time{}
|
||||
}
|
||||
g.HealthyAt[m] = at
|
||||
}
|
||||
}
|
||||
g.Readings = append([]GateReading(nil), from.Readings...)
|
||||
g.TookOver = id
|
||||
}
|
||||
|
||||
// GateBusy is a first-node gate's wait for a busy container (novox/hq ADR 0335 §4).
|
||||
type GateBusy struct {
|
||||
Machine string `json:"machine"`
|
||||
Module string `json:"module"`
|
||||
Container string `json:"container"`
|
||||
// Since is when the machine's first apply that would have replaced it ran, Until when its node-engine
|
||||
// replaces it at the latest; Ended when the machine reported it replaced and the judging began — or when
|
||||
// the controller stopped waiting past the engine's bound, which PastBound says.
|
||||
Since time.Time `json:"since"`
|
||||
Until time.Time `json:"until"`
|
||||
// Seen is when this controller first read the wait, on its own clock: the gate's bound runs from it, never
|
||||
// from the engine's word alone.
|
||||
Seen time.Time `json:"seen,omitzero"`
|
||||
Ended *time.Time `json:"ended,omitempty"`
|
||||
PastBound bool `json:"past_bound,omitempty"`
|
||||
// Said is the last output of its busy command, as the machine reported it.
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// Waiting says the gate waits for a busy container and has not started.
|
||||
func (g *PlanGate) Waiting() bool { return g != nil && g.Busy != nil && g.Busy.Ended == nil }
|
||||
|
||||
// Words is the wait in a line, as the walk says it.
|
||||
func (b *GateBusy) Words() string {
|
||||
said := ""
|
||||
if b.Said != "" {
|
||||
said = "; its busy command says: " + b.Said
|
||||
}
|
||||
return fmt.Sprintf("waits for a busy container on %s: %s holds work, and is replaced once it says idle, by %s "+
|
||||
"at the latest (novox/hq ADR 0335)%s", b.Machine, b.Container, b.Until.UTC().Format(time.RFC3339), said)
|
||||
}
|
||||
|
||||
// GrantsStep is the bus's user list sent to the machine holding the bus ahead of a gated send (novox/hq
|
||||
// issue 490): to which machine, when, and, when it could not be sent, why — the send went on without it.
|
||||
type GrantsStep struct {
|
||||
Node string `json:"node"`
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
Failed string `json:"failed,omitempty"`
|
||||
}
|
||||
|
||||
// GateReading is one reading of a first-node gate.
|
||||
type GateReading struct {
|
||||
At time.Time `json:"at"`
|
||||
Healthy bool `json:"healthy"`
|
||||
// Said is what a reading that did not pass found wanting.
|
||||
Said string `json:"said,omitempty"`
|
||||
}
|
||||
|
||||
// maxReadings bounds a first-node gate's readings: a judging that never passes reads every few seconds for ten minutes.
|
||||
const maxReadings = 24
|
||||
|
||||
// Read keeps one reading: a pass always, and a reading that did not pass only when the one before passed or
|
||||
// there is none, so a judging waiting for a module to start keeps one line of it, not hundreds.
|
||||
func (g *PlanGate) Read(at time.Time, healthy bool, said string) {
|
||||
if !healthy && len(g.Readings) > 0 && !g.Readings[len(g.Readings)-1].Healthy {
|
||||
return
|
||||
}
|
||||
if r := []rune(said); len(r) > 200 {
|
||||
said = string(r[:200])
|
||||
}
|
||||
g.Readings = append(g.Readings, GateReading{At: at, Healthy: healthy, Said: said})
|
||||
if len(g.Readings) > maxReadings {
|
||||
g.Readings = g.Readings[len(g.Readings)-maxReadings:]
|
||||
}
|
||||
}
|
||||
|
||||
// CarriedMove is one module's build moving on a machine with a gated send.
|
||||
type CarriedMove struct {
|
||||
Module string `json:"module"`
|
||||
Node string `json:"node"`
|
||||
From string `json:"from,omitempty"`
|
||||
To string `json:"to"`
|
||||
Build string `json:"build,omitempty"`
|
||||
// Recreates is what the send does to the module's containers, in the mesh's words — how many it
|
||||
// recreates, and whether with a new image or only their declaration (novox/hq ADR 0245); empty when
|
||||
// it recreates none, or when the build the machine ran is not known.
|
||||
Recreates string `json:"recreates,omitempty"`
|
||||
}
|
||||
|
||||
// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build
|
||||
// that waits for a gate, sent one machine at a time, each judged before the next.
|
||||
type PlanRelease struct {
|
||||
Order []string `json:"order"`
|
||||
Next int `json:"next"`
|
||||
// Gate is the first machine being judged; nil between machines.
|
||||
Gate *PlanGate `json:"gate,omitempty"`
|
||||
// Rest says the first machine passed (novox/hq ADR 0353 §1): every other machine is sent at once, each judged by
|
||||
// its own gate in Gates, by machine — a build no gate passed going to one machine of a step only (§2).
|
||||
Rest bool `json:"rest,omitempty"`
|
||||
Gates map[string]*PlanGate `json:"gates,omitempty"`
|
||||
// Failed are the machines of the rest whose gate failed, and Silent those that said nothing on their send
|
||||
// within ApplySilentAfter (ADR 0353 §3 and §4).
|
||||
Failed []string `json:"failed,omitempty"`
|
||||
Silent []string `json:"silent,omitempty"`
|
||||
Done []string `json:"done,omitempty"`
|
||||
// Skipped are the machines not heard from when their turn came, left as they were.
|
||||
Skipped []string `json:"skipped,omitempty"`
|
||||
// By is the person who released it, empty when the mesh did.
|
||||
By string `json:"by,omitempty"`
|
||||
// Settings is set on a settings walk (novox/hq ADR 0344): not the builds waiting for a gate, but a settings
|
||||
// layer the operator approved, sent to the machines it reaches one at a time, its module judged on each.
|
||||
Settings *PlanSettings `json:"settings,omitempty"`
|
||||
}
|
||||
|
||||
// PlanSettings is what a settings walk sends (novox/hq ADR 0344): the module whose layer the operator approved, the
|
||||
// layer's scope, the ask that approved it and what changed, in words.
|
||||
type PlanSettings struct {
|
||||
Module string `json:"module"`
|
||||
// Node is the machine the layer is for, empty for the whole mesh.
|
||||
Node string `json:"node,omitempty"`
|
||||
// Ask is the proposal the operator approved, and Change what its layer changed ("~ prefs").
|
||||
Ask string `json:"ask"`
|
||||
Change string `json:"change,omitempty"`
|
||||
}
|
||||
|
||||
// The states a plan passes through.
|
||||
const (
|
||||
PlanBuilding = "building"
|
||||
PlanRolling = "rolling"
|
||||
PlanDone = "done"
|
||||
PlanFailed = "failed"
|
||||
// PlanSuperseded is a plan a newer merge of the same repository and branch took over (novox/hq
|
||||
// issue 254, ADR 0218): what it had not built is in the newer plan, and its note names it.
|
||||
PlanSuperseded = "superseded"
|
||||
// PlanAssembling is a batch whose merge window is open (novox/hq ADR 0276), and PlanQueued one whose
|
||||
// window closed while a walk is open: neither is a walk yet. Cut, a batch keeps its id and is building.
|
||||
PlanAssembling = "assembling"
|
||||
PlanQueued = "queued"
|
||||
)
|
||||
|
||||
// Open says whether the plan is still being worked.
|
||||
func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRolling }
|
||||
|
||||
// SavePlan writes a plan, new or changed, whole: the plan is small and read as one thing.
|
||||
//
|
||||
// **By compare-and-set on its revision, carrying the epoch** (novox/hq to-be 45 §6): written only if
|
||||
// the plan is still at the revision it was read at — a new one only if it does not exist — and refused
|
||||
// with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records.
|
||||
// On success p's revision and epoch are the ones written, so the caller may save it again.
|
||||
func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error {
|
||||
return i.SavePlansMoving(ctx, []*Plan{p}, nil)
|
||||
}
|
||||
|
||||
// MergeMove is a merge given to the plan that answers it now (novox/hq ADR 0276).
|
||||
type MergeMove struct {
|
||||
Repository, Commit, Plan string
|
||||
Alone bool
|
||||
// Event replaces the merge's kept announcement when it is put back to waiting (Plan empty); nil leaves it.
|
||||
Event json.RawMessage
|
||||
}
|
||||
|
||||
// SavePlansMoving writes plans, as SavePlan does each, and gives merges to the plans that answer them, all in one
|
||||
// transaction: every write and move, or none (novox/hq ADR 0299 decision 7 — a stop that walks merges again never
|
||||
// leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by
|
||||
// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once
|
||||
// the transaction is committed, in order.
|
||||
func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return i.chainSavePlans(ctx, c, plans, moves)
|
||||
}
|
||||
|
||||
// short is a commit as a person reads it.
|
||||
func short(commit string) string {
|
||||
if len(commit) > 8 {
|
||||
return commit[:8]
|
||||
}
|
||||
return commit
|
||||
}
|
||||
|
||||
// OpenPlans is every plan still being worked, oldest first.
|
||||
func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated)
|
||||
}
|
||||
|
||||
// PlansSince is every plan made after a moment, and every plan still being worked, oldest first.
|
||||
func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated)
|
||||
}
|
||||
|
||||
// RecentPlans is the last few plans, newest first, open or not — what the overview shows.
|
||||
func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest)
|
||||
if err == nil && limit >= 0 && len(plans) > limit {
|
||||
plans = plans[:limit]
|
||||
}
|
||||
return plans, err
|
||||
}
|
||||
|
||||
// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh
|
||||
// planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it
|
||||
// recorded when its merge was made.
|
||||
func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return Plan{}, false, err
|
||||
}
|
||||
return i.chainNewestMergeOf(ctx, c, repository, branch)
|
||||
}
|
||||
|
||||
// PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a
|
||||
// caller of the plans verb.
|
||||
func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) {
|
||||
c, err := i.theChain(ctx)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
return i.chainPlanByID(ctx, c, id)
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// **Migration 0108 drops the tables of the delivery plans' machinery** (novox/hq ADR 0539 §17, design 47 release 6, issue 1945):
|
||||
// the record of each send, the assignment generation, the gate's verdicts and the measured durations. The store is built as
|
||||
// the mesh holds it before the migration, with a row in each, and after it the four are gone and what the controller still
|
||||
// keeps stays: the node and its columns, the module, the bus step, the core box and the refused registrations.
|
||||
func TestMigration0108DropsTheTablesOfThePlansMachineryAndKeepsWhatTheControllerStillKeeps(t *testing.T) {
|
||||
inv, migrate := ForTestBefore(t, 108)
|
||||
tables := func() map[string]bool {
|
||||
t.Helper()
|
||||
out := map[string]bool{}
|
||||
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
|
||||
out[strings.Trim(row[0], `"`)] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
gone := []string{"declaration_send", "assignment_generation", "build_gate", "duration"}
|
||||
before := tables()
|
||||
for _, name := range gone {
|
||||
if !before[name] {
|
||||
t.Fatalf("the table %s is not in the store before the migration", name)
|
||||
}
|
||||
}
|
||||
migrate()
|
||||
after := tables()
|
||||
for _, name := range gone {
|
||||
if after[name] {
|
||||
t.Errorf("the table %s is still in the store", name)
|
||||
}
|
||||
}
|
||||
for _, stays := range []string{"node", "module", "bus_step", "core_box", "build_refused", "controller_schema"} {
|
||||
if !after[stays] {
|
||||
t.Errorf("the table %s is gone", stays)
|
||||
}
|
||||
}
|
||||
// The functions and triggers the tables carried go with them.
|
||||
if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" {
|
||||
t.Errorf("the function raise_assignment_generation is still in the store")
|
||||
}
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A plan is a record the mesh keeps and resumes (novox/hq ADR 0162): written whole, read back open,
|
||||
// advanced, and gone from the open ones when done.
|
||||
func TestAPlanIsKeptAdvancedAndResumedFromTheStore(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(),
|
||||
State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}, {"shop"}},
|
||||
Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}, "shop": {}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
open, err := inv.OpenPlans(ctx)
|
||||
if err != nil || len(open) != 1 || open[0].ID != "plan-1" || len(open[0].Tiers) != 3 {
|
||||
t.Fatalf("the plan was not kept whole: %v %+v", err, open)
|
||||
}
|
||||
// Another controller picks it up where it was left: a tier advanced and a module built.
|
||||
now := time.Now().UTC()
|
||||
resumed := open[0]
|
||||
resumed.Tier = 1
|
||||
resumed.Modules["mesh-tools"].State = "built"
|
||||
resumed.Modules["mesh-tools"].BuiltAt = &now
|
||||
resumed.State = PlanRolling
|
||||
resumed.Note = "tier 0 built; waiting for builder on anchor to be applied"
|
||||
if err := inv.SavePlan(ctx, &resumed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := inv.PlanByID(ctx, "plan-1")
|
||||
if err != nil || again.Tier != 1 || again.Modules["mesh-tools"].State != "built" || again.State != PlanRolling {
|
||||
t.Fatalf("the advanced plan did not come back as left: %v %+v", err, again)
|
||||
}
|
||||
again.State = PlanDone
|
||||
if err := inv.SavePlan(ctx, &again); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open, _ = inv.OpenPlans(ctx); len(open) != 0 {
|
||||
t.Fatalf("a done plan is not open: %+v", open)
|
||||
}
|
||||
if recent, _ := inv.RecentPlans(ctx, 5); len(recent) != 1 || recent[0].State != PlanDone {
|
||||
t.Fatalf("a done plan is still among the recent ones: %+v", recent)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 254: a plan keeps the branch its merge went into, and a superseded plan is not open.
|
||||
func TestASupersededPlanIsNotOpen(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
p := Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "abc",
|
||||
Created: time.Now().UTC(), State: PlanBuilding, Tiers: [][]string{{"gitea"}},
|
||||
Modules: map[string]*PlanModule{"gitea": {}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := inv.PlanByID(ctx, "plan-1")
|
||||
if err != nil || kept.Branch != "main" {
|
||||
t.Fatalf("the branch was not kept: %v %+v", err, kept)
|
||||
}
|
||||
kept.State = PlanSuperseded
|
||||
kept.Note = "superseded at tier 0 by plan-2"
|
||||
if err := inv.SavePlan(ctx, &kept); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if open, err := inv.OpenPlans(ctx); err != nil || len(open) != 0 {
|
||||
t.Fatalf("a superseded plan is still open: %v %+v", err, open)
|
||||
}
|
||||
if recent, _ := inv.RecentPlans(ctx, 5); len(recent) != 1 || recent[0].State != PlanSuperseded {
|
||||
t.Fatalf("a superseded plan is not among the recent ones as superseded: %+v", recent)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 349 (review of the follow-up): the newest merge of a branch is the one merged last, whatever
|
||||
// order the plans were made in, in any state; a tie is broken by the plan made last; a release, another
|
||||
// branch and another repository are never it; and its time is kept to the nanosecond.
|
||||
func TestTheNewestMergeOfABranchIsTheOneMergedLast(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
t0 := time.Date(2026, 10, 9, 10, 0, 0, 0, time.UTC)
|
||||
save := func(id, repo, branch string, merged, created time.Time, state string, release bool) {
|
||||
t.Helper()
|
||||
p := Plan{ID: id, Repository: repo, Branch: branch, Commit: id + "-commit", Merged: merged, Created: created,
|
||||
State: state, Tiers: [][]string{}, Modules: map[string]*PlanModule{}}
|
||||
if release {
|
||||
p.Release = &PlanRelease{}
|
||||
}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, found, err := inv.NewestMergeOf(ctx, "novox/mesh-catalog", "main"); err != nil || found {
|
||||
t.Fatalf("a branch with no plan: %v %v", found, err)
|
||||
}
|
||||
// Made in the other order than merged: the later merge's plan made first, and done.
|
||||
save("plan-later", "novox/mesh-catalog", "main", t0.Add(2*time.Minute+250*time.Millisecond), t0, PlanDone, false)
|
||||
save("plan-earlier", "novox/mesh-catalog", "main", t0.Add(time.Minute), t0.Add(5*time.Minute), PlanRolling, false)
|
||||
save("plan-other-branch", "novox/mesh-catalog", "release", t0.Add(time.Hour), t0, PlanRolling, false)
|
||||
save("plan-other-repo", "novox/mesh-controller", "main", t0.Add(time.Hour), t0, PlanRolling, false)
|
||||
save("release-1", "novox/mesh-catalog", "main", t0.Add(time.Hour), t0, PlanRolling, true)
|
||||
save("plan-unknown", "novox/mesh-catalog", "main", time.Time{}, t0.Add(time.Hour), PlanRolling, false)
|
||||
p, found, err := inv.NewestMergeOf(ctx, "Novox/Mesh-Catalog", "main")
|
||||
if err != nil || !found || p.ID != "plan-later" {
|
||||
t.Fatalf("the newest merge: %s %v %v", p.ID, found, err)
|
||||
}
|
||||
if !p.Merged.Equal(t0.Add(2*time.Minute + 250*time.Millisecond)) {
|
||||
t.Fatalf("its merge time was not kept to the nanosecond: %s", p.Merged)
|
||||
}
|
||||
// The same merge time: the plan made last.
|
||||
save("plan-again", "novox/mesh-catalog", "main", t0.Add(2*time.Minute+250*time.Millisecond), t0.Add(time.Minute), PlanBuilding, false)
|
||||
if p, _, _ := inv.NewestMergeOf(ctx, "novox/mesh-catalog", "main"); p.ID != "plan-again" {
|
||||
t.Fatalf("one merge time, two plans: %s", p.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq issue 352: what a machine was last sent is read back by name with its sequence, a report keeps
|
||||
// the declaration it was about and that declaration's sequence, and a gate's sends are kept with the plan.
|
||||
func TestASendAndAReportAreKnownByTheirDeclaration(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.AddNode(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, found, err := inv.SentTo(ctx, "anchor"); err != nil || found {
|
||||
t.Fatalf("a machine never sent anything: %v %v", found, err)
|
||||
}
|
||||
if _, _, err := inv.SentTo(ctx, "nobody"); err == nil {
|
||||
t.Fatal("a machine that does not exist was answered")
|
||||
}
|
||||
seq, err := inv.NextSequence(ctx, record.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSent(ctx, record.ID, "d-1", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent, found, err := inv.SentTo(ctx, "anchor")
|
||||
if err != nil || !found || sent.Digest != "d-1" || sent.Sequence != seq {
|
||||
t.Fatalf("sent %+v %v %v", sent, found, err)
|
||||
}
|
||||
if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Node: "anchor", Outcome: OutcomeApplied, Declared: "d-1", Applied: 1},
|
||||
ReportOrder{Sequence: seq, ReportSequence: 1}, func(ReportOrder) bool { return false }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reports, err := inv.LastReports(ctx)
|
||||
if err != nil || len(reports) != 1 || reports[0].Declared != "d-1" || reports[0].ReportedSequence != seq || !reports[0].Current {
|
||||
t.Fatalf("reports %+v %v", reports, err)
|
||||
}
|
||||
// Sent again, unreported: the report is no longer on the last send, and is still on the first.
|
||||
if err := inv.RecordSent(ctx, record.ID, "d-2", map[string]string{"app": "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reports, _ = inv.LastReports(ctx)
|
||||
if reports[0].Current || !sent.ReportsOn(reports[0]) {
|
||||
t.Fatalf("after a newer send: %+v", reports[0])
|
||||
}
|
||||
at := time.Now().UTC()
|
||||
p := Plan{ID: "plan-352", Repository: "novox/x", Commit: "c", Created: at, State: PlanRolling, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*PlanModule{"app": {Gate: &PlanGate{Machines: []string{"anchor"}, Since: &at,
|
||||
Sent: map[string]SentDeclaration{"anchor": sent}}}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := inv.PlanByID(ctx, "plan-352")
|
||||
if err != nil || !reflect.DeepEqual(kept.Modules["app"].Gate.Sent["anchor"], sent) {
|
||||
t.Fatalf("the gate's send was not kept with the plan: %+v %v", kept.Modules["app"].Gate, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A controller build's reach of the store's schema is kept by its version, and the store's own is read.
|
||||
func TestASchemaReachIsKeptByBuild(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
applied, err := inv.SchemaApplied(ctx)
|
||||
if err != nil || applied < 87 {
|
||||
t.Fatalf("applied %d %v", applied, err)
|
||||
}
|
||||
if _, known, err := inv.SchemaReachOf(ctx, "ad62528c47c7"); err != nil || known {
|
||||
t.Fatalf("an unrecorded build: %v %v", known, err)
|
||||
}
|
||||
if err := inv.RecordSchemaReach(ctx, "", 87); err == nil {
|
||||
t.Fatal("a reach without a build was recorded")
|
||||
}
|
||||
if err := inv.RecordSchemaReach(ctx, "ad62528c47c7", 86); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSchemaReach(ctx, "ad62528c47c7", 87); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if reach, known, err := inv.SchemaReachOf(ctx, "ad62528c47c7"); err != nil || !known || reach != 87 {
|
||||
t.Fatalf("reach %d %v %v", reach, known, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A plan keeps its backups through the store, timestamps to the nanosecond (novox/hq ADR 0404, issue 1065).
|
||||
func TestAPlanKeepsItsBackupsInTheStore(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
at := time.Now().UTC()
|
||||
p := Plan{ID: "plan-backups", Repository: "novox/mesh-catalog", Commit: "abc", Created: at, State: PlanBuilding,
|
||||
Tiers: [][]string{{"mail"}}, Modules: map[string]*PlanModule{"mail": {}},
|
||||
Backups: &PlanBackups{State: BackupPending, NoHolder: []string{"bare"},
|
||||
Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupPending, AskedAt: &at}}}}
|
||||
if err := inv.SavePlan(ctx, &p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := inv.PlanByID(ctx, p.ID)
|
||||
if err != nil || got.Backups == nil || !reflect.DeepEqual(got.Backups, p.Backups) {
|
||||
t.Fatalf("kept %+v (%v), wrote %+v", got.Backups, err, p.Backups)
|
||||
}
|
||||
got.Backups.State = BackupDone
|
||||
if err := inv.SavePlan(ctx, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _ := inv.PlanByID(ctx, p.ID); again.Backups == nil || again.Backups.State != BackupDone {
|
||||
t.Fatalf("the change was not kept: %+v", again.Backups)
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,6 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -37,84 +35,3 @@ const (
|
||||
ProvFailed = "failed"
|
||||
ProvRemoved = "removed"
|
||||
)
|
||||
|
||||
// ProvLookup answers a client's newest report of a module on a machine; false when it has said nothing of it.
|
||||
type ProvLookup func(node, module string) (ProvReport, bool)
|
||||
|
||||
// OutcomeProvisioningFailed begins the outcome of a send whose module's files the provisioner failed to place.
|
||||
const OutcomeProvisioningFailed = "provisioning failed"
|
||||
|
||||
// ProvisionedSend is a machine's report of a send, with its provisioner client's reports of every module the send
|
||||
// handed to it: what a walk counts as that machine's report. engine and reported are the machine's own report and
|
||||
// whether it said one. A failed client report is the send's outcome, whatever the machine said; none, or one not
|
||||
// yet applied on the build sent, leaves the send unreported.
|
||||
func ProvisionedSend(node string, send SentDeclaration, engine AppliedReport, reported bool, prov ProvLookup) (AppliedReport, bool) {
|
||||
if len(send.Provisioned) == 0 {
|
||||
return engine, reported
|
||||
}
|
||||
// The machine's own failure, or its wait for a busy container, is its report whatever the client says.
|
||||
if reported && (engine.Outcome != OutcomeApplied || engine.Waiting) {
|
||||
return engine, true
|
||||
}
|
||||
modules := make([]string, 0, len(send.Provisioned))
|
||||
for m := range send.Provisioned {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
sort.Strings(modules)
|
||||
at, placed := engine.At, true
|
||||
for _, m := range modules {
|
||||
r, said := ProvReport{}, false
|
||||
if prov != nil {
|
||||
r, said = prov(node, m)
|
||||
}
|
||||
switch {
|
||||
case said && r.Status == ProvFailed && aboutTheSend(r, send, send.Provisioned[m]):
|
||||
return AppliedReport{At: r.At, Outcome: fmt.Sprintf("%s (%s: %s)", OutcomeProvisioningFailed, m,
|
||||
orNoReason(r.Reason))}, true
|
||||
case said && r.Status == ProvApplied && r.Templates == send.Provisioned[m]:
|
||||
if r.At.After(at) {
|
||||
at = r.At
|
||||
}
|
||||
default:
|
||||
placed = false
|
||||
}
|
||||
}
|
||||
if !reported || !placed {
|
||||
return AppliedReport{}, false
|
||||
}
|
||||
engine.At = at
|
||||
return engine, true
|
||||
}
|
||||
|
||||
// Only is the send narrowed to what it handed to the provisioner for the named modules: a delivery waits on the
|
||||
// modules it carries, never on another module the same machine's declaration hands (ADR 0362 §1-2, issue 840).
|
||||
// A send handing nothing is returned as it is.
|
||||
func (s SentDeclaration) Only(modules ...string) SentDeclaration {
|
||||
if len(s.Provisioned) == 0 {
|
||||
return s
|
||||
}
|
||||
kept := map[string]string{}
|
||||
for _, m := range modules {
|
||||
if d, ok := s.Provisioned[m]; ok {
|
||||
kept[m] = d
|
||||
}
|
||||
}
|
||||
s.Provisioned = kept
|
||||
return s
|
||||
}
|
||||
|
||||
// aboutTheSend says a client's failure is about what the send handed: on the build's templates, or, said before it
|
||||
// had any to name (its templates could not be fetched), no earlier than the send.
|
||||
func aboutTheSend(r ProvReport, send SentDeclaration, digest string) bool {
|
||||
if r.Templates != "" {
|
||||
return r.Templates == digest
|
||||
}
|
||||
return send.At.IsZero() || !r.At.Before(send.At)
|
||||
}
|
||||
|
||||
func orNoReason(s string) string {
|
||||
if s == "" {
|
||||
return "no reason given"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// A walk counts a provisioned module applied on a machine only once its files are placed (novox/hq ADR 0362, design
|
||||
// 57 P6, issue 672). The walk of issue 582 (one module, three machines of the rest), with g14's send having handed
|
||||
// dunst to the provisioner: g14's own report comes 5 s after its send, shanks's, the last of the others, 15 s
|
||||
// after its own.
|
||||
|
||||
const dunstTemplates = "sha256:" + "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd"
|
||||
|
||||
func walkHandingDunst(t *testing.T) sendRest582 {
|
||||
t.Helper()
|
||||
c := walk582(t)
|
||||
s := c.walk.Modules["build-agent"].Rest["g14"]
|
||||
s.Provisioned = map[string]string{"build-agent": dunstTemplates}
|
||||
c.walk.Modules["build-agent"].Rest["g14"] = s
|
||||
return c
|
||||
}
|
||||
|
||||
// provisioning is a walk's lookup with the machines' clients saying reports.
|
||||
func (w sendRest582) provisioning(reports map[string]ProvReport) AppliedLookup {
|
||||
return func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) {
|
||||
r, ok := w.lookup(node, send, walkSent)
|
||||
return ProvisionedSend(node, send, r, ok, func(node, module string) (ProvReport, bool) {
|
||||
p, said := reports[node+"."+module]
|
||||
return p, said
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func restOf(w *WalkPhases, node string) RestReport {
|
||||
for _, r := range w.Rest {
|
||||
if r.Node == node {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return RestReport{}
|
||||
}
|
||||
|
||||
// The node-engine's report alone does not end the walk: it waits for prov.applied; with it, the walk ends, and
|
||||
// its delivery time runs to the later of the two reports.
|
||||
func TestAWalkWaitsForTheProvisionersReportAndEndsAtTheLater(t *testing.T) {
|
||||
c := walkHandingDunst(t)
|
||||
g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At
|
||||
|
||||
w := c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(nil))
|
||||
if w.End != nil || restOf(w, "g14").State != RestWaiting ||
|
||||
!strings.Contains(phase(w, PhaseApply, -1).Said, "waiting for g14") {
|
||||
t.Fatalf("the walk did not wait for g14's prov.applied: end %v, g14 %+v, apply %+v", w.End, restOf(w, "g14"),
|
||||
phase(w, PhaseApply, -1))
|
||||
}
|
||||
|
||||
// A report on an older build of the templates is not this send's.
|
||||
older := map[string]ProvReport{"g14.build-agent": {Status: ProvApplied, Templates: "sha256:" + strings.Repeat("0", 64),
|
||||
At: g14Sent.Add(2 * time.Second)}}
|
||||
if w := c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(older)); w.End != nil {
|
||||
t.Fatalf("a report on an older build ended the walk at %v", w.End)
|
||||
}
|
||||
|
||||
placed := g14Sent.Add(50 * time.Second) // after shanks's report, 35 s after the judgement
|
||||
w = c.walk.WalkPhases(g14Sent.Add(time.Hour), c.provisioning(map[string]ProvReport{
|
||||
"g14.build-agent": {Status: ProvApplied, Templates: dunstTemplates, At: placed}}))
|
||||
sumsUp(t, w)
|
||||
if w.End == nil || !w.End.Equal(placed) {
|
||||
t.Fatalf("the walk ends at %v, want the provisioner's report at %s: %+v", w.End, placed, w.Rest)
|
||||
}
|
||||
if g := restOf(w, "g14"); g.State != RestApplied || !g.Reported.Equal(placed) {
|
||||
t.Fatalf("g14 is not applied at its later report: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// A failed prov.applied is the machine's failed apply, named in the walk's end with its reason.
|
||||
func TestAFailedProvisioningIsAFailedApply(t *testing.T) {
|
||||
c := walkHandingDunst(t)
|
||||
g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At
|
||||
w := c.walk.WalkPhases(g14Sent.Add(time.Hour), c.provisioning(map[string]ProvReport{
|
||||
"g14.build-agent": {Status: ProvFailed, Reason: "the templates cannot be fetched", At: g14Sent.Add(3 * time.Second)}}))
|
||||
if w.End != nil {
|
||||
t.Fatalf("a failed provisioning ended the walk: %v", w.End)
|
||||
}
|
||||
if g := restOf(w, "g14"); g.State != RestFailed || !strings.Contains(g.Outcome, "the templates cannot be fetched") {
|
||||
t.Fatalf("g14 is not named failed with the reason: %+v", g)
|
||||
}
|
||||
if said := phase(w, PhaseApply, -1).Said; !strings.Contains(said, "not applied on g14 (provisioning failed (build-agent: "+
|
||||
"the templates cannot be fetched))") {
|
||||
t.Fatalf("the walk's end does not name g14's failure: %q", said)
|
||||
}
|
||||
// A failure said before the send, of no build, is not this send's.
|
||||
stale := map[string]ProvReport{"g14.build-agent": {Status: ProvFailed, Reason: "old", At: g14Sent.Add(-time.Hour)}}
|
||||
if g := restOf(c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(stale)), "g14"); g.State != RestWaiting {
|
||||
t.Fatalf("a failure from before the send was read as this send's: %+v", g)
|
||||
}
|
||||
}
|
||||
|
||||
// None within ApplySilentAfter names the machine silent, never counted applied; the others end the walk.
|
||||
func TestNoProvisioningReportWithinTheBoundIsSilence(t *testing.T) {
|
||||
c := walkHandingDunst(t)
|
||||
g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At
|
||||
waiting := map[string]ProvReport{"g14.build-agent": {Status: ProvWaiting, WaitsFor: []string{"config:font-size"},
|
||||
At: g14Sent.Add(3 * time.Second)}}
|
||||
w := c.walk.WalkPhases(g14Sent.Add(ApplySilentAfter+time.Minute), c.provisioning(waiting))
|
||||
sumsUp(t, w)
|
||||
if g := restOf(w, "g14"); g.State != RestSilent {
|
||||
t.Fatalf("g14 is not silent past ApplySilentAfter: %+v", g)
|
||||
}
|
||||
if len(w.Silent) != 1 || w.Silent[0] != "g14" || w.End == nil || !w.End.Equal(c.judged.Add(35*time.Second)) {
|
||||
t.Fatalf("the walk does not end on the others, naming g14 silent: end %v, silent %v", w.End, w.Silent)
|
||||
}
|
||||
}
|
||||
|
||||
// A send that handed nothing ends on the node-engine's report alone, as before.
|
||||
func TestASendThatHandedNothingEndsOnTheEnginesReport(t *testing.T) {
|
||||
c := walk582(t)
|
||||
before := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup)
|
||||
now := c.walk.WalkPhases(c.judged.Add(time.Hour), c.provisioning(nil))
|
||||
if before.End == nil || now.End == nil || !now.End.Equal(*before.End) {
|
||||
t.Fatalf("a walk handing nothing ends differently: before %v, now %v", before.End, now.End)
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
func TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack(t *testing.T) {
|
||||
inv := fresh(t)
|
||||
ctx := t.Context()
|
||||
manifestOf := func(image string) catalogue.Manifest {
|
||||
return catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{
|
||||
{"id": "app", "type": "container", "name": "one", "image": image}}}
|
||||
}
|
||||
record := func(id, image string, asked time.Time) Build {
|
||||
t.Helper()
|
||||
raw, _ := json.Marshal(manifestOf(image))
|
||||
b := Build{ID: id, Module: "one", Commit: id + "aaaaaaaa", Repository: "novox/one", Manifest: raw,
|
||||
Asked: asked, At: asked, Made: []Artifact{{Name: "one", Kind: "image", Reference: image}}}
|
||||
if err := inv.RecordBuild(ctx, b); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, manifestOf(image), Source{Repository: "novox/one", BuiltFrom: b.Commit,
|
||||
Asked: asked}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
x := "store/one@sha256:" + "1111111111111111111111111111111111111111111111111111111111111111"
|
||||
y := "store/one@sha256:" + "2222222222222222222222222222222222222222222222222222222222222222"
|
||||
before := record("x", x, time.Now().Add(-2*time.Hour))
|
||||
failed := record("y", y, time.Now().Add(-time.Minute))
|
||||
if err := inv.RecordGate(ctx, GateVerdict{Build: failed.ID, Module: "one", Commit: failed.Commit,
|
||||
Machines: []string{"anchor"}, Verdict: GateFailed}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RestoreModule(ctx, before); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := inv.Held(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
running, err := inv.RegisteredHeld(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if running["one/one"] == y {
|
||||
t.Fatalf("what runs is read as the build whose gate failed (%s); Held says %s", y, held["one/one"])
|
||||
}
|
||||
if got := running["one/one"]; got != "" && got != x {
|
||||
t.Fatalf("what runs is read as %s, want the build put back %s or nothing", got, x)
|
||||
}
|
||||
}
|
||||
@@ -1,364 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// The assignment generation and the record of every send (novox/hq issue 234, migration 0094).
|
||||
//
|
||||
// A declaration carries the generation of the assignments it was composed from, and a machine refuses one
|
||||
// composed from an older generation than it applied — the sequence orders arrival and cannot tell a later
|
||||
// send carrying an older view. And every send is written down with who sent it, from which generation and
|
||||
// naming which modules, so what a machine was last told, by whom, is read back rather than guessed at.
|
||||
|
||||
// sendsKept is how many sends are kept per machine: the newest. Enough to read back a morning of pushes
|
||||
// on a busy machine; the record is for finding who sent what, not an archive.
|
||||
const sendsKept = 200
|
||||
|
||||
// AssignmentGeneration is the mesh's assignment generation now: raised by the store itself in the same
|
||||
// transaction as every change to what is assigned where (a trigger on the assignment table).
|
||||
func (i *Inventory) AssignmentGeneration(ctx context.Context) (int64, error) {
|
||||
var g int64
|
||||
if err := i.store.Pool().QueryRow(ctx, `select generation from assignment_generation`).Scan(&g); err != nil {
|
||||
return 0, fmt.Errorf("reading the assignment generation: %w", err)
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// RaiseAssignmentGeneration raises the generation past one a machine applied, and answers it: one more
|
||||
// than that, or what it already was when it is past it. It never lowers it.
|
||||
//
|
||||
// For one case only: a machine refused a send composed from the generation the mesh holds now, so the
|
||||
// machine applied a higher one than the mesh's counter — a store put back from a backup. Every send after
|
||||
// would be refused for ever; raised past it, the next send carries a generation the machine takes, composed
|
||||
// from the assignments the store holds now. A stale sender never meets this: its generation is below the
|
||||
// counter, and the counter is not touched for it.
|
||||
func (i *Inventory) RaiseAssignmentGeneration(ctx context.Context, past int64) (int64, error) {
|
||||
var g int64
|
||||
err := i.store.Pool().QueryRow(ctx,
|
||||
`update assignment_generation set generation = greatest(generation, $1 + 1) returning generation`, past).Scan(&g)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("raising the assignment generation past %d: %w", past, err)
|
||||
}
|
||||
return g, nil
|
||||
}
|
||||
|
||||
// SentGeneration is the generation a machine was last sent, by its id; zero for one sent without. What the
|
||||
// mesh WOULD send is stamped with it, so it reads as byte for byte what it DID send when nothing else changed.
|
||||
func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, error) {
|
||||
var g *int64
|
||||
if err := i.store.Pool().QueryRow(ctx, `select sent_generation from node where id = $1`, id).Scan(&g); err != nil {
|
||||
return 0, fmt.Errorf("reading the generation %s was last sent: %w", id, err)
|
||||
}
|
||||
if g == nil {
|
||||
return 0, nil
|
||||
}
|
||||
return *g, nil
|
||||
}
|
||||
|
||||
// ReadsGeneration says a machine's node-engine said it reads a generation in a declaration, by its id.
|
||||
func (i *Inventory) ReadsGeneration(ctx context.Context, id string) (bool, error) {
|
||||
var reads bool
|
||||
if err := i.store.Pool().QueryRow(ctx, `select reads_generation from node where id = $1`, id).Scan(&reads); err != nil {
|
||||
return false, fmt.Errorf("reading whether %s reads a generation: %w", id, err)
|
||||
}
|
||||
return reads, nil
|
||||
}
|
||||
|
||||
// RecordReadsGeneration keeps what a machine's latest report said of reading a generation.
|
||||
func (i *Inventory) RecordReadsGeneration(ctx context.Context, id string, reads bool) error {
|
||||
_, err := i.store.Pool().Exec(ctx, `update node set reads_generation = $2 where id = $1`, id, reads)
|
||||
return err
|
||||
}
|
||||
|
||||
// ReadsModules says a machine's node-engine said it reads a module per resource and a build per module (issue 1145).
|
||||
func (i *Inventory) ReadsModules(ctx context.Context, id string) (bool, error) {
|
||||
var reads bool
|
||||
if err := i.store.Pool().QueryRow(ctx, `select reads_modules from node where id = $1`, id).Scan(&reads); err != nil {
|
||||
return false, fmt.Errorf("reading whether %s reads modules: %w", id, err)
|
||||
}
|
||||
return reads, nil
|
||||
}
|
||||
|
||||
// RecordReadsModules keeps what a machine's latest report said of reading modules.
|
||||
func (i *Inventory) RecordReadsModules(ctx context.Context, id string, reads bool) error {
|
||||
_, err := i.store.Pool().Exec(ctx, `update node set reads_modules = $2 where id = $1`, id, reads)
|
||||
return err
|
||||
}
|
||||
|
||||
// Send is one declaration sent to a machine, as the mesh records it.
|
||||
type Send struct {
|
||||
// Node is the machine's id; NodeName its name, filled where it is read back.
|
||||
Node string
|
||||
NodeName string
|
||||
Sequence int64
|
||||
// Epoch is the lease epoch its sender acted under, zero for one that acted under none.
|
||||
Epoch int64
|
||||
// Sender is who sent it, in words: the caller, and the process and build that composed it.
|
||||
Sender string
|
||||
// Generation is the assignment generation it was composed from, zero when not known.
|
||||
Generation int64
|
||||
Digest string
|
||||
// Modules are the modules it named.
|
||||
Modules []string
|
||||
// Provisioning is the provisioner's plan of each module the send handed to it, by module, as JSON (novox/hq
|
||||
// ADR 0349 §4, design 57 P6): kept on the machine's record with the send, nil when it handed none. Never in
|
||||
// the record of the send itself.
|
||||
Provisioning []byte
|
||||
SentAt time.Time
|
||||
// RefusedAt is when the machine refused it for its generation, nil when it did not; RefusedApplied the
|
||||
// generation the machine said it had applied then; CounterRaisedTo what the mesh's counter was raised to
|
||||
// because the refusal showed it behind the machine, zero when it was not.
|
||||
RefusedAt *time.Time
|
||||
RefusedApplied int64
|
||||
CounterRaisedTo int64
|
||||
// Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown.
|
||||
Recorded bool
|
||||
// Answer is what the machine last reported of this send; filled only where a last send is read back
|
||||
// (LastSend, LastSends), the zero answer elsewhere.
|
||||
Answer SendAnswer
|
||||
}
|
||||
|
||||
// SendAnswer is what a machine reported of one send, read from its last report when that report is about the
|
||||
// send's declaration (novox/hq issue 485): whether it applied it, failed part of it or refused it, and why.
|
||||
// The zero answer is a machine that has not reported on this send — yet, or since a later report replaced it.
|
||||
type SendAnswer struct {
|
||||
// Outcome is applied, failed or refused, as the machine reported it; empty when it has not reported on it.
|
||||
Outcome string
|
||||
// Refused is the machine's own words when it refused the declaration whole.
|
||||
Refused string
|
||||
// Failed is how many of its resources failed, when some did.
|
||||
Failed int
|
||||
// At is when the machine reported it; nil when it has not.
|
||||
At *time.Time
|
||||
}
|
||||
|
||||
// unknownSender is the sender of a refused sequence the mesh has no record of sending.
|
||||
const unknownSender = "a sender the mesh has no record of (no send of this sequence was recorded: sent by " +
|
||||
"hand, before sends were recorded, or by a controller whose record was not written)"
|
||||
|
||||
// SendNotRecordedError is a send whose machine's record was written — its digest and the generation it
|
||||
// carried — and whose record of who sent it was not. The send is away and the machine's record stands; the
|
||||
// caller says this loudly and does not take the send for failed.
|
||||
type SendNotRecordedError struct{ Err error }
|
||||
|
||||
func (e *SendNotRecordedError) Error() string {
|
||||
return "the record of the send was not written: " + e.Err.Error()
|
||||
}
|
||||
func (e *SendNotRecordedError) Unwrap() error { return e.Err }
|
||||
|
||||
// RecordSentWith writes down what a machine was just sent — its digest, the builds it carried, the epoch and
|
||||
// the generation on the wire — and the send itself, who sent it and from which generation (novox/hq issue
|
||||
// 234), in one transaction. The digest and the generation are written together, so the would-send is never
|
||||
// stamped with a generation the machine was not sent. The send's own record is written under a savepoint: if
|
||||
// it cannot be, the machine's record is still committed and a *SendNotRecordedError says so. The oldest
|
||||
// sends beyond the newest 200 for the machine are let go.
|
||||
func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, builds map[string]string, epoch uint64,
|
||||
generation int64, s Send) error {
|
||||
var sentEpoch *int64
|
||||
if epoch > 0 {
|
||||
e := int64(epoch)
|
||||
sentEpoch = &e
|
||||
}
|
||||
tx, err := i.store.Pool().Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback(ctx) }()
|
||||
if _, err := tx.Exec(ctx,
|
||||
`update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3, sent_generation = $4,
|
||||
sent_provisioning = $5::jsonb
|
||||
where id = $1`, node, digest, sentEpoch, nullIfZero(generation), nullIfEmpty(s.Provisioning)); err != nil {
|
||||
return err
|
||||
}
|
||||
recordErr := recordSend(ctx, tx, node, s)
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
// And what it put on the machine, in the assignments (issues 1095, 1109): a write of it that fails fails this record,
|
||||
// for what a machine was sent is read from there.
|
||||
if err := i.recordRuns(ctx, node, builds, time.Now()); err != nil {
|
||||
return fmt.Errorf("%s was sent its declaration, and what each module of it runs there could not be written: %w", node, err)
|
||||
}
|
||||
if recordErr != nil {
|
||||
return &SendNotRecordedError{Err: recordErr}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// recordSend writes one send under a savepoint of tx, which it rolls back when the send cannot be written.
|
||||
func recordSend(ctx context.Context, tx pgx.Tx, node string, s Send) error {
|
||||
sp, err := tx.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = sp.Rollback(ctx) }()
|
||||
modules := s.Modules
|
||||
if modules == nil {
|
||||
modules = []string{}
|
||||
}
|
||||
if _, err := sp.Exec(ctx,
|
||||
`insert into declaration_send (node, sequence, epoch, sender, generation, digest, modules)
|
||||
values ($1, $2, $3, $4, $5, $6, $7)`,
|
||||
node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.Digest,
|
||||
modules); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := pruneSends(ctx, sp, node); err != nil {
|
||||
return err
|
||||
}
|
||||
return sp.Commit(ctx)
|
||||
}
|
||||
|
||||
// pruneSends lets go of a machine's sends beyond the newest 200.
|
||||
func pruneSends(ctx context.Context, q queries, node string) error {
|
||||
_, err := q.Exec(ctx,
|
||||
`delete from declaration_send where node = $1 and id not in
|
||||
(select id from declaration_send where node = $1 order by id desc limit $2)`, node, sendsKept)
|
||||
return err
|
||||
}
|
||||
|
||||
// sendColumns are a send's columns as scanSends reads them.
|
||||
const sendColumns = `s.node, n.name, coalesce(s.sequence, 0), coalesce(s.epoch, 0), s.sender, coalesce(s.generation, 0),
|
||||
s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0), coalesce(s.counter_raised_to, 0),
|
||||
s.recorded`
|
||||
|
||||
func scanSends(rows pgx.Rows) ([]Send, error) {
|
||||
defer rows.Close()
|
||||
var out []Send
|
||||
for rows.Next() {
|
||||
var s Send
|
||||
if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest,
|
||||
&s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// LastSend is the last declaration a machine was sent, by its name, with what the machine answered of it; false
|
||||
// when none was recorded. A refusal of a sequence the mesh has no send for is not a send, and is not it.
|
||||
func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) {
|
||||
sends, err := i.lastSends(ctx, `n.name = $1`, name)
|
||||
if err != nil {
|
||||
return Send{}, false, err
|
||||
}
|
||||
s, found := sends[name]
|
||||
return s, found, nil
|
||||
}
|
||||
|
||||
// LastSends is every machine's last send, by machine name, each with what the machine answered of it: what
|
||||
// `status` says per machine (novox/hq issue 485). A machine never sent anything has no entry.
|
||||
func (i *Inventory) LastSends(ctx context.Context) (map[string]Send, error) {
|
||||
return i.lastSends(ctx, `true`)
|
||||
}
|
||||
|
||||
// lastSends reads the newest recorded send of each machine matching where, beside the machine's last report
|
||||
// when that report is about the send (novox/hq issue 485). It names the send's declaration (its digest), and:
|
||||
// - from an ordered node-engine, one that reports the epoch and sequence it acted on, those are the send's
|
||||
// own. A periodic report about an identical earlier send names the same digest and may arrive after a later
|
||||
// send the machine never received; only the sequence tells them apart. It also holds for a fast machine
|
||||
// whose report lands before the send's row is stamped.
|
||||
// - from an engine that reports no order, the report came after the send — both times the store's own — so
|
||||
// a machine sent again a declaration it once applied does not read as having applied the new send.
|
||||
func (i *Inventory) lastSends(ctx context.Context, where string, args ...any) (map[string]Send, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select distinct on (n.name) `+sendColumns+`,
|
||||
coalesce(r.outcome, ''), coalesce(r.refused, ''), case when jsonb_typeof(r.failed) = 'array' then jsonb_array_length(r.failed) else 0 end, r.at
|
||||
from declaration_send s join node n on n.id = s.node
|
||||
left join node_report r on r.node = s.node and r.declared <> '' and r.declared = s.digest
|
||||
and case when r.reported_sequence is not null
|
||||
then r.reported_sequence = s.sequence and coalesce(r.reported_epoch, 0) = coalesce(s.epoch, 0)
|
||||
else r.at >= s.sent_at end
|
||||
where s.recorded and `+where+`
|
||||
order by n.name, s.id desc`, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]Send{}
|
||||
for rows.Next() {
|
||||
var s Send
|
||||
if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest,
|
||||
&s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded,
|
||||
&s.Answer.Outcome, &s.Answer.Refused, &s.Answer.Failed, &s.Answer.At); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[s.NodeName] = s
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and
|
||||
// answers that send, sender and all; false when no send of that sequence was recorded (RecordUnrecordedRefusal
|
||||
// keeps that one). The newest of that sequence when there are several: a declaration a person sent by hand may
|
||||
// repeat one. raisedTo is what the mesh's counter was raised to for it, zero for none.
|
||||
func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied, raisedTo int64) (Send, bool, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `with refused as (
|
||||
update declaration_send set refused_at = now(), refused_applied = $3, counter_raised_to = $4
|
||||
where id = (select id from declaration_send where node = $1 and sequence = $2 and recorded
|
||||
order by id desc limit 1)
|
||||
returning *)
|
||||
select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied,
|
||||
nullIfZero(raisedTo))
|
||||
if err != nil {
|
||||
return Send{}, false, err
|
||||
}
|
||||
sends, err := scanSends(rows)
|
||||
if err != nil || len(sends) == 0 {
|
||||
return Send{}, false, err
|
||||
}
|
||||
return sends[0], true, nil
|
||||
}
|
||||
|
||||
// RecordUnrecordedRefusal keeps a machine's refusal of a sequence the mesh has no send for, as a row of its
|
||||
// own whose sender is unknown, and answers it: S20 raises it like any other, because a refusal nobody can
|
||||
// attribute is no quieter for that. s carries the machine, the sequence, epoch, generation and digest the
|
||||
// refused declaration carried, and the refusal's applied generation and counter raise.
|
||||
func (i *Inventory) RecordUnrecordedRefusal(ctx context.Context, s Send) (Send, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `with refused as (
|
||||
insert into declaration_send (node, sequence, epoch, sender, generation, digest, refused_at, refused_applied,
|
||||
counter_raised_to, recorded)
|
||||
values ($1, $2, $3, $4, $5, $6, now(), $7, $8, false) returning *)
|
||||
select `+sendColumns+` from refused s join node n on n.id = s.node`,
|
||||
s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), unknownSender, nullIfZero(s.Generation), s.Digest,
|
||||
s.RefusedApplied, nullIfZero(s.CounterRaisedTo))
|
||||
if err != nil {
|
||||
return Send{}, err
|
||||
}
|
||||
sends, err := scanSends(rows)
|
||||
if err != nil {
|
||||
return Send{}, err
|
||||
}
|
||||
if len(sends) == 0 {
|
||||
return Send{}, fmt.Errorf("the refusal of sequence %d was not kept", s.Sequence)
|
||||
}
|
||||
if err := pruneSends(ctx, i.store.Pool(), s.Node); err != nil {
|
||||
return Send{}, err
|
||||
}
|
||||
return sends[0], nil
|
||||
}
|
||||
|
||||
// RefusedSendsSince is every send refused for its generation since a moment, newest first.
|
||||
func (i *Inventory) RefusedSendsSince(ctx context.Context, since time.Time) ([]Send, error) {
|
||||
rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+`
|
||||
from declaration_send s join node n on n.id = s.node
|
||||
where s.refused_at >= $1 order by s.refused_at desc`, since)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return scanSends(rows)
|
||||
}
|
||||
|
||||
// nullIfEmpty is raw JSON, or null for none.
|
||||
func nullIfEmpty(raw []byte) *string {
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
text := string(raw)
|
||||
return &text
|
||||
}
|
||||
@@ -1,342 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// The assignment generation and the record of every send (novox/hq issue 234). On 2026-10-04 a declaration
|
||||
// newer in sequence than every other named four fewer modules than the assignments held, a machine applied
|
||||
// it, and nothing the mesh kept could say who sent it or from what view.
|
||||
|
||||
func generationNow(t *testing.T, inv *Inventory) int64 {
|
||||
t.Helper()
|
||||
g, err := inv.AssignmentGeneration(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return g
|
||||
}
|
||||
|
||||
// **Every change to what is assigned where raises the generation** — an assignment and an unassignment — and a repeated
|
||||
// assignment, which changes nothing, does not.
|
||||
func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "postgres", "web")
|
||||
ctx := t.Context()
|
||||
start := generationNow(t, inv)
|
||||
if start < 1 {
|
||||
t.Fatalf("the generation starts at %d; zero is \"none claimed\" on the wire", start)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, node, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
assigned := generationNow(t, inv)
|
||||
if assigned <= start {
|
||||
t.Fatalf("an assignment left the generation at %d", assigned)
|
||||
}
|
||||
if _, err := inv.Assign(ctx, node, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again := generationNow(t, inv); again != assigned {
|
||||
t.Errorf("an assignment repeated, which changed nothing, moved the generation from %d to %d", assigned, again)
|
||||
}
|
||||
if err := inv.Unassign(ctx, node, "postgres"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unassigned := generationNow(t, inv)
|
||||
if unassigned <= assigned {
|
||||
t.Fatalf("an unassignment left the generation at %d", unassigned)
|
||||
}
|
||||
// A machine's removal, which no verb makes, is read from the machines the store knows: an assignment on a machine it
|
||||
// does not know is none (assignments in the bucket have no key of the store to cascade through).
|
||||
}
|
||||
|
||||
// **The generation never goes down, and is raised past what a machine applied when the mesh's own counter
|
||||
// is behind it** — a store put back from a backup — so the next send is not refused for ever.
|
||||
func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) {
|
||||
inv, _ := aNodeWithModules(t)
|
||||
now := generationNow(t, inv)
|
||||
if raised, err := inv.RaiseAssignmentGeneration(t.Context(), now+40); err != nil || raised != now+41 {
|
||||
t.Fatalf("raised past %d to %d (%v)", now+40, raised, err)
|
||||
}
|
||||
if raised, err := inv.RaiseAssignmentGeneration(t.Context(), 2); err != nil || raised != now+41 {
|
||||
t.Fatalf("a raise below the counter moved it to %d (%v)", raised, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **Every send is recorded: its sequence, its sender, the generation it came from and the modules it
|
||||
// named** — and the machine's last send is what `plan` reads, the would-send is stamped with the generation
|
||||
// written with the digest, and a refusal is kept on the send it refused, naming its sender.
|
||||
func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first := Send{Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)",
|
||||
Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}}
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "d11", nil, 57, 40, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stale := Send{Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor",
|
||||
Generation: 38, Digest: "d12", Modules: []string{"docker"}}
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "d12", nil, 57, 38, stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
last, found, err := inv.LastSend(ctx, node)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the last send is not kept: %v", err)
|
||||
}
|
||||
if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || !last.Recorded ||
|
||||
!slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() {
|
||||
t.Fatalf("the last send reads %+v", last)
|
||||
}
|
||||
if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 38 {
|
||||
t.Fatalf("what the machine was last sent of it reads %d (%v)", generation, err)
|
||||
}
|
||||
|
||||
refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40, 0)
|
||||
if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 ||
|
||||
refused.CounterRaisedTo != 0 {
|
||||
t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err)
|
||||
}
|
||||
since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 {
|
||||
t.Fatalf("the refused sends within the hour read %+v (%v)", since, err)
|
||||
}
|
||||
if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40, 0); err != nil || found {
|
||||
t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A refusal of a sequence the mesh has no send for is kept too, its sender unknown**, so S20 raises it;
|
||||
// it is not the machine's last send, and it says what the counter was raised to.
|
||||
func TestARefusalOfASendNeverRecordedIsKeptWithAnUnknownSender(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
kept, err := inv.RecordUnrecordedRefusal(ctx, Send{Node: record.ID, Sequence: 99, Epoch: 57, Generation: 41,
|
||||
Digest: "d99", RefusedApplied: 44, CounterRaisedTo: 45})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if kept.Recorded || kept.Sender != unknownSender || kept.RefusedAt == nil || kept.CounterRaisedTo != 45 ||
|
||||
kept.NodeName != node {
|
||||
t.Fatalf("the refusal reads %+v", kept)
|
||||
}
|
||||
since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour))
|
||||
if err != nil || len(since) != 1 || since[0].Sequence != 99 {
|
||||
t.Fatalf("the refused sends within the hour read %+v (%v)", since, err)
|
||||
}
|
||||
if _, found, err := inv.LastSend(ctx, node); err != nil || found {
|
||||
t.Errorf("a refusal of a send never recorded reads as the machine's last send: %v, %v", found, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **The machine's record and the send's are written together, and a send record that cannot be written
|
||||
// leaves the machine's record standing and says so**: the digest and generation are the machine's, and a
|
||||
// send that is away must not read as failed, nor the machine as behind.
|
||||
func TestASendRecordThatCannotBeWrittenLeavesTheMachinesRecord(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A NUL byte is text PostgreSQL refuses: the send's own row cannot be written.
|
||||
broken := Send{Sequence: 13, Sender: "a test", Generation: 42, Digest: "d\x0013"}
|
||||
err = inv.RecordSentWith(ctx, record.ID, "d13", nil, 0, 42, broken)
|
||||
var unrecorded *SendNotRecordedError
|
||||
if !errors.As(err, &unrecorded) {
|
||||
t.Fatalf("a send record that could not be written read as %v", err)
|
||||
}
|
||||
if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 42 {
|
||||
t.Fatalf("the machine's generation was not written with its digest: %d (%v)", generation, err)
|
||||
}
|
||||
if sent, err := inv.Outstanding(ctx, node); err != nil || sent != "d13" {
|
||||
t.Fatalf("the machine's digest was not written: %q (%v)", sent, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer.
|
||||
func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
record, err := inv.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, reads := range []bool{false, true, false} {
|
||||
if err := inv.RecordReadsGeneration(t.Context(), record.ID, reads); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := inv.ReadsGeneration(t.Context(), record.ID); err != nil || got != reads {
|
||||
t.Fatalf("recorded %v, read %v (%v)", reads, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **The last send says what the machine answered of it** (novox/hq issue 485): applied, refused with its
|
||||
// words, or not answered yet — read from the machine's report only when that report is about the send's
|
||||
// declaration and came after it, so a report about an earlier send is never this one's answer. And every
|
||||
// machine's last send is read at once, for status, by name; a machine never sent anything has none.
|
||||
func TestTheLastSendSaysWhatTheMachineAnswered(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.AddNode(ctx, "laptop"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
send := func(sequence int64, digest string) {
|
||||
t.Helper()
|
||||
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
|
||||
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest,
|
||||
Modules: []string{"docker", "sudo"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
report := func(d Doing) {
|
||||
t.Helper()
|
||||
if _, err := inv.RecordDoing(ctx, record.ID, d); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
answer := func() SendAnswer {
|
||||
t.Helper()
|
||||
last, found, err := inv.LastSend(ctx, node)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the last send is not read back: %v, %v", found, err)
|
||||
}
|
||||
return last.Answer
|
||||
}
|
||||
|
||||
send(11, "d11")
|
||||
if a := answer(); a.Outcome != "" || a.At != nil {
|
||||
t.Fatalf("a send not reported on reads as answered: %+v", a)
|
||||
}
|
||||
report(Doing{Outcome: OutcomeApplied, Declared: "d11", Applied: 3})
|
||||
if a := answer(); a.Outcome != OutcomeApplied || a.At == nil {
|
||||
t.Fatalf("a send the machine reported applying reads %+v", a)
|
||||
}
|
||||
|
||||
send(12, "d12")
|
||||
if a := answer(); a.Outcome != "" {
|
||||
t.Fatalf("the report about the send before is taken for this one's answer: %+v", a)
|
||||
}
|
||||
report(Doing{Outcome: OutcomeRefused, Declared: "d12", Refused: "unknown field \"generation\"\nand more"})
|
||||
if a := answer(); a.Outcome != OutcomeRefused || a.Refused != "unknown field \"generation\"\nand more" {
|
||||
t.Fatalf("a send the machine refused reads %+v", a)
|
||||
}
|
||||
|
||||
// Sent again a declaration it reported applying before the send: not answered until it reports again.
|
||||
report(Doing{Outcome: OutcomeApplied, Declared: "d12", Applied: 3})
|
||||
send(13, "d12")
|
||||
if a := answer(); a.Outcome != "" {
|
||||
t.Fatalf("a report from before the send is taken for its answer: %+v", a)
|
||||
}
|
||||
|
||||
all, err := inv.LastSends(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(all) != 1 || all[node].Sequence != 13 || all[node].Epoch != 57 || all[node].Generation != 40 ||
|
||||
!slices.Equal(all[node].Modules, []string{"docker", "sudo"}) {
|
||||
t.Fatalf("every machine's last send reads %+v", all)
|
||||
}
|
||||
if _, found := all["laptop"]; found {
|
||||
t.Error("a machine never sent anything has a last send")
|
||||
}
|
||||
}
|
||||
|
||||
// **A report about another declaration, or another send of the same one, is not the last send's answer**
|
||||
// (novox/hq issue 485, review of mesh-controller #234). A report naming an earlier send's digest that arrives
|
||||
// after a later send is not the later send's answer. And from an ordered node-engine, a periodic report about
|
||||
// an identical earlier send — the same digest, arriving after the later send — is not its answer either: only
|
||||
// the report's epoch and sequence say which send it is about.
|
||||
func TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
ctx := t.Context()
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
send := func(sequence int64, digest string) {
|
||||
t.Helper()
|
||||
if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57,
|
||||
Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
ordered := func(epoch, sequence int64, digest string) {
|
||||
t.Helper()
|
||||
if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: digest, Applied: 1},
|
||||
ReportOrder{Epoch: epoch, Sequence: sequence}, func(ReportOrder) bool { return false }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
answer := func() string {
|
||||
t.Helper()
|
||||
last, found, err := inv.LastSend(ctx, node)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the last send is not read back: %v, %v", found, err)
|
||||
}
|
||||
return last.Answer.Outcome
|
||||
}
|
||||
|
||||
// An engine that reports no order: the digest decides.
|
||||
send(11, "d11")
|
||||
send(12, "d12")
|
||||
if _, err := inv.RecordDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d11"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := answer(); a != "" {
|
||||
t.Fatalf("a report about d11, after d12 was sent, reads as d12's answer: %q", a)
|
||||
}
|
||||
|
||||
// An ordered engine: the epoch and sequence decide, whatever the digest and the time.
|
||||
send(20, "d20")
|
||||
ordered(57, 20, "d20")
|
||||
if a := answer(); a != OutcomeApplied {
|
||||
t.Fatalf("an ordered report about sequence 20 is not its answer: %q", a)
|
||||
}
|
||||
send(21, "d20")
|
||||
ordered(57, 20, "d20")
|
||||
if a := answer(); a != "" {
|
||||
t.Fatalf("a periodic report about sequence 20, after 21 was sent with the same digest, reads as 21's answer: %q", a)
|
||||
}
|
||||
ordered(56, 21, "d20")
|
||||
if a := answer(); a != "" {
|
||||
t.Fatalf("a report about sequence 21 of another epoch reads as this send's answer: %q", a)
|
||||
}
|
||||
ordered(57, 21, "d20")
|
||||
if a := answer(); a != OutcomeApplied {
|
||||
t.Fatalf("an ordered report about sequence 21 is not its answer: %q", a)
|
||||
}
|
||||
}
|
||||
|
||||
// 1145/AC3: whether a machine reads modules is its latest word, as whether it reads a generation is.
|
||||
func TestWhetherAMachineReadsModulesIsItsLatestWord(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t)
|
||||
record, err := inv.NodeByName(t.Context(), node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, reads := range []bool{false, true, false} {
|
||||
if err := inv.RecordReadsModules(t.Context(), record.ID, reads); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := inv.ReadsModules(t.Context(), record.ID); err != nil || got != reads {
|
||||
t.Fatalf("1145/AC3: recorded %v, read %v (%v)", reads, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// novox/hq issue 259: a send keeps which build of each module it carried. A machine never sent
|
||||
// anything, or sent with nothing recorded — before this was kept, or by hand — is not known, which
|
||||
// is not the same as having been sent nothing.
|
||||
func TestASendKeepsTheBuildsItCarried(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
node, err := inv.AddNode(ctx, "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil {
|
||||
t.Fatalf("a machine never sent anything has known builds %v (%v): %v", builds, known, err)
|
||||
}
|
||||
|
||||
carried := map[string]string{"resolver": "abc123", "network": ""}
|
||||
if err := inv.RecordSent(ctx, node.ID, "d1", carried); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// What it was sent is read from the keys of the machine's assignments that name a commit (issue 1109): a module carried
|
||||
// with no commit known names none.
|
||||
builds, known, err := inv.SentBuilds(ctx, "anchor")
|
||||
if err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"resolver": "abc123"}) {
|
||||
t.Fatalf("the builds sent were not kept: %v %v %v", builds, known, err)
|
||||
}
|
||||
|
||||
// Sent with nothing carried: known, and empty.
|
||||
if err := inv.RecordSent(ctx, node.ID, "d2", map[string]string{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || !known || len(builds) != 0 {
|
||||
t.Fatalf("an empty send: %v %v %v", builds, known, err)
|
||||
}
|
||||
|
||||
// Sent by hand: not known, and the digest still recorded.
|
||||
if err := inv.RecordSent(ctx, node.ID, "d3", nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil {
|
||||
t.Fatalf("a send whose builds are not known read as %v %v: %v", builds, known, err)
|
||||
}
|
||||
if sent, err := inv.Outstanding(ctx, "anchor"); err != nil || sent != "d3" {
|
||||
t.Fatalf("the digest was not recorded with it: %q %v", sent, err)
|
||||
}
|
||||
|
||||
if _, known, err := inv.SentBuilds(ctx, "nobody"); err != nil || known {
|
||||
t.Fatalf("a machine the mesh does not know: %v %v", known, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A module's current build is the commit its manifest was read at, with its upgrade policy.
|
||||
func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "resolver", Version: "1"},
|
||||
Source{Repository: "novox/mesh-catalog", BuiltFrom: "c1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "by-hand", Version: "1"}, Source{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.SetUpgradeOf(ctx, "by-hand", Upgrade{RollOut: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current, err := inv.CurrentBuilds(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Rolled out by default, one machine first and gated (novox/hq ADR 0236).
|
||||
if got := current["resolver"]; got != (CurrentBuild{Commit: "c1", RollOut: true, From: catalogue.FromDefault}) {
|
||||
t.Errorf("resolver is at %+v", got)
|
||||
}
|
||||
if got := current["by-hand"]; got != (CurrentBuild{RollOut: true, From: catalogue.FromPerson}) {
|
||||
t.Errorf("a module with no source is at %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import "testing"
|
||||
|
||||
// novox/hq issue 249: the digest of the user list a machine was last sent is kept, by its name, and
|
||||
// is empty for a machine never sent one.
|
||||
func TestTheUserListAMachineWasSentIsKept(t *testing.T) {
|
||||
inv := ForTest(t)
|
||||
ctx := t.Context()
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "" {
|
||||
t.Fatalf("a machine never sent a list has %q: %v", sent, err)
|
||||
}
|
||||
if err := inv.RecordSentBusUsers(ctx, "anchor", "abc"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "abc" {
|
||||
t.Fatalf("the list sent was not kept: %q %v", sent, err)
|
||||
}
|
||||
if sent, err := inv.SentBusUsers(ctx, "nobody"); err != nil || sent != "" {
|
||||
t.Fatalf("a machine the mesh does not know: %q %v", sent, err)
|
||||
}
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A send keeps the provisioner's plans it handed (design 57 P6, issue 672, migration 0100): the assignments mirror
|
||||
// carries the plan of the build that machine was sent, in the module's own assignment and no other, and a walk
|
||||
// reads which templates the send handed.
|
||||
func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) {
|
||||
inv, node := aNodeWithModules(t, "dunst", "web")
|
||||
ctx := t.Context()
|
||||
for _, m := range []string{"dunst", "web"} {
|
||||
if _, err := inv.Assign(ctx, node, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat("d", 64)
|
||||
plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest,
|
||||
Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc",
|
||||
Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}}
|
||||
// A plan for a module the machine no longer has assigned is kept with the send and mirrored nowhere; one for a
|
||||
// module only pulled in by another is mirrored under its own key, saying what requires it (issue 765).
|
||||
raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"gone":` + mustJSON(t, plan) + `,"notify":` +
|
||||
strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`)
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sent, _, err := inv.SentTo(ctx, node)
|
||||
if err != nil || !reflect.DeepEqual(sent.Provisioned, map[string]string{"dunst": digest, "gone": digest,
|
||||
"notify": digest}) {
|
||||
t.Fatalf("the send says it handed %v (%v)", sent.Provisioned, err)
|
||||
}
|
||||
plans, err := inv.SentProvisioningPlans(ctx)
|
||||
required := plan
|
||||
required.RequiredBy = "desk"
|
||||
if err != nil || !reflect.DeepEqual(plans, map[string]Provisioning{node + ".dunst": plan, node + ".notify": required}) {
|
||||
t.Fatalf("the plans mirrored are %+v (%v)", plans, err)
|
||||
}
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
mirror := inv.PlanValuesForTest(t)
|
||||
var dunst, web AssignmentMirrored
|
||||
if err := json.Unmarshal(mirror[node+".dunst"], &dunst); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := json.Unmarshal(mirror[node+".web"], &web); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if dunst.Provisioning == nil || !reflect.DeepEqual(*dunst.Provisioning, plan) || web.Provisioning != nil {
|
||||
t.Fatalf("the mirror carries %+v for dunst and %+v for web", dunst.Provisioning, web.Provisioning)
|
||||
}
|
||||
var notify AssignmentMirrored
|
||||
if err := json.Unmarshal(mirror[node+".notify"], ¬ify); err != nil {
|
||||
t.Fatalf("no value for the pulled-in module: %s (%v)", mirror[node+".notify"], err)
|
||||
}
|
||||
if notify.RequiredBy != "desk" || !notify.Assigned.IsZero() || notify.Provisioning == nil ||
|
||||
!reflect.DeepEqual(*notify.Provisioning, plan) {
|
||||
t.Fatalf("the mirror carries %s for the pulled-in module", mirror[node+".notify"])
|
||||
}
|
||||
if _, has := mirror[node+".gone"]; has {
|
||||
t.Fatalf("a module no longer assigned is mirrored: %s", mirror[node+".gone"])
|
||||
}
|
||||
|
||||
// The next send hands nothing: nothing is kept, and the assignment's value is the bytes it was.
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sent, _, err := inv.SentTo(ctx, node); err != nil || sent.Provisioned != nil {
|
||||
t.Fatalf("a send handing nothing says it handed %v (%v)", sent.Provisioned, err)
|
||||
}
|
||||
if mirror := inv.PlanValuesForTest(t); strings.Contains(string(mirror[node+".dunst"]), "provisioning") {
|
||||
t.Fatalf("dunst's assignment still carries a plan: %s", mirror[node+".dunst"])
|
||||
} else if _, has := mirror[node+".notify"]; has {
|
||||
t.Fatalf("a module the machine no longer runs is mirrored: %s", mirror[node+".notify"])
|
||||
}
|
||||
}
|
||||
|
||||
func mustJSON(t *testing.T, v any) string {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(raw)
|
||||
}
|
||||
@@ -30,7 +30,6 @@ type sourceRow struct {
|
||||
made []byte
|
||||
manifest []byte
|
||||
manifestDigest string
|
||||
failedGate bool
|
||||
}
|
||||
|
||||
// sourceRows is every successful build of each module (of one module, when named), newest request
|
||||
@@ -53,13 +52,10 @@ func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][
|
||||
func standing(builds []sourceRow, from int) int {
|
||||
at := from
|
||||
fp := builds[from].fingerprint
|
||||
if fp == "" || builds[from].failedGate {
|
||||
if fp == "" {
|
||||
return at
|
||||
}
|
||||
for j := from + 1; j < len(builds); j++ {
|
||||
if builds[j].failedGate {
|
||||
break
|
||||
}
|
||||
if builds[j].fingerprint != fp {
|
||||
if !newForm(fp) || newForm(builds[j].fingerprint) || !sameDigests(builds[at].made, builds[j].made) {
|
||||
break
|
||||
|
||||
@@ -39,10 +39,14 @@ func TestAStoredBuildWithASilentMergeableFileIsStillReadAndComposed(t *testing.T
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
m, found, err := inv.ManifestAt(ctx, "mail-client", b.Commit)
|
||||
kept, found, err := inv.BuildByID(ctx, b.ID)
|
||||
if err != nil || !found {
|
||||
t.Fatalf("the stored build is not read again: found %v, %v", found, err)
|
||||
}
|
||||
m, err := catalogue.ParseManifest(kept.Manifest)
|
||||
if err != nil {
|
||||
t.Fatalf("the stored manifest is not read again: %v", err)
|
||||
}
|
||||
if written := catalogue.WrittenSettingsProblems(m); len(written) != 1 {
|
||||
t.Fatalf("the rule this build predates should name its accounts file where it is written: %v", written)
|
||||
}
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Issue 1886: R4e entered tier 1 at 02:01, was stopped, and was resumed at 07:43 at the same tier. Its save kept the
|
||||
// tier's entry of 02:01, so at 07:50 the plan read "waiting for mesh-build-image on shanks to be applied for 5h48m53s"
|
||||
// and its progress watchdog measured the tier from before the stop. A walk set open again at the tier it ended at
|
||||
// enters that tier anew: its time in the tier runs from the resume.
|
||||
func TestAWalkResumedAtItsTierEntersItAnew(t *testing.T) {
|
||||
entered := time.Date(2026, 10, 23, 0, 1, 7, 0, time.UTC)
|
||||
resumed := time.Date(2026, 10, 23, 5, 43, 30, 0, time.UTC)
|
||||
old := &Plan{ID: "plan-r4e", State: PlanFailed, Tier: 1, Tiers: [][]string{{"a"}, {"b"}, {"c"}},
|
||||
Created: entered.Add(-2 * time.Hour), TierEntered: entered}
|
||||
p := *old
|
||||
p.State = PlanBuilding
|
||||
got, d := tierLeftOf(old, p, resumed)
|
||||
if !got.Equal(resumed) || d != nil {
|
||||
t.Fatalf("1886: a walk resumed at tier 1 entered it at %s (duration %v); want %s, no tier left", got, d, resumed)
|
||||
}
|
||||
// A walk still in its tier keeps its entry; one that ended keeps it too.
|
||||
open := *old
|
||||
open.State = PlanRolling
|
||||
if got, _ := tierLeftOf(&open, open, resumed); !got.Equal(entered) {
|
||||
t.Fatalf("an open walk still in its tier was stamped anew: %s", got)
|
||||
}
|
||||
if got, _ := tierLeftOf(old, *old, resumed); !got.Equal(entered) {
|
||||
t.Fatalf("a failed walk saved again was stamped anew: %s", got)
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user