Drop the plan state from the store and the assignments: migration 0108 and placement-only assignments (release 6, issue 1945 B9, B4)
This commit is contained in:
@@ -495,52 +495,6 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is
|
||||
// composed after it — never sent after it with the node still adopted. And the send inside the
|
||||
// flip is not made to wait on the flip's own hold.
|
||||
func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var heldElsewhere, heldHere error
|
||||
sendNodes = func(inner context.Context, open *stores, names []string) error {
|
||||
// Another caller cannot hold the node while the flip sends it.
|
||||
waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
||||
defer cancel()
|
||||
if release, err := open.inventory.HoldNodes(waiting, names); err == nil {
|
||||
release()
|
||||
heldElsewhere = errors.New("another caller held the node while the flip sent it")
|
||||
}
|
||||
// The flip's own send holds it without waiting on itself.
|
||||
_, release, err := holdNodes(inner, open, names)
|
||||
if err != nil {
|
||||
heldHere = err
|
||||
return err
|
||||
}
|
||||
release()
|
||||
return nil
|
||||
}
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if heldElsewhere != nil || heldHere != nil {
|
||||
t.Fatalf("%v %v", heldElsewhere, heldHere)
|
||||
}
|
||||
// And given back once it is done.
|
||||
release, err := open.inventory.HoldNodes(ctx, []string{"anchor"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release()
|
||||
}
|
||||
|
||||
// novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found,
|
||||
// not only its files, and the digest changes when any of them does.
|
||||
func TestThePreviewNamesEveryHeldKind(t *testing.T) {
|
||||
@@ -611,33 +565,3 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) {
|
||||
t.Fatal("a refused flip changed something")
|
||||
}
|
||||
}
|
||||
|
||||
// An assignment cannot land between a preview and the flip that takes every module: assigning
|
||||
// holds the node, so it waits for whatever is converging it.
|
||||
func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) {
|
||||
open, _ := anAdoptedAnchor(t)
|
||||
ctx := t.Context()
|
||||
if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportsHolding(t, open, heldFile)
|
||||
preview, err := converge(ctx, open, "anchor", false, "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll
|
||||
inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond
|
||||
defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }()
|
||||
|
||||
var whileFlipping error
|
||||
sendNodes = func(context.Context, *stores, []string) error {
|
||||
_, whileFlipping = assign(ctx, open, "anchor", "notes")
|
||||
return nil
|
||||
}
|
||||
if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !errors.Is(whileFlipping, inventory.ErrNodeBusy) {
|
||||
t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// busWaitBound is how long a module whose ask the bus did not answer is asked again at every look before the plan fails
|
||||
// (issue 1334).
|
||||
const busWaitBound = 10 * time.Minute
|
||||
|
||||
// busWaiting says an open plan has a module of its current tier that the bus did not answer an ask of: the look must come back
|
||||
// for it, as nothing else will when no build of the tier is in flight (issue 1334).
|
||||
func busWaiting(p inventory.Plan) bool {
|
||||
return len(busWaitingIn(p)) > 0
|
||||
}
|
||||
|
||||
// busWaitingIn are the modules of the plan's current tier left unasked because the bus did not answer, sorted.
|
||||
func busWaitingIn(p inventory.Plan) []string {
|
||||
if !p.Open() || p.Tier >= len(p.Tiers) {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, m := range p.Tiers[p.Tier] {
|
||||
if s := p.Modules[m]; s != nil && s.State == "" && s.BusWaitFrom != nil {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// busWaitWords is how a verb says modules are waiting for the bus: not asked yet, and asked again by the plan at its next look.
|
||||
func busWaitWords(p inventory.Plan, modules []string) string {
|
||||
return fmt.Sprintf("%s not asked yet: the bus did not answer; %s asks again at its next look", strings.Join(modules, ", "), p.ID)
|
||||
}
|
||||
|
||||
// slicesWithout is names without those in drop, in order.
|
||||
func slicesWithout(names, drop []string) []string {
|
||||
var out []string
|
||||
for _, n := range names {
|
||||
if !slices.Contains(drop, n) {
|
||||
out = append(out, n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||
// delivery goes on with the machines heard, says the others were away, and the away ones get the builds by the send
|
||||
// they are given when heard again (ADR 0353, ADR 0360 §5). Away is the silent check's one verdict, machineFacts.silent
|
||||
// over the watchman's presence (ADR 0433 §3), never a second rule; a sleeping machine is away once presence says so.
|
||||
|
||||
// awayOf is the machines out of touch at a moment: those the silent check calls silent (a sleeping machine
|
||||
// is away once presence says so). A machine
|
||||
// never heard from has not joined and is not away.
|
||||
func awayOf(machines []machineFacts, now time.Time) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, m := range machines {
|
||||
if m.silent(now) {
|
||||
out[m.name] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeAway is the walk's reading of which machines are away as a tier is sent: every node with its last-heard time and
|
||||
// the watchman's verdict on its presence key. A bus that cannot be asked leaves none away, which is what a walk did
|
||||
// before: a machine is not passed on a reading that was not made.
|
||||
var judgeAway = func(ctx context.Context, open *stores, now time.Time) (map[string]bool, error) {
|
||||
nodes, err := open.inventory.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
address, err := broker.BusAddress()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js, err := broker.Dial(address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer js.Close()
|
||||
machines := make([]machineFacts, len(nodes))
|
||||
for i, n := range nodes {
|
||||
machines[i] = machineFacts{name: n.Name, lastHeard: n.LastSeen}
|
||||
}
|
||||
if err := withPresence(ctx, js.Conn(), machines, now); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return awayOf(machines, now), nil
|
||||
}
|
||||
|
||||
// rememberAway keeps on the plan the machines of running that are away: sorted, each once.
|
||||
func rememberAway(p *inventory.Plan, running []string, away map[string]bool) {
|
||||
for _, n := range running {
|
||||
if away[n] && !slices.Contains(p.Away, n) {
|
||||
p.Away = append(p.Away, n)
|
||||
}
|
||||
}
|
||||
sort.Strings(p.Away)
|
||||
}
|
||||
|
||||
// runningHeard is the machines of running that the walk waits on for a module: those a send of the walk reached, and
|
||||
// those it did not pass as away.
|
||||
func runningHeard(p inventory.Plan, running []string, s *inventory.PlanModule) []string {
|
||||
return slices.DeleteFunc(slices.Clone(running), func(n string) bool {
|
||||
if !slices.Contains(p.Away, n) {
|
||||
return false
|
||||
}
|
||||
_, sent := s.Rest[n]
|
||||
return !sent
|
||||
})
|
||||
}
|
||||
|
||||
// awaySaying is what a plan says of the machines it passed as away, empty when it passed none.
|
||||
func awaySaying(p inventory.Plan) string {
|
||||
var said []string
|
||||
for _, n := range p.Away {
|
||||
said = append(said, n+" was away: sent when it is heard again")
|
||||
}
|
||||
return strings.Join(said, "; ")
|
||||
}
|
||||
|
||||
// judgedAway reads which machines are away for a tier about to be sent, and says when it cannot.
|
||||
func judgedAway(ctx context.Context, open *stores, now time.Time) map[string]bool {
|
||||
away, err := judgeAway(ctx, open, now)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "which machines are away cannot be read, so none is passed over: %v\n", err)
|
||||
return nil // empty-on-error: none is passed over on a reading that was not made; the error is said just above
|
||||
}
|
||||
return away
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the
|
||||
// walk judges it with the silent check's one verdict, machineFacts.silent, and a sleeping machine is away once presence says so.
|
||||
|
||||
// 1161/AC1: away is the silent check's verdict; AC4: a machine heard is not away.
|
||||
func TestAMachineIsAwayAsTheSilentCheckJudgesIt(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
machines := []machineFacts{
|
||||
{name: "heard", lastHeard: now.Add(-time.Minute)},
|
||||
{name: "quiet", lastHeard: now.Add(-silentBound - time.Minute)},
|
||||
{name: "stuck", lastHeard: now, presence: presenceSilent},
|
||||
{name: "back", lastHeard: now.Add(-time.Hour), presence: presenceHeard},
|
||||
{name: "new"},
|
||||
}
|
||||
got := awayOf(machines, now)
|
||||
for node, want := range map[string]bool{"heard": false, "quiet": true, "stuck": true, "back": false,
|
||||
"new": false} {
|
||||
if got[node] != want {
|
||||
t.Errorf("1161/AC1: %s away is %v; want %v (away: %v)", node, got[node], want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC1, AC2: the plan keeps the machines of the tier it passed as away, sorted and once.
|
||||
func TestAPlanKeepsTheMachinesItPassedAsAway(t *testing.T) {
|
||||
p := &inventory.Plan{}
|
||||
away := map[string]bool{"g14": true, "ace": true}
|
||||
rememberAway(p, []string{"anchor", "g14"}, away)
|
||||
rememberAway(p, []string{"g14", "ace"}, away)
|
||||
if !slices.Equal(p.Away, []string{"ace", "g14"}) {
|
||||
t.Fatalf("1161/AC2: the plan keeps %v away; want ace, g14", p.Away)
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC2: the plan says it, where its other passed machines are said.
|
||||
func TestAPlanSaysTheMachinesItPassedAsAway(t *testing.T) {
|
||||
say := awaySaying(inventory.Plan{Away: []string{"g14"}})
|
||||
if !strings.Contains(say, "g14 was away: sent when it is heard again") {
|
||||
t.Fatalf("1161/AC2: the plan says %q", say)
|
||||
}
|
||||
if got := awaySaying(inventory.Plan{}); got != "" {
|
||||
t.Fatalf("1161/AC2: a plan that passed no machine says %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// 1161/AC1: the gate does not wait for a machine passed as away, unless a send reached it; AC4: a machine heard is waited for.
|
||||
func TestAGateDoesNotWaitForAMachinePassedAsAway(t *testing.T) {
|
||||
p := inventory.Plan{Away: []string{"g14"}}
|
||||
none := &inventory.PlanModule{}
|
||||
if got := runningHeard(p, []string{"anchor", "g14"}, none); !slices.Equal(got, []string{"anchor"}) {
|
||||
t.Fatalf("1161/AC1: the gate waits on %v; want anchor alone", got)
|
||||
}
|
||||
sent := &inventory.PlanModule{Rest: map[string]inventory.SentDeclaration{"g14": {Digest: "d"}}}
|
||||
if got := runningHeard(p, []string{"anchor", "g14"}, sent); !slices.Equal(got, []string{"anchor", "g14"}) {
|
||||
t.Fatalf("1161/AC4: a machine a send reached is waited for: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -150,16 +150,6 @@ func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild)
|
||||
return notRegistered(err)
|
||||
}
|
||||
}
|
||||
// **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard
|
||||
// twice, or replayed, would otherwise make the build a rollback put back what the module is again,
|
||||
// and the next push would send it.
|
||||
if failed, err := inv.GateFailed(ctx, kept.ID); err != nil {
|
||||
return manifest, err
|
||||
} else if failed {
|
||||
return manifest, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+
|
||||
"back: it is recorded and not registered again — a newer build is", j.by, manifest.Module,
|
||||
short(j.commit))
|
||||
}
|
||||
// Then the module is judged against the list the mesh publishes, as every manifest read again is: one naming a
|
||||
// key that list does not hold, or of another type, is refused here and never enters the catalogue.
|
||||
if j.carried != nil {
|
||||
@@ -283,8 +273,4 @@ type answers struct {
|
||||
// applied, is not well: an assignment somebody made is not made yet, or will not be.
|
||||
pending []inventory.PendingAssignment
|
||||
pendingUnread string
|
||||
// lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq
|
||||
// issue 485); lastSentUnread why it could not be read.
|
||||
lastSent map[string]inventory.Send
|
||||
lastSentUnread string
|
||||
}
|
||||
|
||||
@@ -524,7 +524,6 @@ type registrations interface {
|
||||
SourceIdentity(ctx context.Context, module string) (int64, error)
|
||||
Catalogued(ctx context.Context) ([]inventory.Entry, error)
|
||||
BuildRequestByID(ctx context.Context, id string) (inventory.BuildRequest, bool, error)
|
||||
MergeOf(ctx context.Context, repository, commit string) (inventory.BatchedMerge, bool, error)
|
||||
}
|
||||
|
||||
// mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in.
|
||||
|
||||
@@ -129,41 +129,6 @@ func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "evil"})
|
||||
fork.Commit = "c0ffee0123456789" // the commit sudo was registered at
|
||||
keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false)
|
||||
if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) {
|
||||
t.Fatalf("the fork's build was taken in: %v", err)
|
||||
}
|
||||
// Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older
|
||||
// than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main.
|
||||
failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo",
|
||||
map[string]any{"module": "sudo", "version": "2"})
|
||||
failed.Commit = "badbadbad0123456"
|
||||
if _, _, err := takeIn(ctx, inv, failed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, _, err := inv.BuildByID(ctx, failed.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if found && previous.ID == fork.ID {
|
||||
t.Fatalf("a rollback would put back the fork's build %s", previous.ID)
|
||||
}
|
||||
if !found || previous.ID != "build-sudo" {
|
||||
t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) {
|
||||
open := theCatalogue(t)
|
||||
ctx := t.Context()
|
||||
@@ -305,10 +270,11 @@ func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) {
|
||||
t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk })
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var forgeMerge keptMerge
|
||||
_ = json.Unmarshal(event, &forgeMerge)
|
||||
wasForge := keptForgeMerge
|
||||
t.Cleanup(func() { keptForgeMerge = wasForge })
|
||||
keptForgeMerge = func(context.Context, string, string, string) (keptMerge, bool, error) { return forgeMerge, true, nil }
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git",
|
||||
Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil {
|
||||
|
||||
@@ -7,14 +7,9 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not
|
||||
@@ -165,43 +160,3 @@ func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) {
|
||||
t.Fatalf("a send that asserted everything left the condition open: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Against a real bus, through the send's own grant: a module assigned after start has its consumer
|
||||
// once a declaration is sent, and a holder assigned after start its seat's worker.
|
||||
func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) {
|
||||
url := testbus.URL(t)
|
||||
open := aMesh(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} {
|
||||
_ = js.Context().DeleteStream(s)
|
||||
}
|
||||
// The controller starting, before either was assigned.
|
||||
if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
consumer := aCarriedConsumer(t, open)
|
||||
worker := aLateHolder(t, open)
|
||||
_ = js.Context().DeleteConsumer(worker.Stream, worker.Name)
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) {
|
||||
t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
|
||||
server := link.ConnectNats(js, nil, nil)
|
||||
if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range []broker.Consumer{consumer, worker} {
|
||||
if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil {
|
||||
t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err)
|
||||
}
|
||||
}
|
||||
if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised {
|
||||
t.Fatal("a send that asserted everything raised a condition")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,7 +54,6 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
||||
// The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build
|
||||
// (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes.
|
||||
// And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds).
|
||||
go copySentBuildsSoon(ctx, inv)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -62,39 +61,6 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link.
|
||||
// succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run.
|
||||
const kindSentBuildsNotCopied = "sent-builds-not-copied"
|
||||
|
||||
// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon
|
||||
// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails,
|
||||
// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key
|
||||
// that names a commit and empties each column it has copied.
|
||||
func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) {
|
||||
failed := false
|
||||
for {
|
||||
_, err := inv.CopySentBuilds(ctx)
|
||||
o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied,
|
||||
Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator,
|
||||
Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "}
|
||||
if err != nil {
|
||||
fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err)
|
||||
if keeper := conditionsFrom; keeper != nil && !failed {
|
||||
o.Summary += err.Error()
|
||||
if _, oerr := keeper.Observe(ctx, o); oerr == nil {
|
||||
failed = true
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if keeper := conditionsFrom; keeper != nil && failed {
|
||||
_, _ = keeper.Clear(ctx, o.Key(), "copied")
|
||||
}
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(chainMoveRetry):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// chainMoveRetry is how long the copy waits before it tries again.
|
||||
const chainMoveRetry = 30 * time.Second
|
||||
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// The links a rollout makes (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): the send of a plan's tier puts a
|
||||
// build on a machine; the assignment of the module there names the build, its commit and the rollout (written by the record of
|
||||
// the send, inventory.RecordSent, from the context a send is made in), and the rollout lists the assignments it changed with the
|
||||
// build before and after.
|
||||
|
||||
// rolloutModules is what a plan sends of each module: the commit and the build of it.
|
||||
func rolloutModules(p *inventory.Plan) map[string]inventory.RolloutModule {
|
||||
out := make(map[string]inventory.RolloutModule, len(p.Modules))
|
||||
for m, s := range p.Modules {
|
||||
if s != nil && s.Commit != "" {
|
||||
out[m] = inventory.RolloutModule{Commit: s.Commit, Build: s.Build}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// sendingAs is the context a plan's send is made in: what is recorded of the send names this rollout.
|
||||
func sendingAs(ctx context.Context, p *inventory.Plan) context.Context {
|
||||
return inventory.WithRollout(ctx, p.ID, rolloutModules(p))
|
||||
}
|
||||
|
||||
// ranBefore is what each module on a machine ran before a send, as the chain names it: its assignment's build and commit,
|
||||
// else the commit the machine was last sent for it. Read before the send, which writes the new ones.
|
||||
func ranBefore(ctx context.Context, inv *inventory.Inventory, node string) map[string]chain.Ref {
|
||||
was := map[string]chain.Ref{}
|
||||
if sent, known, err := inv.SentBuilds(ctx, node); err == nil && known {
|
||||
for m, commit := range sent {
|
||||
was[m] = chain.Ref{Commit: commit}
|
||||
}
|
||||
}
|
||||
if ran, err := inv.AssignmentRuns(ctx, node); err == nil {
|
||||
for m, r := range ran {
|
||||
was[m] = r
|
||||
}
|
||||
}
|
||||
return was
|
||||
}
|
||||
|
||||
// noteChanges writes into a plan that a send to a machine changed the assignments of the modules it carried: the build and
|
||||
// commit before and after, the builds the rollout carried, and the step taken. A change noted already for the machine and the
|
||||
// module is replaced, so a send made again says it once.
|
||||
func noteChanges(p *inventory.Plan, node string, modules []string, was map[string]chain.Ref, at time.Time) {
|
||||
for _, m := range modules {
|
||||
s := p.Modules[m]
|
||||
if s == nil {
|
||||
continue
|
||||
}
|
||||
change := chain.Change{Node: node, Module: m, Before: was[m], After: chain.Ref{Build: s.Build, Commit: s.Commit}}
|
||||
if at := slices.IndexFunc(p.Changed, func(c chain.Change) bool { return c.Node == node && c.Module == m }); at >= 0 {
|
||||
change.Before = p.Changed[at].Before
|
||||
p.Changed[at] = change
|
||||
} else {
|
||||
p.Changed = append(p.Changed, change)
|
||||
}
|
||||
if s.Build != "" && !slices.Contains(p.Builds, s.Build) {
|
||||
p.Builds = append(p.Builds, s.Build)
|
||||
}
|
||||
}
|
||||
if !slices.ContainsFunc(p.Steps, func(s chain.Step) bool { return s.Tier == p.Tier && s.Node == node }) {
|
||||
sent := at.UTC()
|
||||
p.Steps = append(p.Steps, chain.Step{Tier: p.Tier, Node: node, Sent: &sent})
|
||||
}
|
||||
}
|
||||
|
||||
// refSaid is a build and the commit it was made from as a person reads them; "nothing" for a module that ran none.
|
||||
func refSaid(r chain.Ref) string {
|
||||
switch {
|
||||
case r.Build == "" && r.Commit == "":
|
||||
return "nothing"
|
||||
case r.Build == "":
|
||||
return "commit " + short(r.Commit)
|
||||
}
|
||||
return r.Build + " (" + short(r.Commit) + ")"
|
||||
}
|
||||
|
||||
// changedLines say the assignments a rollout changed: the machine, the module, and the build before and after.
|
||||
func changedLines(p inventory.Plan) []string {
|
||||
if len(p.Changed) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := []string{" changed:"}
|
||||
for _, c := range p.Changed {
|
||||
out = append(out, fmt.Sprintf(" %s/%s %s → %s", c.Node, c.Module, refSaid(c.Before), refSaid(c.After)))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// runLines say what each module assigned to a machine runs there, as its assignment names it: the build, the commit, the
|
||||
// rollout that put it there and since when.
|
||||
func runLines(assigned []chain.Assignment) []string {
|
||||
var out []string
|
||||
for _, a := range assigned {
|
||||
if a.Build == "" && a.Commit == "" {
|
||||
continue
|
||||
}
|
||||
line := fmt.Sprintf(" %-22s %s", a.Module, refSaid(chain.Ref{Build: a.Build, Commit: a.Commit}))
|
||||
if a.Rollout != "" {
|
||||
line += ", sent by " + a.Rollout
|
||||
}
|
||||
if !a.Since.IsZero() {
|
||||
line += " at " + a.Since.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
return append([]string{" what they run, as the chain names it:"}, out...)
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/chain"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **A rollout lists the assignments it changed, each with the build before and after** (ADR 0408 §1, issue 1095): one
|
||||
// change for a machine and a module however often the send is noted, the build before kept from the first, the builds the
|
||||
// rollout carried once each, and one step for a machine and a tier.
|
||||
func TestARolloutListsTheAssignmentsItChangedWithTheBuildBeforeAndAfter(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
p := inventory.Plan{ID: "plan-1", Tier: 1, Modules: map[string]*inventory.PlanModule{
|
||||
"notes": {Commit: "c2", Build: "b2"}, "wiki": {Commit: "c2w", Build: "b2w"}, "nothing": nil}}
|
||||
was := map[string]chain.Ref{"notes": {Build: "b1", Commit: "c1"}, "wiki": {Commit: "c1w"}}
|
||||
|
||||
noteChanges(&p, "anchor", []string{"notes", "wiki", "nothing"}, was, at)
|
||||
noteChanges(&p, "laptop", []string{"notes"}, nil, at)
|
||||
// The send noted again, after what was sent had become what the machine runs: the first before stands.
|
||||
noteChanges(&p, "anchor", []string{"notes"}, map[string]chain.Ref{"notes": {Build: "b2", Commit: "c2"}}, at.Add(time.Minute))
|
||||
|
||||
want := []chain.Change{
|
||||
{Node: "anchor", Module: "notes", Before: chain.Ref{Build: "b1", Commit: "c1"}, After: chain.Ref{Build: "b2", Commit: "c2"}},
|
||||
{Node: "anchor", Module: "wiki", Before: chain.Ref{Commit: "c1w"}, After: chain.Ref{Build: "b2w", Commit: "c2w"}},
|
||||
{Node: "laptop", Module: "notes", After: chain.Ref{Build: "b2", Commit: "c2"}},
|
||||
}
|
||||
if len(p.Changed) != len(want) {
|
||||
t.Fatalf("changed %+v", p.Changed)
|
||||
}
|
||||
for k := range want {
|
||||
if p.Changed[k] != want[k] {
|
||||
t.Errorf("change %d is %+v, want %+v", k, p.Changed[k], want[k])
|
||||
}
|
||||
}
|
||||
if len(p.Builds) != 2 || p.Builds[0] != "b2" || p.Builds[1] != "b2w" {
|
||||
t.Errorf("builds %v", p.Builds)
|
||||
}
|
||||
if len(p.Steps) != 2 || p.Steps[0].Node != "anchor" || p.Steps[0].Tier != 1 || p.Steps[0].Sent == nil || p.Steps[1].Node != "laptop" {
|
||||
t.Errorf("steps %+v", p.Steps)
|
||||
}
|
||||
lines := changedLines(p)
|
||||
if len(lines) != 4 || !strings.Contains(lines[1], "anchor/notes b1 (c1) → b2 (c2)") ||
|
||||
!strings.Contains(lines[3], "laptop/notes nothing → b2 (c2)") {
|
||||
t.Errorf("changed lines %q", lines)
|
||||
}
|
||||
if got := changedLines(inventory.Plan{}); got != nil {
|
||||
t.Errorf("a plan that changed nothing says %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhatAMachineRunsIsSaidAsItsAssignmentNamesIt(t *testing.T) {
|
||||
at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC)
|
||||
lines := runLines([]chain.Assignment{
|
||||
{Node: "anchor", Module: "notes", Build: "b2", Commit: "c2c2c2c2c2c2", Rollout: "plan-1", Since: at},
|
||||
{Node: "anchor", Module: "web", Commit: "c3"},
|
||||
{Node: "anchor", Module: "never-sent"},
|
||||
})
|
||||
if len(lines) != 3 || !strings.Contains(lines[1], "notes") || !strings.Contains(lines[1], "b2 (c2c2c2c2)") ||
|
||||
!strings.Contains(lines[1], "sent by plan-1") || !strings.Contains(lines[2], "web") || !strings.Contains(lines[2], "commit c3") {
|
||||
t.Fatalf("lines %q", lines)
|
||||
}
|
||||
if runLines([]chain.Assignment{{Node: "a", Module: "m"}}) != nil {
|
||||
t.Fatal("a module never sent is said to run something")
|
||||
}
|
||||
}
|
||||
@@ -248,25 +248,11 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento
|
||||
if out.Epoch, err = inv.SentEpoch(ctx, n.ID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.Generation, err = inv.SentGeneration(ctx, n.ID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if out.ReadsModules, err = inv.ReadsModules(ctx, n.ID); err != nil {
|
||||
return out, err
|
||||
}
|
||||
// Every node-engine of release 6 reads a module per resource and a build per module.
|
||||
out.ReadsModules = true
|
||||
if out.SentBuilds, out.SentKnown, err = inv.SentBuilds(ctx, n.Name); err != nil {
|
||||
return out, err
|
||||
}
|
||||
runs, err := inv.AssignmentRuns(ctx, n.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
if len(runs) > 0 {
|
||||
out.Runs = make(map[string]compose.Run, len(runs))
|
||||
for module, ref := range runs {
|
||||
out.Runs[module] = compose.Run{Commit: ref.Commit, Build: ref.Build}
|
||||
}
|
||||
}
|
||||
stated, has, err := inv.HealthOf(ctx, n.Name)
|
||||
if err != nil {
|
||||
return out, err
|
||||
|
||||
@@ -243,54 +243,6 @@ func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696
|
||||
}
|
||||
}
|
||||
|
||||
// fallbackDelivery refuses the first declaration it is sent with the bucket's failure, as a bus that has not yet been given the
|
||||
// controller's grant does, and takes the second.
|
||||
type fallbackDelivery struct {
|
||||
bodies [][]byte
|
||||
first error
|
||||
}
|
||||
|
||||
func (f *fallbackDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *fallbackDelivery) declare(_ context.Context, _ readyNode, body []byte) (string, error) {
|
||||
f.bodies = append(f.bodies, body)
|
||||
if len(f.bodies) == 1 && f.first != nil {
|
||||
return "", fmt.Errorf("x: %w: %w", errFilesNotPut, f.first)
|
||||
}
|
||||
return "digest", nil
|
||||
}
|
||||
|
||||
// 1696/AC5: when the bucket cannot be written yet (the grant travels in the declaration of the machine holding the bus), that send goes
|
||||
// with every content inline and is made; a send that fails for another reason is not retried inline.
|
||||
func TestASendWhoseFilesCannotBePutGoesInlineOnce(t *testing.T) { // 1696/AC5
|
||||
users := contentOf("users", 100<<10)
|
||||
s := readyNode{node: "novox", declared: sendable{fileReferences: true, Resources: []map[string]any{aFileResource("nats.bus-users", users)}}}
|
||||
f := &fallbackDelivery{first: errors.New("timeout")}
|
||||
digests, err := deliver(t.Context(), f, "novox", []readyNode{s})
|
||||
if err != nil || digests["novox"] != "digest" || len(f.bodies) != 2 {
|
||||
t.Fatalf("%v %v %d", digests, err, len(f.bodies))
|
||||
}
|
||||
if strings.Contains(string(f.bodies[0]), users[:100]) || !strings.Contains(string(f.bodies[1]), users[:100]) {
|
||||
t.Error("the first body should name the file by digest and the second carry it inline")
|
||||
}
|
||||
other := &fallbackDelivery{}
|
||||
other.first = nil
|
||||
failing := &failingDelivery{err: errors.New("the bus is down")}
|
||||
if _, err := deliver(t.Context(), failing, "novox", []readyNode{s}); err == nil || failing.calls != 1 {
|
||||
t.Fatalf("a failure of another kind: %v after %d send(s)", err, failing.calls)
|
||||
}
|
||||
}
|
||||
|
||||
type failingDelivery struct {
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *failingDelivery) grant(context.Context, []readyNode) error { return nil }
|
||||
func (f *failingDelivery) declare(context.Context, readyNode, []byte) (string, error) {
|
||||
f.calls++
|
||||
return "", f.err
|
||||
}
|
||||
|
||||
// A bucket for the sweep.
|
||||
type sweepBucketFake struct {
|
||||
objects map[string]time.Time // zero: named at the last look
|
||||
@@ -502,33 +454,3 @@ func whileHeld(node string) bool {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// 1696/AC9: a send holds the node's lock from the put to the publication, so the sweep never removes an object between them. The
|
||||
// test goes through overTheBus.declare: while another holds the lock the declaration is not published, and at the publication the
|
||||
// lock is held.
|
||||
func TestASendHoldsTheNodesLockFromThePutToThePublication(t *testing.T) { // 1696/AC9
|
||||
const node = "lock-send-node"
|
||||
bus := &aBusThatSays{published: make(chan string, 1)}
|
||||
bus.heldAtPublish = func() bool { return whileHeld(node) }
|
||||
send := overTheBus{server: link.SendingOn(bus), signer: aSignerOfZeros{}}
|
||||
release := lockFilesOf(node) // a sweep is judging this node's bucket
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := send.declare(t.Context(), readyNode{node: node}, []byte(`{}`))
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case n := <-bus.published:
|
||||
t.Fatalf("%s was sent its declaration while the sweep held the node's bucket", n)
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case <-bus.published:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("the send did not go on once the sweep let go")
|
||||
}
|
||||
if err := <-done; err == nil || !strings.Contains(err.Error(), "nothing is recorded") {
|
||||
t.Fatalf("the send did not reach the publication (a lock held at it): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What the core's bounds are set from (novox/hq to-be 45 Phase 0).
|
||||
//
|
||||
// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog
|
||||
// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times
|
||||
// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional
|
||||
// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving
|
||||
// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a
|
||||
// build's outcome — and summarised here per machine, repository or module.
|
||||
|
||||
// durationSummary is one subject's measurements of one kind.
|
||||
type durationSummary struct {
|
||||
Kind string `json:"kind"`
|
||||
Subject string `json:"subject"`
|
||||
Count int `json:"count"`
|
||||
Median string `json:"median"`
|
||||
P90 string `json:"p90"`
|
||||
Max string `json:"max"`
|
||||
// Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured
|
||||
// time: three times the slowest apply (S2), three times the median word interval (S1).
|
||||
Suggests string `json:"suggests,omitempty"`
|
||||
}
|
||||
|
||||
func summarise(ds []inventory.Duration) []durationSummary {
|
||||
type key struct{ kind, subject string }
|
||||
by := map[key][]time.Duration{}
|
||||
for _, d := range ds {
|
||||
k := key{d.Kind, d.Subject}
|
||||
by[k] = append(by[k], d.Took)
|
||||
}
|
||||
var out []durationSummary
|
||||
for k, took := range by {
|
||||
slices.Sort(took)
|
||||
at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] }
|
||||
s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took),
|
||||
Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])}
|
||||
switch k.kind {
|
||||
case inventory.DurationApply:
|
||||
s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90"
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind)
|
||||
if ki != kj {
|
||||
return ki < kj
|
||||
}
|
||||
return out[i].Subject < out[j].Subject
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func round(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Second:
|
||||
return d.Round(time.Millisecond).String()
|
||||
case d < time.Minute:
|
||||
return d.Round(100 * time.Millisecond).String()
|
||||
default:
|
||||
return d.Round(time.Second).String()
|
||||
}
|
||||
}
|
||||
|
||||
// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data.
|
||||
func durationsCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("durations", flag.ContinueOnError)
|
||||
kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", "))
|
||||
days := set.Int("days", 14, "how many days back")
|
||||
asJSON := set.Bool("json", false, "the summary as data")
|
||||
all := set.Bool("all", false, "every measurement rather than the summary")
|
||||
if _, err := parseAround(set, args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) {
|
||||
return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", "))
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *all {
|
||||
body, err := json.MarshalIndent(ds, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
summary := summarise(ds)
|
||||
if *asJSON {
|
||||
body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
if len(summary) == 0 {
|
||||
fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, "+
|
||||
"plan-tier and build durations as it hears them\n", *days)
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days)
|
||||
fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max")
|
||||
for _, s := range summary {
|
||||
fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max)
|
||||
if s.Suggests != "" {
|
||||
fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// forgettingOldDurations removes what is older than a month, at start and daily after.
|
||||
func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) {
|
||||
for {
|
||||
if n, err := inv.ForgetOldDurations(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err)
|
||||
} else if n > 0 {
|
||||
fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor)
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(24 * time.Hour):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,266 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
// The assignment generation a declaration was composed from, and the record of every send (novox/hq
|
||||
// issue 234).
|
||||
//
|
||||
// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the
|
||||
// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and
|
||||
// cannot tell a later send that carries an older view of the assignments. So a declaration now carries
|
||||
// the generation of the assignments it was composed from — a counter the store raises in the same
|
||||
// transaction as every assignment change — and a machine refuses one older than it applied, unless it is
|
||||
// a gate's put-back. And every send is written down with who sent it, from which generation and naming
|
||||
// which modules: the controller logged no send then, and which process sent the stale declaration could
|
||||
// not be read back from anything the mesh kept.
|
||||
|
||||
// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from.
|
||||
const kindOlderGeneration = "older-generation"
|
||||
|
||||
// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is.
|
||||
const generationRefusalsSaid = advisoryQuiet
|
||||
|
||||
// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and
|
||||
// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the
|
||||
// record of the send whether or not the machine is sent them.
|
||||
//
|
||||
// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its
|
||||
// declaration is allotted here like any other and carries the generation as it stands — never older than
|
||||
// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never
|
||||
// needs to send one.
|
||||
func (o order) stamp(d *sendable) {
|
||||
d.Sequence, d.Epoch = o.sequence, o.epoch
|
||||
d.composedFrom, d.actingEpoch = o.generation, o.acting
|
||||
d.Generation = 0
|
||||
if o.readsGeneration && o.generation > 0 {
|
||||
d.Generation = o.generation
|
||||
}
|
||||
}
|
||||
|
||||
// sentRecord is what the record of a send keeps beside its digest.
|
||||
type sentRecord struct {
|
||||
sequence int64
|
||||
epoch uint64
|
||||
generation int64
|
||||
// toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine
|
||||
// whose node-engine has not said it reads one.
|
||||
toldGeneration int64
|
||||
sender string
|
||||
modules []string
|
||||
// plans is each module the send handed to the provisioner, its plan, as JSON (design 57 P6); nil for none.
|
||||
plans []byte
|
||||
}
|
||||
|
||||
// sentRecordOf is a composed send's record: its sender is the caller this process acts for.
|
||||
func sentRecordOf(ctx context.Context, d sendable) sentRecord {
|
||||
r := sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom,
|
||||
toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules}
|
||||
if len(d.plans) > 0 {
|
||||
kept := map[string]keptPlan{}
|
||||
for module, p := range d.plans {
|
||||
kept[module] = keptPlan{Plan: p, RequiredBy: d.requiredBy[module]}
|
||||
}
|
||||
// A map of plain strings and slices: it marshals.
|
||||
r.plans, _ = json.Marshal(kept)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// keptPlan is a handed module's plan as a send keeps it (migration 0100): the plan, and for a module only pulled in
|
||||
// by another, the module that requires it (issue 765, inventory.SentProvisioningPlans).
|
||||
type keptPlan struct {
|
||||
catalogue.Plan
|
||||
RequiredBy string `json:"required-by,omitempty"`
|
||||
}
|
||||
|
||||
// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the
|
||||
// caller the controller ran this command for, or the account at the shell.
|
||||
func callerOf(ctx context.Context) string {
|
||||
if c := link.CallerIn(ctx); c != "" {
|
||||
return c
|
||||
}
|
||||
return link.Caller()
|
||||
}
|
||||
|
||||
// senderOf is a send's sender in words: who asked, and the process and build that composed it — the
|
||||
// answer issue 234 could not find, because two controllers and a one-shot push were all sending then.
|
||||
func senderOf(caller string) string {
|
||||
host, _ := os.Hostname()
|
||||
return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version)
|
||||
}
|
||||
|
||||
// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender
|
||||
// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that
|
||||
// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the
|
||||
// mesh cannot attribute is the louder fact, not a quieter one.
|
||||
//
|
||||
// **And raises the mesh's counter past what the machine applied, when the refused send was composed from
|
||||
// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind
|
||||
// the machine, which is a store put back from a backup — and every send after would be refused for ever. A
|
||||
// stale sender's generation is below the counter, and the counter is left alone for it.
|
||||
//
|
||||
// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that
|
||||
// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185.
|
||||
// S20 names `push <node>` for that case instead.
|
||||
func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) {
|
||||
r := report.OlderGeneration
|
||||
if r == nil || inv == nil || report.Node == "" {
|
||||
return
|
||||
}
|
||||
node, err := inv.NodeByName(ctx, report.Node)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+
|
||||
"read, so it is not raised: %v\n", report.Node, err)
|
||||
return
|
||||
}
|
||||
var raised int64
|
||||
if now, err := inv.AssignmentGeneration(ctx); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+
|
||||
"read: %v\n", report.Node, err)
|
||||
} else if counterBehind(*r, now) {
|
||||
if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+
|
||||
"and could not be raised: %v\n", now, report.Node, r.Applied, err)
|
||||
raised = 0
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+
|
||||
"a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n",
|
||||
now, report.Node, r.Applied, raised, report.Node)
|
||||
}
|
||||
}
|
||||
send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised)
|
||||
if err == nil && !found {
|
||||
send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence,
|
||||
Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied,
|
||||
CounterRaisedTo: raised})
|
||||
}
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+
|
||||
"kept, so it is not raised: %v\n", report.Node, report.Sequence, err)
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+
|
||||
"and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied)
|
||||
}
|
||||
|
||||
// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the
|
||||
// refused send carried the counter as it stands now, and the machine applied more than that.
|
||||
func counterBehind(r link.GenerationRefusal, now int64) bool {
|
||||
return r.Generation >= now && r.Applied > r.Generation
|
||||
}
|
||||
|
||||
// modulesWords is a send's modules as a sentence says them.
|
||||
func modulesWords(modules []string) string {
|
||||
if len(modules) == 0 {
|
||||
return "no module the mesh recorded"
|
||||
}
|
||||
return strings.Join(modules, ", ")
|
||||
}
|
||||
|
||||
// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and
|
||||
// what it answered (issue 485): nothing when the mesh has not recorded a send to it.
|
||||
func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error {
|
||||
s, found, err := inv.LastSend(ctx, node)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(w, "%s:\n", node)
|
||||
for _, line := range lastSendLines(s) {
|
||||
fmt.Fprintln(w, line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when,
|
||||
// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the
|
||||
// machine answered of it — the refusal in its own words.
|
||||
func lastSendLines(s inventory.Send) []string {
|
||||
const stamp = "2006-01-02 15:04:05"
|
||||
epoch := "under no lease epoch"
|
||||
if s.Epoch > 0 {
|
||||
epoch = fmt.Sprintf("under lease epoch %d", s.Epoch)
|
||||
}
|
||||
generation := "no assignment generation recorded"
|
||||
if s.Generation > 0 {
|
||||
generation = fmt.Sprintf("assignment generation %d", s.Generation)
|
||||
}
|
||||
lines := []string{
|
||||
fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)),
|
||||
fmt.Sprintf(" by %s", s.Sender),
|
||||
fmt.Sprintf(" %s, composed from %s", epoch, generation),
|
||||
fmt.Sprintf(" naming %s", modulesWords(s.Modules)),
|
||||
}
|
||||
answer := func(words string) { lines = append(lines, " its answer "+words) }
|
||||
switch a := s.Answer; {
|
||||
case s.RefusedAt != nil:
|
||||
// Before the report: a refusal for the generation is kept on the send itself and is the machine's
|
||||
// answer to this send in particular.
|
||||
answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one",
|
||||
s.RefusedAt.Local().Format(stamp), s.RefusedApplied))
|
||||
case a.Outcome == "" || a.At == nil:
|
||||
answer("the machine has not reported on it yet")
|
||||
case a.Outcome == inventory.OutcomeApplied:
|
||||
answer("applied it at " + a.At.Local().Format(stamp))
|
||||
case a.Outcome == inventory.OutcomeRefused:
|
||||
answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:")
|
||||
for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") {
|
||||
lines = append(lines, " "+strings.TrimSpace(line))
|
||||
}
|
||||
default:
|
||||
answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome,
|
||||
a.At.Local().Format(stamp), a.Failed))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line.
|
||||
func sendAnswerWord(s inventory.Send) string {
|
||||
switch a := s.Answer; {
|
||||
case s.RefusedAt != nil:
|
||||
return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied)
|
||||
case a.Outcome == "" || a.At == nil:
|
||||
return "not reported on yet"
|
||||
case a.Outcome == inventory.OutcomeFailed:
|
||||
return fmt.Sprintf("failed (%d resource(s))", a.Failed)
|
||||
default:
|
||||
return a.Outcome
|
||||
}
|
||||
}
|
||||
|
||||
// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment
|
||||
// generation and what it answered (novox/hq issue 485). `node show <node>` says the send in full.
|
||||
func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) {
|
||||
if unread != "" {
|
||||
fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread)
|
||||
return
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
return
|
||||
}
|
||||
fmt.Println("last sent, per machine:")
|
||||
for _, n := range nodes {
|
||||
s, found := sends[n.Name]
|
||||
if !found {
|
||||
fmt.Printf(" %-12s no send recorded\n", n.Name)
|
||||
continue
|
||||
}
|
||||
generation := "no generation"
|
||||
if s.Generation > 0 {
|
||||
generation = fmt.Sprintf("generation %d", s.Generation)
|
||||
}
|
||||
fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender,
|
||||
generation, sendAnswerWord(s))
|
||||
}
|
||||
fmt.Printf("\n `node show <node>` says a machine's last send in full\n\n")
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// 1394/AC1 of C9 (ADR 0528 clause 2): when the controller has issued mesh-x's credential for fingerprint F (mesh-x
|
||||
// registered at a build of F, its account minted, the bus's machine sent its declaration with the user list, through
|
||||
// overTheBus.declare as a push sends it), then grants.mesh-x.F exists on the bus,
|
||||
// and a second pass writes it again as nothing new.
|
||||
func TestTheGrantOfAnIssuedCredentialIsOnTheBus(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
fp := grantFP('d')
|
||||
m := catalogue.Manifest{Module: "mesh-x", Version: "1", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh-x/broker"}}}
|
||||
if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-x1", Repository: "novox/mesh-x", Module: "mesh-x", Commit: "c0ffee1",
|
||||
SourceFingerprint: fp}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-x", BuiltFrom: "c0ffee1", Head: "c0ffee1"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := assign(ctx, open, "anchor", "mesh-x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
user := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: "mesh-x"}.Username()
|
||||
if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule, Node: "anchor", Module: "mesh-x"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn, err := nats.Connect(testbus.URL(t))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
js, err := jetstream.New(conn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := broker.OnConn(conn).EnsureControllerBuckets(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The live path: a send of the bus's machine's declaration carrying the user list (overTheBus.declare), which records
|
||||
// the list sent and then writes the grants it issued.
|
||||
legacy, err := conn.JetStream()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = js.DeleteStream(ctx, "GRANTS_DECLARED")
|
||||
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "GRANTS_DECLARED", Subjects: []string{link.DeclareSubject("anchor")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = js.DeleteStream(context.Background(), "GRANTS_DECLARED") })
|
||||
send := overTheBus{open: open, server: link.SendingOn(link.OverNATS{Conn: conn, JS: legacy}), signer: aSignerOfZeros{}}
|
||||
ready := readyNode{node: "anchor", declared: sendable{BusUsers: "the user list the bus's machine is sent"}}
|
||||
if _, err := send.declare(ctx, ready, []byte(`{}`)); err != nil {
|
||||
t.Fatalf("the send of the bus's machine: %v", err)
|
||||
}
|
||||
kv, err := js.KeyValue(ctx, broker.GrantsBucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := sdkdelivery.ControllerGrantKey("mesh-x", fp)
|
||||
e, err := kv.Get(ctx, key)
|
||||
if err != nil {
|
||||
t.Fatalf("no grant %s after the list was issued: %v", key, err)
|
||||
}
|
||||
g, err := sdkdelivery.DecodeControllerGrant(e.Value())
|
||||
if err != nil || len(g.Accounts) != 1 || g.Accounts[0] != user {
|
||||
t.Fatalf("the grant reads %+v, %v; want the account %s", g, err, user)
|
||||
}
|
||||
if _, err := send.declare(ctx, ready, []byte(`{}`)); err != nil {
|
||||
t.Fatalf("the second send: %v", err)
|
||||
}
|
||||
if again, _ := kv.Get(ctx, key); again.Revision() != e.Revision() {
|
||||
t.Errorf("a second pass wrote the grant again (revision %d, was %d)", again.Revision(), e.Revision())
|
||||
}
|
||||
}
|
||||
@@ -4,9 +4,6 @@ import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// With a reason, the seat passes it to the command, and a verb that only reads is not held to one.
|
||||
@@ -27,20 +24,6 @@ func TestARepairByHandCarriesItsReason(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// The summary of durations says, per kind and subject, what a bound would be set from.
|
||||
func TestDurationsAreSummarisedPerSubject(t *testing.T) {
|
||||
var ds []inventory.Duration
|
||||
for i := 1; i <= 10; i++ {
|
||||
ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor",
|
||||
Took: time.Duration(i) * time.Second})
|
||||
}
|
||||
got := summarise(ds)
|
||||
if len(got) != 1 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 ||
|
||||
got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses
|
||||
// the cause, so a repair cannot pass for a drill by the word it gives.
|
||||
func TestADrillIsRecordedThroughItsOwnVerb(t *testing.T) {
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// heldKey carries the nodes this call already holds, so an act that holds a node and then sends
|
||||
// through sendTo does not wait on itself.
|
||||
type heldKey struct{}
|
||||
|
||||
// holdNodes holds the named nodes for composing and sending their declarations (novox/hq ADR
|
||||
// 0100), skipping any the context already holds, and returns a context that says it holds them.
|
||||
// Release gives back only what this call took.
|
||||
func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) {
|
||||
already, _ := ctx.Value(heldKey{}).(map[string]bool)
|
||||
var take []string
|
||||
for _, n := range names {
|
||||
if !already[n] {
|
||||
take = append(take, n)
|
||||
}
|
||||
}
|
||||
if len(take) == 0 {
|
||||
return ctx, func() {}, nil
|
||||
}
|
||||
release, err := open.inventory.HoldNodes(ctx, take)
|
||||
if err != nil {
|
||||
return ctx, nil, err
|
||||
}
|
||||
held := map[string]bool{}
|
||||
for n := range already {
|
||||
held[n] = true
|
||||
}
|
||||
for _, n := range take {
|
||||
held[n] = true
|
||||
}
|
||||
return context.WithValue(ctx, heldKey{}, held), release, nil
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 348: on 2026-10-09 the control node's resolver stopped answering on its private address
|
||||
// at 10:57:57 UTC; the machine's network condition was raised at 10:58:45. The node-engine and the
|
||||
// controller were sent at 10:59:34. The new node-engine's first statement judged the names once — unknown,
|
||||
// "one look failed; a second decides" — and that statement cleared the condition, though its last evidence
|
||||
// still said "connection refused". The next look raised it again at 11:00:23, after the send, and both
|
||||
// builds failed their gate at 11:10 with "raised since it was sent" and were put back, for a fault that
|
||||
// began before they were sent.
|
||||
|
||||
// TestReplay348 replays the statements of the outage: the condition raised before the send is not
|
||||
// cleared by the restarted engine's first, undecided statement, and the gate does not count it against
|
||||
// the builds sent after it began.
|
||||
|
||||
// A release walks its modules without a record per module: D10 counts what its tier names as rolling,
|
||||
// so the node-engine a release walks is not "behind, and no plan is rolling it out" on its first machine.
|
||||
func TestAReleaseRollsOutWhatItsTierNames(t *testing.T) {
|
||||
plans := []inventory.Plan{
|
||||
{ID: "release-1", State: inventory.PlanRolling, Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{}},
|
||||
{ID: "plan-2", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"letta": {}}},
|
||||
}
|
||||
got := rollingModules(plans)
|
||||
if !got["mesh-host"] || !got["letta"] || len(got) != 2 {
|
||||
t.Fatalf("rolling: %v", got)
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) {
|
||||
inv := inventory.ForTest(t)
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cancel() // the sender is going away: its context is cancelled between the send and the record
|
||||
body := []byte(`{"declaration":1,"resources":[]}`)
|
||||
digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"})
|
||||
if err != nil {
|
||||
// NodeByName on the cancelled context may itself refuse; the record must still be possible
|
||||
// through the detached context, so look the node up again on a live one.
|
||||
t.Fatalf("recording a send after cancellation failed: %v", err)
|
||||
}
|
||||
outstanding, err := inv.Outstanding(t.Context(), "anchor")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if outstanding != digest || digest != digestOf(body) {
|
||||
t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest)
|
||||
}
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// A declaration composed earlier is numbered lower than one composed later, whatever order the two
|
||||
// are sent in (novox/hq issue 204). The number used to be taken at send time, after composing, so a
|
||||
// declaration composed before an assignment changed and sent after a newer one carried the higher
|
||||
// number — and the machine, which refuses a lower number, took the older content as the mesh's
|
||||
// newest word. Taken before the composition reads anything, the order of numbers is the order of
|
||||
// compositions, and the host's refusal does what it is for.
|
||||
func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *testing.T) {
|
||||
allot := numbered()
|
||||
var composed []string
|
||||
compose := func(stamp string) func(string) (sendable, error) {
|
||||
return func(node string) (sendable, error) {
|
||||
composed = append(composed, stamp)
|
||||
return sendable{Resources: []map[string]any{{"id": node + "." + stamp}}}, nil
|
||||
}
|
||||
}
|
||||
// Composed first — before an assignment changed — and sent last.
|
||||
stale, _ := composeEach([]string{"anchor"}, allot, compose("before"))
|
||||
// Composed after the change, sent first.
|
||||
fresh, _ := composeEach([]string{"anchor"}, allot, compose("after"))
|
||||
|
||||
if stale[0].declared.Sequence != 1 || fresh[0].declared.Sequence != 2 {
|
||||
t.Fatalf("the numbers do not follow the compositions: before=%d after=%d",
|
||||
stale[0].declared.Sequence, fresh[0].declared.Sequence)
|
||||
}
|
||||
// Sent in the other order, the numbers do not change — so the machine that has applied the
|
||||
// fresh one (2) refuses the stale one (1) when it arrives late.
|
||||
if !(stale[0].declared.Sequence < fresh[0].declared.Sequence) {
|
||||
t.Fatal("a declaration composed earlier must carry the lower number, however late it is sent")
|
||||
}
|
||||
if len(composed) != 2 || composed[0] != "before" {
|
||||
t.Fatalf("compositions happened in an unexpected order: %v", composed)
|
||||
}
|
||||
}
|
||||
|
||||
// The number is taken before the first read of the composition, not after it: an allotter that
|
||||
// fails leaves nothing composed for that machine, and the others are still composed.
|
||||
func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) {
|
||||
calls := 0
|
||||
allot := func(node string) (order, error) {
|
||||
if node == "anchor" {
|
||||
return order{}, context.DeadlineExceeded
|
||||
}
|
||||
return order{sequence: 7}, nil
|
||||
}
|
||||
sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) {
|
||||
calls++
|
||||
if node == "anchor" {
|
||||
t.Fatal("anchor was composed although its number could not be taken")
|
||||
}
|
||||
return sendable{}, nil
|
||||
})
|
||||
if calls != 1 || len(sending) != 1 || sending[0].node != "laptop" || sending[0].declared.Sequence != 7 {
|
||||
t.Fatalf("laptop should be composed with its number and anchor refused: %v / %v", sending, refusals)
|
||||
}
|
||||
if len(refusals) != 1 {
|
||||
t.Fatalf("anchor's failed number should be a refusal naming it: %v", refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// What was sent is written down even when the sender's context is already cancelled (issue 204): a
|
||||
// controller replaced mid-send had told the machine and never recorded it, so status read "applied,
|
||||
// current" over a machine that had just been sent something else.
|
||||
@@ -1,51 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
record, err := open.inventory.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12,
|
||||
Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
||||
Modules: []string{"docker", "sudo"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused,
|
||||
Declared: "d12", Refused: "unknown field \"generation\""}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
shown := printed(t, func() error { return showNode(ctx, open, "laptop") })
|
||||
for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40",
|
||||
"docker, sudo", "unknown field \"generation\""} {
|
||||
if !strings.Contains(shown, want) {
|
||||
t.Errorf("node show laptop does not say %q:\n%s", want, shown)
|
||||
}
|
||||
}
|
||||
body := printed(t, func() error { return statusFor(ctx, open, true) })
|
||||
var got struct {
|
||||
LastSent []struct {
|
||||
Node string `json:"node"`
|
||||
Sender string `json:"sender"`
|
||||
Answer string `json:"answer"`
|
||||
} `json:"lastSent"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &got); err != nil {
|
||||
t.Fatalf("%v:\n%s", err, body)
|
||||
}
|
||||
if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" ||
|
||||
got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" {
|
||||
t.Fatalf("status --json's last sends read %+v", got.LastSent)
|
||||
}
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan`
|
||||
// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show`
|
||||
// says the last send in full and what the machine answered; and `status --json` carries it as fields, since
|
||||
// the store's reader may not read the controller's tables and these verbs are the only window onto them.
|
||||
|
||||
func aLastSend(answer inventory.SendAnswer) inventory.Send {
|
||||
sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local)
|
||||
return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57,
|
||||
Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12",
|
||||
Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer}
|
||||
}
|
||||
|
||||
func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
||||
asked := answers{
|
||||
nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}},
|
||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{
|
||||
Outcome: inventory.OutcomeApplied, At: &at})},
|
||||
}
|
||||
shown := printed(t, func() error { return printStatus(asked) })
|
||||
var laptop, anchor []string
|
||||
for _, line := range strings.Split(shown, "\n") {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) > 0 && fields[0] == "laptop" {
|
||||
laptop = append(laptop, line)
|
||||
}
|
||||
if len(fields) > 0 && fields[0] == "anchor" {
|
||||
anchor = append(anchor, line)
|
||||
}
|
||||
}
|
||||
if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") ||
|
||||
!strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") ||
|
||||
!strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") {
|
||||
t.Fatalf("status does not say laptop's last send in one line:\n%s", shown)
|
||||
}
|
||||
if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") {
|
||||
t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) {
|
||||
asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}},
|
||||
lastSentUnread: "the store went away"}
|
||||
if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") {
|
||||
t.Fatalf("a record of sends that could not be read is not said:\n%s", shown)
|
||||
}
|
||||
if asked.well() {
|
||||
t.Error("a mesh whose sends could not be read is called well")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeShowSaysTheLastSendInFull(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local)
|
||||
refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
||||
Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n")
|
||||
for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor",
|
||||
"lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} {
|
||||
if !strings.Contains(refused, want) {
|
||||
t.Errorf("node show's last send does not say %q:\n%s", want, refused)
|
||||
}
|
||||
}
|
||||
if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied,
|
||||
At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") {
|
||||
t.Errorf("an applied send is not said applied:\n%s", applied)
|
||||
}
|
||||
if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting,
|
||||
"not reported on it yet") {
|
||||
t.Errorf("a send not reported on is not said so:\n%s", waiting)
|
||||
}
|
||||
stale := aLastSend(inventory.SendAnswer{})
|
||||
when := at
|
||||
stale.RefusedAt, stale.RefusedApplied = &when, 44
|
||||
if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") ||
|
||||
!strings.Contains(s, "generation 44") {
|
||||
t.Errorf("a send refused for its generation is not said so:\n%s", s)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) {
|
||||
at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC)
|
||||
asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}},
|
||||
lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused,
|
||||
Refused: "unknown field", At: &at})}}
|
||||
body, err := statusAsJSON(asked)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got struct {
|
||||
LastSent []map[string]any `json:"lastSent"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.LastSent) != 1 {
|
||||
t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body)
|
||||
}
|
||||
s := got.LastSent[0]
|
||||
if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) ||
|
||||
s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" ||
|
||||
s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" ||
|
||||
s["answeredAt"] == nil {
|
||||
t.Fatalf("laptop's last send reads %v", s)
|
||||
}
|
||||
if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" {
|
||||
t.Fatalf("the modules it named read %v", s["modules"])
|
||||
}
|
||||
}
|
||||
|
||||
// And through the store: a send recorded and refused by the machine is what `node show` and `status --json`
|
||||
// read back.
|
||||
@@ -135,11 +135,6 @@ func TestTwoControllersOneActs(t *testing.T) {
|
||||
if _, err := inv.AddNode(ctx, "anchor"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(),
|
||||
State: inventory.PlanBuilding}
|
||||
if err := inv.SavePlan(ctx, &saved); err == nil {
|
||||
t.Fatal("B wrote a plan after losing the lease")
|
||||
}
|
||||
store := conditions.NewInMemory()
|
||||
keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store,
|
||||
Epoch: func() (uint64, error) { return b.epoch(ctx) }})
|
||||
|
||||
@@ -86,7 +86,7 @@ func run() error {
|
||||
case "builds":
|
||||
return buildsCommand(ctx, args[1:])
|
||||
case "forget-checks", "queue", "cancel", "clear", "kill", "replay", "pause", "resume", "plans", "plan", "push", "deliver",
|
||||
"delivery-stop", "delivery-backups-again", "delivery-walks", "delivery-order", "delivery", "upgrade":
|
||||
"delivery-stop", "delivery-backups-again", "delivery-walks", "delivery-order", "delivery":
|
||||
return removedByName(args[0])
|
||||
case "try":
|
||||
return tryCommand(ctx, args[1:])
|
||||
@@ -128,6 +128,8 @@ func run() error {
|
||||
case "facts":
|
||||
return factsCommand(ctx, args[1:])
|
||||
// The bus as a planned step (novox/hq to-be 45 §8, ADR 0236).
|
||||
case "upgrade":
|
||||
return upgradeCommand(ctx, args[1:])
|
||||
case "bus":
|
||||
return busCommand(ctx, args[1:])
|
||||
case "overlay":
|
||||
@@ -163,7 +165,7 @@ func run() error {
|
||||
return handActCommand(ctx, append([]string{"list"}, args[1:]...))
|
||||
// What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0).
|
||||
case "durations":
|
||||
return durationsCommand(ctx, args[1:])
|
||||
return removedByName(args[0])
|
||||
// What is wrong, and the self-check (novox/hq to-be 45 §2, §4).
|
||||
case "conditions":
|
||||
return conditionsCommand(ctx, args[1:])
|
||||
|
||||
@@ -284,14 +284,14 @@ func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, as
|
||||
if !onForge {
|
||||
return "", errors.New("the repository is not on the mesh's forge")
|
||||
}
|
||||
merged, known, err := inv.MergeOf(ctx, owner+"/"+name, commit)
|
||||
merged, known, err := keptForgeMerge(ctx, owner, name, commit)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !known {
|
||||
return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, commit)
|
||||
}
|
||||
return approvedAnnounced(ctx, inv, owner, name, announcedOf(merged))
|
||||
return approvedAnnounced(ctx, inv, owner, name, merged)
|
||||
}
|
||||
|
||||
// errWordNotAnApprove says the merge cannot be shown to be the operator's Approve of its head: its announcement names no
|
||||
@@ -343,15 +343,67 @@ func claimantsOf(entries []inventory.Entry, seat string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none.
|
||||
func announcedOf(m inventory.BatchedMerge) keptMerge {
|
||||
var k keptMerge
|
||||
_ = json.Unmarshal(m.Event, &k)
|
||||
return k
|
||||
// forgeMergeKey is where the forge's announcement of a merge is kept, by the merge commit: the one thing a build of a merge
|
||||
// names, from which the pull request and its head are found (novox/hq ADR 0420 §7).
|
||||
func forgeMergeKey(owner, repo, commit string) string {
|
||||
return fmt.Sprintf("merge/%s/%s/%s", owner, repo, commit)
|
||||
}
|
||||
|
||||
// keptMerge is a merge as the record keeps it: the forge's announcement, and the modules it moved when it was
|
||||
// heard. The announcement reads back as a SourceMoved alone, which ignores the rest.
|
||||
// keepForgeMerge keeps the forge's announcement of a merge in the bucket of the merges, a week at most as that bucket
|
||||
// keeps them. The controller plans nothing from a merge any more (ADR 0539 §17); this is all it keeps of one.
|
||||
var keepForgeMerge = func(ctx context.Context, m link.SourceMoved) error {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return fmt.Errorf("%w: the bus is not held", link.ErrTryAgain)
|
||||
}
|
||||
api, err := jetstream.New(serving.Conn())
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%w: the bucket of the merges: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
body, err := json.Marshal(keptMerge{SourceMoved: m})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := kv.Put(ctx, forgeMergeKey(m.Owner, m.Repo, m.Commit), body); err != nil {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// keptForgeMerge is the forge's announcement of a merge, by its commit; false when none was kept. A variable so a test can
|
||||
// hand it one.
|
||||
var keptForgeMerge = func(ctx context.Context, owner, repo, commit string) (keptMerge, bool, error) {
|
||||
serving := servingBus.Load()
|
||||
if serving == nil {
|
||||
return keptMerge{}, false, errors.New("only the serving controller keeps what the forge announced")
|
||||
}
|
||||
api, err := jetstream.New(serving.Conn())
|
||||
if err != nil {
|
||||
return keptMerge{}, false, err
|
||||
}
|
||||
kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket)
|
||||
if err != nil {
|
||||
return keptMerge{}, false, err
|
||||
}
|
||||
e, err := kv.Get(ctx, forgeMergeKey(owner, repo, commit))
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return keptMerge{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return keptMerge{}, false, err
|
||||
}
|
||||
var k keptMerge
|
||||
if err := json.Unmarshal(e.Value(), &k); err != nil {
|
||||
return keptMerge{}, false, err
|
||||
}
|
||||
return k, true, nil
|
||||
}
|
||||
|
||||
// keptMerge is a merge as the record keeps it: the forge's announcement.
|
||||
type keptMerge struct {
|
||||
link.SourceMoved
|
||||
Moves []string `json:"moves,omitempty"`
|
||||
|
||||
@@ -103,8 +103,11 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh
|
||||
const commit = "c0ffee0123456789"
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
var forgeMerge link.SourceMoved
|
||||
if err := json.Unmarshal(event, &forgeMerge); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := keepForgeMerge(ctx, forgeMerge); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
@@ -19,7 +18,7 @@ import (
|
||||
type anEmptyMesh struct {
|
||||
request inventory.BuildRequest
|
||||
asked bool
|
||||
merge inventory.BatchedMerge
|
||||
merge keptMerge
|
||||
}
|
||||
|
||||
func (m *anEmptyMesh) Catalogue(context.Context) (map[string]catalogue.Manifest, error) {
|
||||
@@ -33,9 +32,6 @@ func (m *anEmptyMesh) Catalogued(context.Context) ([]inventory.Entry, error) { r
|
||||
func (m *anEmptyMesh) BuildRequestByID(_ context.Context, id string) (inventory.BuildRequest, bool, error) {
|
||||
return m.request, m.asked && id == m.request.ID, nil
|
||||
}
|
||||
func (m *anEmptyMesh) MergeOf(context.Context, string, string) (inventory.BatchedMerge, bool, error) {
|
||||
return m.merge, true, nil
|
||||
}
|
||||
|
||||
// aDeliveryMergeOf is the Approved merge of a pull request of novox/mesh-delivery, its heads keyed by that repository,
|
||||
// and the ask and warrant bound to exactly those heads (the digest the Approve left covers them).
|
||||
@@ -102,8 +98,12 @@ func TestANewModuleIsRegisteredOnTheOperatorsApproveOfItsMergeAndRefusedOtherwis
|
||||
}
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7,
|
||||
"head_sha": head, "merge_commit_sha": commit})
|
||||
mesh := &anEmptyMesh{asked: true, merge: inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main",
|
||||
Commit: commit, Merged: time.Now(), Event: event},
|
||||
var forgeMerge keptMerge
|
||||
_ = json.Unmarshal(event, &forgeMerge)
|
||||
wasForge := keptForgeMerge
|
||||
t.Cleanup(func() { keptForgeMerge = wasForge })
|
||||
keptForgeMerge = func(context.Context, string, string, string) (keptMerge, bool, error) { return forgeMerge, true, nil }
|
||||
mesh := &anEmptyMesh{asked: true,
|
||||
request: inventory.BuildRequest{ID: "build-1", Repository: "novox/mesh-delivery", Seat: "git", Path: "",
|
||||
Ref: "main", Commit: commit, For: c.request}}
|
||||
forms := &sourceForms{bases: map[string]string{}, base: func(seat string) string {
|
||||
|
||||
@@ -104,8 +104,11 @@ func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheB
|
||||
const commit = "c0ffee0123456789"
|
||||
event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7,
|
||||
"head_sha": a.merge.HeadSHA, "merge_commit_sha": commit})
|
||||
if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit,
|
||||
Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil {
|
||||
var forgeMerge link.SourceMoved
|
||||
if err := json.Unmarshal(event, &forgeMerge); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := keepForgeMerge(ctx, forgeMerge); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano())
|
||||
|
||||
@@ -39,9 +39,6 @@ func TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild(t *testing.T)
|
||||
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 {
|
||||
t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err)
|
||||
}
|
||||
// The indexer writes the module's trunk value; the builder builds it, then its key is done.
|
||||
fp := fpOf('7')
|
||||
manifest := b.indexed(t, "systemd-resolved", "1", "novox/mesh-catalog", "modules/systemd-resolved", commit, fp)
|
||||
|
||||
@@ -796,19 +796,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
if err := showMode(ctx, inv, node); err != nil {
|
||||
return err
|
||||
}
|
||||
// What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what
|
||||
// it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader.
|
||||
// Unreadable is said, not taken for none.
|
||||
switch last, found, err := inv.LastSend(ctx, name); {
|
||||
case err != nil:
|
||||
fmt.Printf(" what it was last sent could NOT be read: %v\n", err)
|
||||
case !found:
|
||||
fmt.Printf(" last sent no send recorded since the mesh began keeping them\n")
|
||||
default:
|
||||
for _, line := range lastSendLines(last) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
// Whom agents run as here, and whether that account can become root without a person (ADR 0266).
|
||||
for _, line := range agentAccountLines(ctx, inv, node) {
|
||||
fmt.Println(line)
|
||||
@@ -915,12 +902,6 @@ func showNode(ctx context.Context, stored *stores, name string) error {
|
||||
}
|
||||
if len(assigned) > 0 {
|
||||
fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", "))
|
||||
// And what each runs, as its assignment names it (novox/hq ADR 0408, issue 1095).
|
||||
if on, err := inv.AssignmentsOn(ctx, name); err == nil {
|
||||
for _, line := range runLines(on) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
}
|
||||
// And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same
|
||||
// reason its condition says. Not computable is said, never read as "all applied".
|
||||
|
||||
@@ -10,7 +10,6 @@ package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -697,16 +696,6 @@ func namesWords(names []string, most int) string {
|
||||
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
|
||||
}
|
||||
|
||||
// planModules are the modules a walk moves, sorted.
|
||||
func planModules(p inventory.Plan) []string {
|
||||
out := make([]string, 0, len(p.Modules))
|
||||
for m := range p.Modules {
|
||||
out = append(out, m)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else
|
||||
// their count and the repository.
|
||||
func deliveryWhat(modules []string, repository string) string {
|
||||
@@ -727,18 +716,6 @@ func deliveryName(modules []string, repository string) string {
|
||||
return repoName(repository) + " delivery"
|
||||
}
|
||||
|
||||
// walkWaitingWords explains a walk waiting for its delivery's word.
|
||||
func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string {
|
||||
what := deliveryWhat(w.modules, w.repository)
|
||||
if severity == conditions.Urgent {
|
||||
return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+
|
||||
"delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in))
|
||||
}
|
||||
return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+
|
||||
"decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.",
|
||||
what, humanDuration(in), humanDuration(waitUrgentAfter))
|
||||
}
|
||||
|
||||
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
|
||||
// Start and Stop are also asked of the operator (novox/hq ADR 0259); the condition's own words keep saying
|
||||
// where they are given without a channel, and the ask's text drops that (askText).
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -278,182 +275,6 @@ func listensLines(m catalogue.Manifest) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
func planCommand(ctx context.Context, args []string) error {
|
||||
// One reading of the mesh for every machine this looks at.
|
||||
ctx = withInputCache(ctx)
|
||||
set := flag.NewFlagSet("plan", flag.ContinueOnError)
|
||||
// Because "one resource" does not tell you whether the settings landed. Being able to read
|
||||
// the file before it is sent is the difference between believing a merge worked and knowing.
|
||||
show := set.Bool("files", false, "print the files this node would be given")
|
||||
// The declaration exactly as the node would receive it. For handing to something else --
|
||||
// checking it against the host's own parser, most usefully, which is the only way to know
|
||||
// that what the control plane emits is what the host accepts.
|
||||
asJSON := set.Bool("json", false, "print the declaration this node would be sent")
|
||||
// What a push would change, against what the machine was last sent (novox/hq ADR 0217): read
|
||||
// before sending, so a change that removes, moves or recreates something is seen first.
|
||||
asDiff := set.Bool("diff", false, "what a push would change on this node, against what it was last sent")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(positionals) != 1 {
|
||||
return errors.New("plan <node> [--files] [--json] [--diff]")
|
||||
}
|
||||
args = positionals
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
|
||||
plan, settings, err := resolveNode(ctx, open, args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(plan.Modules) == 0 {
|
||||
fmt.Printf("%s is assigned nothing\n", args[0])
|
||||
return nil
|
||||
}
|
||||
// Which optional requirements are unbound (novox/hq ADR 0330 §7): beside the document on the
|
||||
// error stream where the answer is the declaration itself, which carries no such field — the
|
||||
// node-engine refuses an envelope key it does not know.
|
||||
if *asDiff || *asJSON {
|
||||
for _, line := range unboundLines(plan) {
|
||||
fmt.Fprintf(os.Stderr, "%s:%s\n", args[0], line)
|
||||
}
|
||||
}
|
||||
if *asDiff {
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// A module left out is not in the body, so the diff alone would say "nothing would change" for
|
||||
// a module just assigned whose settings cannot compose — success-shaped silence. Said first, with
|
||||
// why, as push and the plain plan say it (novox/hq ADR 0163, rule 6).
|
||||
reportLeftOut(args[0], declared)
|
||||
return writePlanDiff(ctx, open.inventory, args[0], body)
|
||||
}
|
||||
if *asJSON {
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// The bytes a push would send, indented: one marshaller, so `plan --json` cannot show an
|
||||
// envelope other than the one sent.
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var indented bytes.Buffer
|
||||
if err := json.Indent(&indented, body, "", " "); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(indented.String())
|
||||
return nil
|
||||
}
|
||||
|
||||
// What the machine was last told, by whom and from which assignment generation (novox/hq issue 234),
|
||||
// beside what it would be told now. A record that cannot be read is said, not taken for none.
|
||||
if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil {
|
||||
fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err)
|
||||
}
|
||||
// Which modules a push would leave out, and why — said before the plan, since the plan is of
|
||||
// what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan`
|
||||
// without --json allocates nothing.
|
||||
if record, err := open.inventory.NodeByName(ctx, args[0]); err == nil {
|
||||
left := plan.LeftOut(settings, record.Adopted)
|
||||
reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left})
|
||||
}
|
||||
// And which of its modules no provider will grant, because the identity overflows the bound of
|
||||
// what it requires (novox/hq ADR 0225) — said on the machine the remedy is for.
|
||||
for _, o := range plan.Overflowing() {
|
||||
fmt.Printf("%s: %s\n", args[0], o)
|
||||
}
|
||||
// And a setting that reaches nothing — refused where it is stored, and said here for one
|
||||
// stored before its definition moved from under it.
|
||||
for _, m := range plan.Modules {
|
||||
for _, stray := range catalogue.UnusedSettings(m, settings[m.Module]) {
|
||||
fmt.Printf("%s: a setting reaches nothing — %s\n", args[0], stray)
|
||||
}
|
||||
}
|
||||
fmt.Printf("%s would run:\n", args[0])
|
||||
for _, m := range plan.Modules {
|
||||
fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module])
|
||||
for _, line := range listensLines(m) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
}
|
||||
// What was assigned here and cannot run here. Said with the rest rather than as a refusal: it is
|
||||
// one module on the wrong machine, the others still run, and the remedy is to move this one.
|
||||
for _, u := range plan.Unhostable {
|
||||
for _, c := range u.Missing {
|
||||
fmt.Printf(" %-20s not applied — %s\n", u.Module, catalogue.WrongMachine(u.Module, c, args[0]))
|
||||
}
|
||||
}
|
||||
for _, c := range plan.Claims {
|
||||
fmt.Printf(" holds %s, one per %s\n", c.Claim, c.Scope)
|
||||
}
|
||||
// What this machine depends on that is not on it. Worth saying out loud: it is the only part
|
||||
// of a node's set that stops working when a *different* machine goes away, and nothing else
|
||||
// in this output would have told anybody that.
|
||||
for _, n := range plan.Needs {
|
||||
fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For)
|
||||
// And a served key the provider left out, because its machine composes no name it asks for
|
||||
// (novox/hq ADR 0346 §3).
|
||||
for _, line := range n.LeftOut {
|
||||
fmt.Printf(" %s\n", line)
|
||||
}
|
||||
}
|
||||
// What the providers on this machine leave out of what they serve, and why: said where the remedy
|
||||
// is (novox/hq ADR 0346 §3).
|
||||
for _, line := range plan.ServedLeftOut(settings) {
|
||||
fmt.Printf("%s: %s\n", args[0], line)
|
||||
}
|
||||
for _, line := range unboundLines(plan) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
// And a container's busy that this machine's node-engine is too old to be sent (novox/hq ADR 0335): its
|
||||
// container is replaced at once, as before. A statement that cannot be read is said as such.
|
||||
if reads, err := engineReadsBusy(ctx, open.inventory, args[0]); err != nil {
|
||||
fmt.Printf("%s: whether its node-engine reads a container's busy could not be read: %v\n", args[0], err)
|
||||
} else {
|
||||
for _, line := range busyNotSent(plan.Modules, reads) {
|
||||
fmt.Printf("%s: %s\n", args[0], line)
|
||||
}
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, args[0], plan, settings)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resources := declared.Resources
|
||||
for module, layers := range settings {
|
||||
for _, layer := range layers {
|
||||
fmt.Printf(" %-20s settings from %s\n", module, layer.From)
|
||||
}
|
||||
}
|
||||
// What each port reached from the local network resolved to on this machine (novox/hq ADR 0340 §8),
|
||||
// or that it resolved to none.
|
||||
for _, p := range declared.localNetwork {
|
||||
fmt.Println(" " + p.Said())
|
||||
}
|
||||
fmt.Printf("\n%d resource(s)\n", len(resources))
|
||||
|
||||
if *show {
|
||||
for _, r := range resources {
|
||||
content, ok := r["content"].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
fmt.Printf("\n--- %s ---\n%s", shownAs(r), content)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// shownAs is the heading `plan --show` puts over a resource's content.
|
||||
//
|
||||
// **A file written into says so.** Its content is the mesh's part of a file that is otherwise the
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan
|
||||
// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there after the
|
||||
// write; so no code but chainSavePlans (which SavePlansMoving calls, and no other function) writes a plan to the rollouts
|
||||
// bucket by anything but a change of the merges it holds. Checked by reading the sources, no store needed.
|
||||
func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) {
|
||||
write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`)
|
||||
var found []string
|
||||
for _, root := range []string{"../../cmd", "../../internal"} {
|
||||
err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
|
||||
return nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
text := string(raw)
|
||||
// The regular expression is the costly part of reading every source: only a file that names the table can match.
|
||||
if strings.Contains(strings.ToLower(text), "release_plan") && write.MatchString(text) {
|
||||
found = append(found, path+" writes the plan table")
|
||||
}
|
||||
// A plan record is written by the bucket's raw create and update, which only the plans' own file calls.
|
||||
if filepath.ToSlash(path) != "../../internal/inventory/chain_rollouts.go" && filepath.ToSlash(path) != "../../internal/chain/chain.go" &&
|
||||
strings.Contains(text, "chain.RolloutsOf") {
|
||||
found = append(found, path+" reaches the rollouts bucket")
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(found) > 0 {
|
||||
t.Fatalf("a plan is written without plan-moved being said:\n%s", strings.Join(found, "\n"))
|
||||
}
|
||||
raw, err := os.ReadFile("../../internal/inventory/plans.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
text := string(raw)
|
||||
save := strings.Index(text, "func (i *Inventory) SavePlansMoving")
|
||||
next := strings.Index(text[save:], "\nfunc ")
|
||||
if next < 0 {
|
||||
next = len(text) - save
|
||||
}
|
||||
if !strings.Contains(text[save:save+next], "chainSavePlans(") {
|
||||
t.Error("SavePlansMoving does not write through chainSavePlans")
|
||||
}
|
||||
chainRaw, err := os.ReadFile("../../internal/inventory/chain_rollouts.go")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
chainText := string(chainRaw)
|
||||
at := strings.Index(chainText, "func (i *Inventory) chainSavePlans")
|
||||
end := strings.Index(chainText[at:], "\nfunc ")
|
||||
if !strings.Contains(chainText[at:at+end], "PlanSaved(i, *p)") {
|
||||
t.Error("chainSavePlans does not say each plan it wrote")
|
||||
}
|
||||
if n := strings.Count(chainText, "createRaw(ctx, c, p.ID"); n != 1 {
|
||||
t.Errorf("a plan is made from %d places of chain_rollouts.go, want the one in chainSavePlans", n)
|
||||
}
|
||||
}
|
||||
|
||||
// movesSaid records what the hook hears: for each plan, the revisions it was said at.
|
||||
type movesSaid struct {
|
||||
mu sync.Mutex
|
||||
revs map[string][]int64
|
||||
}
|
||||
|
||||
func (s *movesSaid) hear(_ *inventory.Inventory, p inventory.Plan) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.revs[p.ID] = append(s.revs[p.ID], p.Revision)
|
||||
}
|
||||
|
||||
// sayingEverySave installs a PlanSaved that records, as the serving controller's says plan-moved.
|
||||
func sayingEverySave(t *testing.T) *movesSaid {
|
||||
t.Helper()
|
||||
s := &movesSaid{revs: map[string][]int64{}}
|
||||
was := inventory.PlanSaved
|
||||
inventory.PlanSaved = s.hear
|
||||
t.Cleanup(func() { inventory.PlanSaved = was })
|
||||
return s
|
||||
}
|
||||
|
||||
// noUnsaid fails the test for a plan whose stored revision is not one the hook heard: a move nobody said.
|
||||
func (s *movesSaid) noUnsaid(t *testing.T, open *stores) {
|
||||
t.Helper()
|
||||
recent, err := open.inventory.RecentPlans(t.Context(), 50)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if len(recent) == 0 {
|
||||
t.Fatal("no plan was kept")
|
||||
}
|
||||
for _, p := range recent {
|
||||
revs := s.revs[p.ID]
|
||||
if len(revs) == 0 || revs[len(revs)-1] != p.Revision {
|
||||
t.Errorf("%s is kept at revision %d and was said at %v: a move nobody said", p.ID, p.Revision, revs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store: the moves a merge makes of a plan — the batch it opens, the walk it is cut into, the tier that
|
||||
// is asked — are each said, and a look that changes nothing says nothing (novox/hq ADR 0406 §2).
|
||||
@@ -0,0 +1,35 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// **The controller opens on a store whose plan tables are gone** (novox/hq ADR 0539 §17, issue 1945 B9): migration 0108 has
|
||||
// dropped the record of sends, the assignment generation, the gate's verdicts and the durations, and the controller still
|
||||
// assigns a module, reads where it runs, takes it off and reads the nodes.
|
||||
func TestTheControllerOpensOnAStoreWithoutThePlanTables(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) {
|
||||
switch name := strings.Trim(row[0], `"`); name {
|
||||
case "declaration_send", "assignment_generation", "build_gate", "duration":
|
||||
t.Errorf("the table %s is in the store", name)
|
||||
}
|
||||
}
|
||||
register(t, open, catalogue.Manifest{Module: "notes", Version: "1"})
|
||||
if _, err := assign(ctx, open, "anchor", "notes"); err != nil {
|
||||
t.Fatalf("assign on the migrated store: %v", err)
|
||||
}
|
||||
if on, err := inv.Assigned(ctx, "anchor"); err != nil || len(on) != 1 || on[0] != "notes" {
|
||||
t.Fatalf("assigned %v: %v", on, err)
|
||||
}
|
||||
if nodes, err := inv.Nodes(ctx); err != nil || len(nodes) == 0 {
|
||||
t.Fatalf("nodes %v: %v", nodes, err)
|
||||
}
|
||||
}
|
||||
@@ -1087,16 +1087,12 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
plans, err := inv.OpenPlans(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
shelf, err := inv.Catalogue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hostVersions := deliveredVersions(shelf[hostModule])
|
||||
rolling := rollingModules(plans)
|
||||
rolling := map[string]bool{} // no plan rolls a module out any more (ADR 0539 §17)
|
||||
nodes, err := inv.Nodes(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1154,26 +1150,6 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// rollingModules is every module an open plan is rolling out: those it keeps a record of, and those its
|
||||
// tiers name. **A release keeps no record per module** — its walk is per machine, its modules only in its
|
||||
// tier — so reading the records alone, D10 said "no plan is rolling them out" about the node-engine while
|
||||
// a release walked it, and the gate on that release's first machine waited on what its own send causes
|
||||
// (novox/hq issue 348). Pure.
|
||||
func rollingModules(plans []inventory.Plan) map[string]bool {
|
||||
rolling := map[string]bool{}
|
||||
for _, p := range plans {
|
||||
for m := range p.Modules {
|
||||
rolling[m] = true
|
||||
}
|
||||
for _, tier := range p.Tiers {
|
||||
for _, m := range tier {
|
||||
rolling[m] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return rolling
|
||||
}
|
||||
|
||||
// deliveredVersions are the versions a module's registered build is delivered as: the last element
|
||||
// of every resource path under a `versions/` directory, which registration filled from the artifact's
|
||||
// digest (catalogue `${version}`). The node-engine names itself by that directory.
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// What a machine's provisioner client said of a module's files (novox/hq ADR 0362, ADR 0349 §15, design 57 P6,
|
||||
// issue 672), read where a walk's end and a gate count a machine's send applied: a module the send handed to the
|
||||
// provisioner is applied there only once the client reported `prov.applied` on that build's templates as well.
|
||||
|
||||
// provAppliedOn is the client's newest report of a module on a machine, from the stream that keeps it
|
||||
// (broker.ProvAppliedStream); false when it has said nothing of it. A variable so a test can say the client's words.
|
||||
var provAppliedOn = func(ctx context.Context, node, module string) (inventory.ProvReport, bool, error) {
|
||||
var raw []byte
|
||||
var said bool
|
||||
read := func(js nats.JetStreamContext) error {
|
||||
var err error
|
||||
raw, said, err = broker.LastProvApplied(js, node, module)
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
if d := doctorFrom; d != nil && d.js != nil {
|
||||
err = read(d.js.Context())
|
||||
} else {
|
||||
err = onTheBus(func(conn *nats.Conn) error {
|
||||
js, err := conn.JetStream(nats.Context(ctx))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return read(js)
|
||||
})
|
||||
}
|
||||
if err != nil || !said {
|
||||
return inventory.ProvReport{}, false, err
|
||||
}
|
||||
var r inventory.ProvReport
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return inventory.ProvReport{}, false, fmt.Errorf("what %s's provisioner client said of %s is not its report: %w",
|
||||
node, module, err)
|
||||
}
|
||||
return r, true, nil
|
||||
}
|
||||
|
||||
// provLookup answers the clients' reports for one reading; a report that cannot be read is not counted, which
|
||||
// leaves the machine waiting, then silent, never applied — and is said.
|
||||
func provLookup(ctx context.Context) inventory.ProvLookup {
|
||||
return func(node, module string) (inventory.ProvReport, bool) {
|
||||
r, said, err := provAppliedOn(ctx, node, module)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "what %s's provisioner client said of %s could not be read: %v\n", node, module, err)
|
||||
return inventory.ProvReport{}, false
|
||||
}
|
||||
return r, said
|
||||
}
|
||||
}
|
||||
|
||||
// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from
|
||||
// the stream that keeps their reports.
|
||||
func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) {
|
||||
all, err := broker.ProvAppliedReports(js)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []inventory.ProvReport
|
||||
for subject, raw := range all {
|
||||
var r inventory.ProvReport
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err)
|
||||
}
|
||||
if r.Status == inventory.ProvFailed {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// provisionedOutcome is a machine's report of one send as a delivery's end counts it (novox/hq ADR 0362 §1-5, issue
|
||||
// 840): the engine's own report, combined with its provisioner client's for a send that handed modules to it. The
|
||||
// outcome of a client's failure begins inventory.OutcomeProvisioningFailed; a report not yet applied on both sides
|
||||
// is none (false). A send handing nothing is the engine's report unchanged. onSend says the report is about the send.
|
||||
func provisionedOutcome(node string, sent inventory.SentDeclaration, r inventory.Reported, onSend bool,
|
||||
prov inventory.ProvLookup) (string, bool) {
|
||||
if len(sent.Provisioned) == 0 {
|
||||
return r.Outcome, onSend
|
||||
}
|
||||
engine := inventory.AppliedReport{Outcome: r.Outcome}
|
||||
if r.At != nil {
|
||||
engine.At = *r.At
|
||||
}
|
||||
got, ok := inventory.ProvisionedSend(node, sent, engine, onSend, prov)
|
||||
return got.Outcome, ok
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
)
|
||||
|
||||
// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from
|
||||
// the stream that keeps their reports.
|
||||
func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) {
|
||||
all, err := broker.ProvAppliedReports(js)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []inventory.ProvReport
|
||||
for subject, raw := range all {
|
||||
var r inventory.ProvReport
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err)
|
||||
}
|
||||
if r.Status == inventory.ProvFailed {
|
||||
out = append(out, r)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -4,22 +4,18 @@ import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/conditions"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
@@ -194,7 +190,6 @@ func serve(ctx context.Context) (err error) {
|
||||
// Never by a read.
|
||||
go settlingPending(ctx, open)
|
||||
// The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0).
|
||||
go forgettingOldDurations(ctx, inv)
|
||||
// And what the catalogue decided a build meant. The builder's own result is already handled
|
||||
// above; this is the other half — the control plane is the only one of the three that knows
|
||||
// which machines run the thing, so it is the one that acts (novox/hq ADR 0072).
|
||||
@@ -374,108 +369,6 @@ func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest
|
||||
}
|
||||
}
|
||||
|
||||
// readyNode is one machine and the declaration it would be sent.
|
||||
type readyNode struct {
|
||||
node string
|
||||
declared sendable
|
||||
}
|
||||
|
||||
// composeEach works out what each named machine should be, and never lets one machine's answer
|
||||
// decide another's.
|
||||
//
|
||||
// **A machine whose set cannot be worked out is that machine's problem** (novox/hq ADR 0066). A
|
||||
// whole-mesh push used to refuse outright when any one node failed to resolve, so a single
|
||||
// unanswerable requirement on a single machine — one module requiring a provision nobody had
|
||||
// assigned a provider for — left every other machine in the mesh unconverged, including machines
|
||||
// with no relation to it at all. Nothing was sent anywhere, and the machines that could not be sent
|
||||
// were the ones with nothing wrong with them.
|
||||
//
|
||||
// It is the same rule a92c11b established one level down, where an un-hostable module stopped
|
||||
// taking down the healthy modules beside it, applied one level up: **the blast radius of a fault is
|
||||
// the thing that has it.** What could not be worked out is named and returned, so a push still ends
|
||||
// with a non-zero outcome and nobody mistakes a partial convergence for a whole one.
|
||||
//
|
||||
// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential
|
||||
// across two machines that must agree — and dropped here, where it never did.
|
||||
func composeEach(names []string, allot func(node string) (order, error),
|
||||
compose func(node string) (sendable, error)) ([]readyNode, []string) {
|
||||
|
||||
var sending []readyNode
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
// **Numbered before it is composed, not before it is sent** (novox/hq issue 204). The
|
||||
// number says where this declaration stands against every other the mesh composed for the
|
||||
// machine, and the host refuses one lower than the last it applied. Taken at send time, as
|
||||
// it was, a declaration composed a minute ago — before an assignment changed — went out with
|
||||
// a number higher than one composed after the change and sent before it, and the machine
|
||||
// took the older content as the newer word: on 2026-10-02 a runtime assigned and applied on
|
||||
// two machines was undone two seconds later by exactly that. Taken here, before the first
|
||||
// read, what was composed earlier is numbered lower whatever order the sends happen in.
|
||||
numbered, err := allot(name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
declared, err := compose(name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
numbered.stamp(&declared)
|
||||
if len(declared.Resources) == 0 {
|
||||
// Sent, not skipped (novox/hq issue 127). A node whose declaration composes to
|
||||
// nothing may have HELD something before — the broker opening a placement gave it,
|
||||
// say — and skipping the empty declaration leaves that last resource in force
|
||||
// forever, re-applied by the node's own heartbeat, with no way for the mesh to say
|
||||
// it is gone. An empty declaration is the correction: the host drops what the mesh
|
||||
// owned and keeps what it found (the adoption envelope still rides along). A node
|
||||
// that never held anything applies it as the no-op it is.
|
||||
fmt.Printf("%s owns nothing now — sent so it drops what it last held\n", name)
|
||||
}
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
return sending, refusals
|
||||
}
|
||||
|
||||
// keeping is what a round of sends keeps of the machines it composed, before any is delivered: nil
|
||||
// keeps them all. A push holds back the machines whose running modules' data would move (holdingBack).
|
||||
type keeping func(held context.Context, sending []readyNode) ([]readyNode, error)
|
||||
|
||||
// holdingBack leaves out each machine whose declaration would move a running module's data, says so,
|
||||
// and names it among those held back (novox/hq ADR 0217); the rest go on to be delivered, grants first
|
||||
// (issue 249). A declaration's body is the same bytes however often it is read.
|
||||
func holdingBack(ctx context.Context, inv *inventory.Inventory, sending []readyNode, movable map[string]bool,
|
||||
heldBack *[]string) ([]readyNode, error) {
|
||||
var out []readyNode
|
||||
for _, s := range sending {
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
moves, err := heldMoves(ctx, inv, s.node, body, movable)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(moves) > 0 {
|
||||
sayDataHeld(os.Stdout, s.node, moves)
|
||||
*heldBack = append(*heldBack, s.node)
|
||||
continue
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// delivery is the two acts of sending machines what they should be, apart, so the order between
|
||||
// them is one function's and can be read and tested there (novox/hq issue 249).
|
||||
type delivery interface {
|
||||
// grant issues what the machines' modules may do — each module's state raised and its
|
||||
// membership issued — for every machine about to be sent.
|
||||
grant(ctx context.Context, sending []readyNode) error
|
||||
// declare sends one machine its declaration and records it sent, answering the digest.
|
||||
declare(ctx context.Context, s readyNode, body []byte) (string, error)
|
||||
}
|
||||
|
||||
// errGrants marks a send that stopped because what the machines' modules may do could not be issued
|
||||
// (novox/hq issue 249). Nothing about the machines is wrong; asked again, it is likely to work, so an
|
||||
// announcement that hits it is held and asked again.
|
||||
@@ -496,158 +389,9 @@ func (g *grantsRefused) Error() string {
|
||||
strings.Join(names, ", "), g.nodes[names[0]])
|
||||
}
|
||||
|
||||
// deliver sends the machines their declarations: **the machine holding the bus, then the grants,
|
||||
// then the rest** (novox/hq issue 249).
|
||||
//
|
||||
// A merge gave a module a new state; its bundle reached every machine within a minute, and the
|
||||
// machines' permissions on the bus did not include the state until somebody pushed by hand: the code
|
||||
// arrived before the right to use it. A module that read its new state on start failed its start; the
|
||||
// one that was there retried for two minutes. The memberships were issued after the declarations —
|
||||
// "because the runtime it is for arrives with it" — and a membership is retained last-per-subject on
|
||||
// the bus (internal/link/bus.go), so issued first it waits for the runtime that arrives after it. A
|
||||
// runtime still on the old code merely holds a grant it does not use yet.
|
||||
//
|
||||
// **The holder's declaration before the grants, though.** The bus's user list travels in it, and the
|
||||
// controller's own right to publish memberships and raise buckets is in that list (the precedent of
|
||||
// issue 183): grants first, and a grant the controller is not yet allowed to make would hold the very
|
||||
// declaration that allows it — a lock only a hand on the broker could open. A runtime already running
|
||||
// on that machine follows a membership issued after its declaration, as it always has.
|
||||
//
|
||||
// **A grant that cannot be issued holds back what it concerns, and says so as an error.** It used to
|
||||
// be said and passed over — "the machines keep what they derive until the next push" — which reported
|
||||
// a rollout done that had delivered code its machines could not run. A membership that failed holds
|
||||
// back its own machine; a failure that names no machine (the buckets) holds back every machine but the
|
||||
// holder, already sent. The error carries errGrants, so the caller's rollout is not marked sent and is
|
||||
// tried again.
|
||||
//
|
||||
// The grants are issued, not waited on: a membership is a retained message the runtime reads when it
|
||||
// comes, and the bus answers its publication; nothing here waits for a runtime to have read one.
|
||||
func deliver(ctx context.Context, d delivery, holder string, sending []readyNode) (map[string]string, error) {
|
||||
digests := map[string]string{}
|
||||
if len(sending) == 0 {
|
||||
return digests, nil
|
||||
}
|
||||
send := func(s readyNode) error {
|
||||
// The number is inside the signed bytes, so a replayed older declaration cannot borrow a
|
||||
// newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204).
|
||||
body, err := s.declared.Body()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digest, err := d.declare(ctx, s, body)
|
||||
if errors.Is(err, errFilesNotPut) && s.declared.fileReferences {
|
||||
// **The grant that lets the controller write the bucket travels in the very declaration of the machine holding the
|
||||
// bus** (novox/hq ADR 0486 §2): until the bus has it, a put is refused, and the declaration that would deliver the
|
||||
// grant would wait for it. So that send, and only that, goes with every content inline, as to a node that reads no
|
||||
// references, and says so; the next one tries the bucket again.
|
||||
fmt.Printf("%s: the files that travel by digest could not be put in the bucket (%v): sending them inline\n", s.node, err)
|
||||
s.declared.fileReferences = false
|
||||
if body, err = s.declared.Body(); err != nil {
|
||||
return err
|
||||
}
|
||||
digest, err = d.declare(ctx, s, body)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digests[s.node] = digest
|
||||
return nil
|
||||
}
|
||||
var rest []readyNode
|
||||
for _, s := range sending {
|
||||
if holder != "" && s.node == holder {
|
||||
if err := send(s); err != nil {
|
||||
return digests, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
rest = append(rest, s)
|
||||
}
|
||||
held := map[string]error{}
|
||||
if err := d.grant(ctx, sending); err != nil {
|
||||
var some *grantsRefused
|
||||
if !errors.As(err, &some) {
|
||||
var names []string
|
||||
for _, s := range rest {
|
||||
names = append(names, s.node)
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return digests, fmt.Errorf("%w: %w", errGrants, err)
|
||||
}
|
||||
return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(names, ", "), err)
|
||||
}
|
||||
held = some.nodes
|
||||
}
|
||||
var notSent []string
|
||||
for _, s := range rest {
|
||||
if _, refused := held[s.node]; refused {
|
||||
notSent = append(notSent, s.node)
|
||||
continue
|
||||
}
|
||||
if err := send(s); err != nil {
|
||||
return digests, err
|
||||
}
|
||||
}
|
||||
if len(notSent) > 0 {
|
||||
return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(notSent, ", "),
|
||||
&grantsRefused{nodes: held})
|
||||
}
|
||||
if len(held) > 0 {
|
||||
// Only the holder's own memberships failed, and it was sent before them.
|
||||
return digests, fmt.Errorf("%w: %w", errGrants, &grantsRefused{nodes: held})
|
||||
}
|
||||
return digests, nil
|
||||
}
|
||||
|
||||
// overTheBus is delivery as the mesh does it: memberships on the bus, declarations signed.
|
||||
type overTheBus struct {
|
||||
open *stores
|
||||
server *link.Server
|
||||
signer link.Signer
|
||||
// indent is put before each "sent" line, for the callers whose output is nested.
|
||||
indent string
|
||||
}
|
||||
|
||||
// putFiles puts the contents s names by digest in the bucket.
|
||||
func (b overTheBus) putFiles(ctx context.Context, s readyNode) error {
|
||||
objects, err := s.declared.objects()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: the files that travel by digest cannot be read: %w", s.node, err)
|
||||
}
|
||||
if len(objects) == 0 {
|
||||
return nil
|
||||
}
|
||||
bus, ok := b.server.Bus().(link.OverNATS)
|
||||
if !ok {
|
||||
return fmt.Errorf("%s: the declaration names files by digest and this bus has no object bucket", s.node)
|
||||
}
|
||||
putting, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
store, err := openBucket(putting, bus.Conn, s.node)
|
||||
if err == nil {
|
||||
err = putObjects(putting, store, objects)
|
||||
}
|
||||
notePut(ctx, s.node, err)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket.
|
||||
var errFilesNotPut = errors.New("the files it names by digest are not in the bucket")
|
||||
|
||||
func (b overTheBus) grant(ctx context.Context, sending []readyNode) error {
|
||||
// The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised
|
||||
// when they cannot be, and never what holds the send back (assertOnSend).
|
||||
if bus, ok := b.server.Bus().(link.OverNATS); ok {
|
||||
js := broker.OnConn(bus.Conn)
|
||||
js.Note = func(format string, args ...any) { fmt.Printf(b.indent+" "+format+"\n", args...) }
|
||||
_ = assertOnSend(ctx, b.open.inventory, js, b.indent)
|
||||
}
|
||||
return issueMemberships(ctx, b.open, b.server, sending)
|
||||
}
|
||||
|
||||
// brokerFirst is the machines to send in the order a grant needs (novox/hq issue 249): the machine
|
||||
// holding the bus first — its declaration carries the bus's user list (composeBusUsers), and a
|
||||
// module's new permissions are refused by the bus until that list says them. Among the machines it
|
||||
@@ -755,219 +499,6 @@ func couldNotBeResolved(refusals []string, sent int) error {
|
||||
len(refusals), sent, strings.Join(refusals, "\n\n"))
|
||||
}
|
||||
|
||||
// sendTo resolves and sends to exactly the machines named, or refuses without sending anything.
|
||||
//
|
||||
// The all-or-nothing rule push deliberately does NOT follow, and for a reason that holds here and
|
||||
// not there: a rotation that reached the
|
||||
// consumer and refused on the provider would leave one end holding a credential the other has
|
||||
// never heard of — which is the state this whole mechanism exists to make impossible.
|
||||
func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
_, err := sendToEach(ctx, open, names)
|
||||
return err
|
||||
}
|
||||
|
||||
// sendToEach is sendTo, answering the machines it sent: those named, and before them the machine
|
||||
// holding the bus when its user list must go first (novox/hq issue 249) — so a caller that waits for
|
||||
// the machines it sent waits for that one too.
|
||||
func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
inv := open.inventory
|
||||
// **No send replaces the bus but its planned step** (novox/hq ADR 0236). A plan's send to the bus's
|
||||
// machine for some other module carried the bus's new build with it on 2026-10-06, and the bus
|
||||
// restarted under every machine with nobody having asked. Refused whole — the send cannot leave the
|
||||
// bus behind and carry the rest (ADR 0221 option 2) — and said with the remedy; the plan tries again.
|
||||
if !busStepSending(ctx) {
|
||||
held, err := busHeld(ctx, inv, names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, n := range names {
|
||||
if why, h := held[n]; h {
|
||||
return nil, fmt.Errorf("%w: %s", errBusWaits, why)
|
||||
}
|
||||
}
|
||||
}
|
||||
ident, err := openIdentity(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer ident.Close()
|
||||
|
||||
// The private network every declaration of the send is composed with: not computable refuses the send up front.
|
||||
if _, err := generatorsOf(ctx, open); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
holder, behind, err := brokerBehind(ctx, open, names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names = brokerFirst(names, holder, behind)
|
||||
// Held from composing to sending (novox/hq ADR 0100); a caller that holds them already —
|
||||
// converge, which flips the node and then sends it — is not made to wait on itself.
|
||||
ctx, release, err := holdNodes(ctx, open, names)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
|
||||
var sending []readyNode
|
||||
var refusals []string
|
||||
for _, name := range names {
|
||||
// Numbered before composing, for the reason composeEach gives (novox/hq issue 204).
|
||||
numbered, err := allot(ctx, inv, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
// Read afresh for each machine: the send before it made and recorded things.
|
||||
composing := freshInput(ctx)
|
||||
plan, _, err := resolveNode(composing, open, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
reportUnhostable(name, plan)
|
||||
reportKept(ctx, plan)
|
||||
declared, err := composeSend(composing, open, name)
|
||||
if err != nil {
|
||||
refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err))
|
||||
continue
|
||||
}
|
||||
numbered.stamp(&declared)
|
||||
reportLeftOut(name, declared)
|
||||
sending = append(sending, readyNode{name, declared})
|
||||
}
|
||||
if len(refusals) > 0 {
|
||||
return nil, fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s",
|
||||
len(refusals), strings.Join(refusals, "\n\n"))
|
||||
}
|
||||
|
||||
server, err := connectLink(ctx, nil, nil, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer server.Close()
|
||||
|
||||
// And every assignment on those machines its membership (novox/hq ADR 0160): composed from the
|
||||
// same records the bus's accounts are, so what a runtime serves and what its account may are one
|
||||
// composition. **Issued before the declarations** (novox/hq issue 249): the runtime the
|
||||
// membership is for arrives with the declaration, and a membership waits for it on the bus; the
|
||||
// code arriving first was refused its own state until somebody pushed.
|
||||
if _, err := deliver(ctx, overTheBus{open: open, server: server, signer: ident, indent: " "}, holder, sending); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sent := make([]string, 0, len(sending))
|
||||
for _, s := range sending {
|
||||
sent = append(sent, s.node)
|
||||
}
|
||||
return sent, nil
|
||||
}
|
||||
|
||||
// issueMemberships publishes the membership of every module on the machines just sent.
|
||||
//
|
||||
// Each carries what its module receives and the private network's addresses, from the same
|
||||
// composition as the declaration it was sent (novox/hq ADR 0167): a provider reads what it is
|
||||
// given on the bus, and the file written beside it says the same thing.
|
||||
func issueMemberships(ctx context.Context, open *stores, server *link.Server, sent []readyNode) error {
|
||||
records, err := open.inventory.BusRecords(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bus, ok := server.Bus().(link.OverNATS)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
// Which machines are root-free now (novox/hq ADR 0259 §8): a channel's verified sender is composed for the
|
||||
// router only from one, beside a router on one. Judged once per push, by the root-free verb's judgement.
|
||||
var whys map[string]string
|
||||
records.RootFree, whys = rootVerdictsOf(judgeRootFree(ctx, newRootReader(ctx, open.inventory, bus.Conn),
|
||||
records.Nodes, time.Now()))
|
||||
where := broker.PlacementsOf(records, records.Interchangeable)
|
||||
// What the judgement kept off is said, with its reason (novox/hq issue 930); the self-check gives it back.
|
||||
for _, line := range droppedVerifiedSenders(records, where, whys) {
|
||||
fmt.Println(line)
|
||||
}
|
||||
// **Every declared state's bucket, before the memberships that name it** (novox/hq ADR 0201). The
|
||||
// raise at start asserts them too, but a module registered and assigned since would otherwise have
|
||||
// its bucket only after the control plane next restarts — found the first time a module declared
|
||||
// state: its bundle asked for a bucket that did not exist. Idempotent and cheap.
|
||||
//
|
||||
// **A failure here is the send's failure** (novox/hq issue 249). It was said and the push stood,
|
||||
// because the declarations were already away; they are sent after this now — all but the bus's
|
||||
// own machine, sent before it (deliver) — and a module whose state does not exist is a module
|
||||
// that fails its start, so they are not sent and the caller tries again rather than reporting the
|
||||
// rollout done.
|
||||
buckets, err := open.inventory.DeclaredBuckets(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the modules' state could not be read, so no bucket was asserted: %w", err)
|
||||
}
|
||||
if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil {
|
||||
return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err)
|
||||
}
|
||||
// **And every declared log, for the same reason** (novox/hq ADR 0297 §2): a membership names its
|
||||
// logs, and a module whose log does not exist fails its first append.
|
||||
logs, err := open.inventory.DeclaredLogs(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the modules' logs could not be read, so no log was asserted: %w", err)
|
||||
}
|
||||
if _, err := broker.RaiseLogs(broker.OnConn(bus.Conn), logs); err != nil {
|
||||
return fmt.Errorf("the modules' logs could not be asserted on the bus: %w", err)
|
||||
}
|
||||
// Every membership is tried, and the first failure named once.
|
||||
issued := 0
|
||||
refused := map[string]error{}
|
||||
for _, s := range sent {
|
||||
node := s.node
|
||||
for _, d := range records.Assigned[node] {
|
||||
membership := broker.MembershipFor(node, d, where)
|
||||
membership.Mesh = s.declared.Mesh
|
||||
for requirement, given := range s.declared.Received[d.Module] {
|
||||
raw, err := json.Marshal(given)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if membership.Receives == nil {
|
||||
membership.Receives = map[string]json.RawMessage{}
|
||||
}
|
||||
membership.Receives[requirement] = raw
|
||||
}
|
||||
body, err := json.Marshal(membership)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
|
||||
if refused[node] == nil {
|
||||
refused[node] = fmt.Errorf("%s: %w", d.Module, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
issued++
|
||||
}
|
||||
}
|
||||
if issued > 0 {
|
||||
fmt.Printf(" issued %d membership(s)\n", issued)
|
||||
}
|
||||
// **And what every other machine hears of a module's stream** (novox/hq ADR 0363 §6): a grant the provider's
|
||||
// setting changed reaches a consumer on a machine this send does not reach. Its membership is issued again
|
||||
// as last issued, with only what it hears changed, so nothing else of it moves before its own send.
|
||||
reheard := reissueHearing(ctx, bus, records, where, sent)
|
||||
if reheard > 0 {
|
||||
fmt.Printf(" issued %d membership(s) again for what they hear of a module's stream\n", reheard)
|
||||
}
|
||||
// **And says again what it keeps** (novox/hq ADR 0495 clause 2, issue 1790): its word at start is refused until a push
|
||||
// carries its grant; kept only when it changed, so saying it at every push changes nothing else.
|
||||
if js := servingBus.Load(); js != nil {
|
||||
if err := announceOwnStates(ctx, js); err != nil {
|
||||
fmt.Printf(" states: %v\n", err)
|
||||
}
|
||||
}
|
||||
if len(refused) > 0 {
|
||||
// Returned, never passed over (novox/hq issue 249): the declarations of the machines they are
|
||||
// for are not sent, and the rollout that asked is tried again rather than waiting for a push.
|
||||
return &grantsRefused{nodes: refused}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// digestOf is what the mesh compares to answer "has this machine been sent what it should be".
|
||||
//
|
||||
// Over the same bytes that are sent, so the comparison is of the thing itself rather than of
|
||||
@@ -1027,11 +558,6 @@ func wouldSendFrom(ctx context.Context, open *stores,
|
||||
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment
|
||||
// elsewhere in the mesh is not a change of this machine.
|
||||
if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := declared.Body()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1165,120 +691,10 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107).
|
||||
// allotting is allot over one inventory, in the shape composeEach takes.
|
||||
func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (order, error) {
|
||||
return func(node string) (order, error) { return allot(ctx, inv, node) }
|
||||
}
|
||||
|
||||
// epochForActs is the lease's gate as a composition asks it; a variable so a test can act under an epoch
|
||||
// without a bus.
|
||||
var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) }
|
||||
|
||||
// order is what a declaration carries of its writer's order (link/order.go): its sequence, and the
|
||||
// epoch of the lease it is composed under — zero for a machine that has not said it reads one.
|
||||
type order struct {
|
||||
sequence int64
|
||||
epoch uint64
|
||||
// generation is the assignment generation read before the composition, and readsGeneration whether the
|
||||
// machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts
|
||||
// under, whether or not the machine is sent it — both kept for the record of the send.
|
||||
generation int64
|
||||
readsGeneration bool
|
||||
acting uint64
|
||||
}
|
||||
|
||||
// allot takes the next sequence for a machine — the number its next declaration carries — under the
|
||||
// lease: a process that may not act takes none, and composes nothing (novox/hq to-be 45 §6).
|
||||
func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, error) {
|
||||
epoch, err := epochForActs(ctx)
|
||||
if err != nil {
|
||||
return order{}, fmt.Errorf("nothing was composed for %s: %w", node, err)
|
||||
}
|
||||
record, err := inv.NodeByName(ctx, node)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
acting := epoch
|
||||
if epoch > 0 {
|
||||
reads, err := inv.ReadsEpoch(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
if !reads {
|
||||
epoch = 0
|
||||
}
|
||||
}
|
||||
// **The generation is read here, before the composition reads a single assignment** (novox/hq issue
|
||||
// 234). A generation only grows, so one read before is never newer than the view composed after it:
|
||||
// the declaration may claim a generation older than its content, never newer — and a claim newer than
|
||||
// the content is exactly the stale send the machine must be able to refuse.
|
||||
generation, err := inv.AssignmentGeneration(ctx)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
readsGeneration, err := inv.ReadsGeneration(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
seq, err := inv.NextSequence(ctx, record.ID)
|
||||
if err != nil {
|
||||
return order{}, err
|
||||
}
|
||||
return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration,
|
||||
acting: acting}, nil
|
||||
}
|
||||
|
||||
// recordSent writes down what a machine was just sent, and returns the digest.
|
||||
//
|
||||
// **On a context that outlives the caller's** (novox/hq issue 204). The record is written after the
|
||||
// declaration is away, so a send that failed is never recorded as current — and a controller being
|
||||
// replaced mid-send had its context cancelled between the two, so the machine was told and the mesh
|
||||
// never wrote it down: status read "applied, current" over a machine that had just been sent
|
||||
// something else. What was sent was sent; the record of it must not depend on the sender living
|
||||
// another second. Bounded, so a store that is away does not hold a dying process open for ever.
|
||||
//
|
||||
// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known:
|
||||
// what tells a machine held back by a policy or a plan from one a push left behind.
|
||||
func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte,
|
||||
builds map[string]string, epoch uint64, sent sentRecord) (string, error) {
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer cancel()
|
||||
record, err := inv.NodeByName(kept, node)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
digest := digestOf(body)
|
||||
// The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the
|
||||
// would-send is never stamped with a generation the machine was not sent; and the send itself, who sent
|
||||
// it and from which generation, beside them. A record of the send that cannot be written does not make a
|
||||
// send that is away look failed: it is said, loudly, and the machine's own record stands.
|
||||
err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{
|
||||
Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation,
|
||||
Digest: digest, Modules: sent.modules, Provisioning: sent.plans})
|
||||
var unrecorded *inventory.SendNotRecordedError
|
||||
switch {
|
||||
case errors.As(err, &unrecorded):
|
||||
fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+
|
||||
"sent it and from which generation could not be written down, so a refusal of it will name no sender "+
|
||||
"(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err)
|
||||
fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err)
|
||||
case err != nil:
|
||||
return "", err
|
||||
}
|
||||
// And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217).
|
||||
// Never a reason to fail a send that is already away: a summary that cannot be kept means the
|
||||
// next comparison has nothing to hold against, which is how every machine starts.
|
||||
if summary, err := summarize(body); err == nil {
|
||||
if raw, err := json.Marshal(summary); err == nil {
|
||||
if err := inv.RecordSentSummary(kept, record.ID, raw); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: what it was sent is not kept for comparison: %v\n", node, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
// reportUnheldPushed says what a push's machines lack of the seats their modules depend on
|
||||
// (novox/hq ADR 0207): every line for a machine the push named, since that is the machine somebody
|
||||
// is looking at, and one line per machine otherwise — a list per machine across the mesh is the
|
||||
@@ -1314,134 +730,3 @@ func schemaReach() (int, error) {
|
||||
}
|
||||
return reach, nil
|
||||
}
|
||||
|
||||
// reissueHearing issues again, on every machine not just sent, the membership of each assignment whose
|
||||
// `consumes` names a module owning a stream, when what it hears changed: read as last issued from the bus,
|
||||
// its `hears` replaced, and published. One that was never issued is left to its own send. Failures are said
|
||||
// and passed over: the send that carries the grant is not the consumer's, and its own next send issues it.
|
||||
func reissueHearing(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements,
|
||||
sent []readyNode) int {
|
||||
reached := map[string]bool{}
|
||||
for _, s := range sent {
|
||||
reached[s.node] = true
|
||||
}
|
||||
issued := 0
|
||||
for node, declared := range records.Assigned {
|
||||
if reached[node] {
|
||||
continue
|
||||
}
|
||||
for _, d := range declared {
|
||||
if !d.HearsAStream {
|
||||
continue
|
||||
}
|
||||
subject := broker.MembershipSubject(node, d.Module)
|
||||
last, err := bus.JS.GetLastMsg(broker.AssignmentsStream, subject)
|
||||
if err != nil {
|
||||
if !errors.Is(err, nats.ErrMsgNotFound) {
|
||||
fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n",
|
||||
d.Module, node, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
body, changed, err := withHearing(last.Data, broker.HeardIssuedFor(node, d))
|
||||
if err != nil {
|
||||
fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n",
|
||||
d.Module, node, err)
|
||||
continue
|
||||
}
|
||||
if !changed {
|
||||
continue
|
||||
}
|
||||
if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil {
|
||||
fmt.Printf(" %s on %s: what it hears could not be issued again: %v\n", d.Module, node, err)
|
||||
continue
|
||||
}
|
||||
issued++
|
||||
}
|
||||
}
|
||||
return issued
|
||||
}
|
||||
|
||||
// withHearing is a membership as issued with its `hears` replaced, and whether that changed it. Pure.
|
||||
func withHearing(issued []byte, hears []broker.HeardIssued) ([]byte, bool, error) {
|
||||
var fields map[string]json.RawMessage
|
||||
if err := json.Unmarshal(issued, &fields); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
var had []broker.HeardIssued
|
||||
if raw, ok := fields["hears"]; ok {
|
||||
if err := json.Unmarshal(raw, &had); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
}
|
||||
if slices.Equal(had, hears) {
|
||||
return issued, false, nil
|
||||
}
|
||||
if len(hears) == 0 {
|
||||
delete(fields, "hears")
|
||||
} else {
|
||||
raw, err := json.Marshal(hears)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
fields["hears"] = raw
|
||||
}
|
||||
body, err := json.Marshal(fields)
|
||||
return body, true, err
|
||||
}
|
||||
|
||||
func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) {
|
||||
// The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node
|
||||
// that applies it finds every one. A node reading no references has none.
|
||||
// **Under the node's lock** from the put to the publication, so the sweep of its bucket cannot remove an object between them.
|
||||
unlock := lockFilesOf(s.node)
|
||||
err := b.putFiles(ctx, s)
|
||||
if err != nil {
|
||||
unlock()
|
||||
return "", err
|
||||
}
|
||||
declared := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second)
|
||||
unlock()
|
||||
if declared != nil {
|
||||
return "", declared
|
||||
}
|
||||
// After it is away, not before. A digest recorded for something that failed to send would make
|
||||
// the machine look current for a declaration it never received.
|
||||
digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch,
|
||||
sentRecordOf(ctx, s.declared))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if len(s.declared.Bindings) > 0 {
|
||||
// And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232):
|
||||
// what the next resolution keeps it at. On the same outliving context as the send's record.
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
err := b.open.inventory.RecordBindings(kept, s.node, s.declared.Bindings)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s was sent its declaration, and where its consumers are bound to their "+
|
||||
"data could not be recorded: %w", s.node, err)
|
||||
}
|
||||
}
|
||||
if s.declared.BusUsers != "" {
|
||||
// And the user list it carried, so the next send reads whether it must go first from the
|
||||
// list alone (novox/hq issue 249). On the same outliving context as the send's record.
|
||||
kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
err := b.open.inventory.RecordSentBusUsers(kept, s.node, digestOf([]byte(s.declared.BusUsers)))
|
||||
cancel()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
// The list sent issued the credential of every module it carried, at the fingerprint each is registered at:
|
||||
// the grants the assigner's sixth rule reads (novox/hq ADR 0528 clause 2, issue 1394).
|
||||
if bus, ok := b.server.Bus().(link.OverNATS); ok {
|
||||
if js, err := jetstream.New(bus.Conn); err == nil {
|
||||
writing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
|
||||
grantsIssued(writing, b.open.inventory, js, b.indent)
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
fmt.Printf("%ssent %s %d resource(s)\n", b.indent, s.node, len(s.declared.Resources))
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// One machine that cannot be worked out is not a reason to leave the mesh unconverged.
|
||||
//
|
||||
// A whole-mesh push refused outright the moment any single node failed to resolve, so a module on
|
||||
// the anchor requiring a provision nobody had assigned a provider for stopped every OTHER machine
|
||||
// from being sent anything — machines with no relation to the fault, and nothing wrong with them.
|
||||
// The failure and the punishment were on different machines.
|
||||
//
|
||||
// It is the same rule an un-hostable module already follows one level down (a92c11b: one module on
|
||||
// the wrong machine no longer refuses the whole node), applied one level up.
|
||||
func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) {
|
||||
sending, refusals := composeEach(
|
||||
[]string{"anchor", "home-server", "laptop"}, numbered(),
|
||||
func(node string) (sendable, error) {
|
||||
if node == "anchor" {
|
||||
return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`)
|
||||
}
|
||||
return sendable{Resources: []map[string]any{{"id": node + ".thing"}}}, nil
|
||||
})
|
||||
|
||||
var told []string
|
||||
for _, s := range sending {
|
||||
told = append(told, s.node)
|
||||
}
|
||||
if strings.Join(told, ",") != "home-server,laptop" {
|
||||
t.Errorf("a machine with nothing wrong with it was not sent: %v", told)
|
||||
}
|
||||
if len(refusals) != 1 || !strings.Contains(refusals[0], "anchor") ||
|
||||
!strings.Contains(refusals[0], "acme-ca") {
|
||||
t.Errorf("the machine that could not be worked out was not named with its reason: %v",
|
||||
refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// A machine whose declaration composes to nothing is SENT the empty declaration, not skipped
|
||||
// (novox/hq issue 127): it may have held something before, and only sending the empty
|
||||
// declaration tells it to drop what the mesh owned. It is never a refusal.
|
||||
func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) {
|
||||
sending, refusals := composeEach([]string{"spare"}, numbered(),
|
||||
func(string) (sendable, error) { return sendable{}, nil })
|
||||
if len(sending) != 1 || len(refusals) != 0 {
|
||||
t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals)
|
||||
}
|
||||
}
|
||||
|
||||
// A push that skipped a machine still ends badly, and says what was sent.
|
||||
//
|
||||
// **Skipping is not succeeding.** The mesh is not in the state somebody asked for, so the command
|
||||
// exits non-zero — but it says how many machines it did reach, because the old message ("nothing
|
||||
// was sent") was the very claim that had become untrue.
|
||||
func TestASkippedMachineIsStillAnError(t *testing.T) {
|
||||
if err := couldNotBeResolved(nil, 3); err != nil {
|
||||
t.Fatalf("a push that resolved every machine reported a problem: %v", err)
|
||||
}
|
||||
|
||||
err := couldNotBeResolved([]string{"anchor:\nnothing provides \"acme-ca\""}, 2)
|
||||
if err == nil {
|
||||
t.Fatal("a push that could not work out a machine reported success")
|
||||
}
|
||||
said := err.Error()
|
||||
if strings.Contains(said, "nothing was sent") {
|
||||
t.Errorf("the push says nothing was sent, and it sent two machines: %q", said)
|
||||
}
|
||||
for _, want := range []string{"anchor", "acme-ca", "2 other node(s) were"} {
|
||||
if !strings.Contains(said, want) {
|
||||
t.Errorf("the refusal does not say %q: %q", want, said)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// numbered is an allotter for tests: one higher per call, as the inventory's is per machine.
|
||||
func numbered() func(string) (order, error) {
|
||||
var n int64
|
||||
return func(string) (order, error) { n++; return order{sequence: n}, nil }
|
||||
}
|
||||
@@ -111,33 +111,6 @@ type meshStatus struct {
|
||||
// why they could not be read.
|
||||
Pending []pendingStatus `json:"pending,omitempty"`
|
||||
PendingUnread string `json:"pendingUnread,omitempty"`
|
||||
// LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which
|
||||
// assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485).
|
||||
// A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read.
|
||||
LastSent []machineLastSent `json:"lastSent"`
|
||||
LastSentUnread string `json:"lastSentUnread,omitempty"`
|
||||
}
|
||||
|
||||
// machineLastSent is one machine's last send as status says it.
|
||||
type machineLastSent struct {
|
||||
Node string `json:"node"`
|
||||
Sequence int64 `json:"sequence"`
|
||||
SentAt time.Time `json:"sentAt"`
|
||||
Sender string `json:"sender"`
|
||||
Epoch int64 `json:"epoch"`
|
||||
Generation int64 `json:"generation"`
|
||||
Digest string `json:"digest"`
|
||||
Modules []string `json:"modules"`
|
||||
// Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it
|
||||
// has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed.
|
||||
Answer string `json:"answer"`
|
||||
Refused string `json:"refused,omitempty"`
|
||||
Failed int `json:"failed,omitempty"`
|
||||
AnsweredAt *time.Time `json:"answeredAt,omitempty"`
|
||||
// RefusedForGeneration is when the machine refused it for coming from an older assignment generation than
|
||||
// it applied, and AppliedGeneration that generation; absent when it did not.
|
||||
RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"`
|
||||
AppliedGeneration int64 `json:"appliedGeneration,omitempty"`
|
||||
}
|
||||
|
||||
// pendingStatus is one pending assignment as status says it.
|
||||
@@ -297,7 +270,6 @@ func statusAsJSON(asked answers) ([]byte, error) {
|
||||
out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State,
|
||||
Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note})
|
||||
}
|
||||
out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread
|
||||
for name := range asked.refused {
|
||||
out.Unresolved = append(out.Unresolved, machineUnresolved{
|
||||
Node: name, Problem: asked.refused[name]})
|
||||
@@ -354,20 +326,3 @@ func say(value any) error {
|
||||
fmt.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485).
|
||||
func lastSentStatus(sends map[string]inventory.Send) []machineLastSent {
|
||||
out := []machineLastSent{}
|
||||
for _, s := range sends {
|
||||
modules := s.Modules
|
||||
if modules == nil {
|
||||
modules = []string{}
|
||||
}
|
||||
out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender,
|
||||
Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome,
|
||||
Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At,
|
||||
RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node })
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
)
|
||||
|
||||
// A membership issued again for what it hears changes nothing else of it (novox/hq ADR 0363 §6).
|
||||
func TestAMembershipIssuedAgainChangesOnlyWhatItHears(t *testing.T) {
|
||||
issued := []byte(`{"node":"ace","module":"reader","receives":{"x":[1]},"emits":"mesh.mod.reader.event.{event}"}`)
|
||||
hears := []broker.HeardIssued{{Stream: "MAIL", Consumer: "ace_reader"}}
|
||||
body, changed, err := withHearing(issued, hears)
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("changed %v, %v", changed, err)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(body, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["receives"] == nil || got["emits"] != "mesh.mod.reader.event.{event}" || got["hears"] == nil {
|
||||
t.Fatalf("issued again as %s", body)
|
||||
}
|
||||
if _, again, _ := withHearing(body, hears); again {
|
||||
t.Fatal("the same hearing issued again counts as a change")
|
||||
}
|
||||
gone, changed, err := withHearing(body, nil)
|
||||
if err != nil || !changed {
|
||||
t.Fatalf("a withdrawal: %v, %v", changed, err)
|
||||
}
|
||||
got = nil
|
||||
if err := json.Unmarshal(gone, &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, still := got["hears"]; still {
|
||||
t.Fatalf("a withdrawn hearing is still issued: %s", gone)
|
||||
}
|
||||
}
|
||||
@@ -99,7 +99,7 @@ var removedVerbs = map[string]bool{"build": true, "rebuild": true, "forget-check
|
||||
// Release 6 (ADR 0539 §5): a delivery is each machine following its wants, so the verbs that planned, started, stopped,
|
||||
// retried or closed one are gone.
|
||||
"plans": true, "plan": true, "push": true, "deliver": true, "delivery-stop": true, "delivery-backups-again": true,
|
||||
"delivery-walks": true, "delivery-order": true, "upgrade": true}
|
||||
"delivery-walks": true, "delivery-order": true}
|
||||
|
||||
// controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the
|
||||
// arguments are checked against the table compiled beside argvFor, not a row a newer or older build
|
||||
@@ -618,6 +618,22 @@ func (a *verbArguments) commandLine() ([]string, error) {
|
||||
argv = append(argv, "--retired")
|
||||
}
|
||||
return argv, nil
|
||||
case "upgrade":
|
||||
m, policy := str("module"), str("policy")
|
||||
if m == "" {
|
||||
if policy != "" {
|
||||
return nil, errors.New("upgrade takes a policy only for a module")
|
||||
}
|
||||
return []string{"upgrade"}, nil
|
||||
}
|
||||
argv := []string{"upgrade", m}
|
||||
if policy != "" {
|
||||
argv = append(argv, policy)
|
||||
if w := str("why"); w != "" {
|
||||
argv = append(argv, "--why", w)
|
||||
}
|
||||
}
|
||||
return argv, nil
|
||||
case "bus":
|
||||
if !on("upgrade") {
|
||||
return []string{"bus"}, nil
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package main
|
||||
|
||||
import "context"
|
||||
|
||||
// What was once a send is no longer one (ADR 0539 §5, §8): the controller writes the compositions that changed instead
|
||||
// (composeModules), and each machine follows its wants. The calls below stand where a send was asked until the part of the
|
||||
// controller that asked it writes compositions in its place; the subtask that replaces the last caller deletes this file.
|
||||
|
||||
// sendTo sends nothing; see sendToEach.
|
||||
func sendTo(ctx context.Context, open *stores, names []string) error {
|
||||
_, err := sendToEach(ctx, open, names)
|
||||
return err
|
||||
}
|
||||
|
||||
// sendToEach answers the machines it was asked for, and sends them nothing.
|
||||
func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) {
|
||||
return names, nil
|
||||
}
|
||||
|
||||
// holdNodes held machines from composing to sending; nothing is sent, so it holds nothing.
|
||||
func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) {
|
||||
return ctx, func() {}, nil
|
||||
}
|
||||
@@ -89,16 +89,6 @@ func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) {
|
||||
t.Fatal("the digest the mesh compares is not of the declaration push sends")
|
||||
}
|
||||
|
||||
// plan --json prints that same envelope.
|
||||
printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) })
|
||||
var compact bytes.Buffer
|
||||
if err := json.Compact(&compact, []byte(printed)); err != nil {
|
||||
t.Fatalf("plan --json is not JSON: %v\n%s", err, printed)
|
||||
}
|
||||
if digestOf(compact.Bytes()) != digestOf(body) {
|
||||
t.Fatalf("plan --json shows something other than what push sends:\n%s", printed)
|
||||
}
|
||||
|
||||
// Taking the module moves its resources out of untaken.
|
||||
if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -1,101 +0,0 @@
|
||||
//go:build integration
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
)
|
||||
|
||||
func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
inv := open.inventory
|
||||
client := aClientModule()
|
||||
moved, kept := aBot("moved-bot", true), aBot("kept-bot", false)
|
||||
moved.Provides = []catalogue.Offer{{Name: "moved-bot-api"}}
|
||||
kept.Provides = []catalogue.Offer{{Name: "kept-bot-api"}}
|
||||
desk := catalogue.Manifest{Module: "desk", Version: "1", Requires: []string{"moved-bot-api", "kept-bot-api"}}
|
||||
for _, m := range []catalogue.Manifest{client, moved, kept, desk} {
|
||||
register(t, open, m)
|
||||
}
|
||||
for _, m := range []string{client.Module, desk.Module} {
|
||||
if _, err := assign(ctx, open, "laptop", m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract,
|
||||
SaidAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
plan, settings, err := resolveNode(ctx, open, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plan.RequiredBy("moved-bot") != "desk" || plan.RequiredBy("kept-bot") != "desk" {
|
||||
t.Fatalf("the bots are on laptop because %v", plan.Because)
|
||||
}
|
||||
declared, err := declarationFor(ctx, open, "laptop", plan, settings)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var movedToken, keptToken, movedConf map[string]any
|
||||
for _, r := range declared.Resources {
|
||||
switch r["id"] {
|
||||
case "moved-bot.needs-token":
|
||||
movedToken = r
|
||||
case "kept-bot.needs-token":
|
||||
keptToken = r
|
||||
case "moved-bot.conf":
|
||||
movedConf = r
|
||||
}
|
||||
}
|
||||
if movedToken == nil || movedToken["sealed"] != nil || movedToken["owner-at-runtime"] != "owned" {
|
||||
t.Fatalf("the pulled-in moved module's secret file is sent as %v: placed, with no value", movedToken)
|
||||
}
|
||||
if movedConf == nil || movedConf["content"] != nil || movedConf["owner-at-runtime"] != "owned" {
|
||||
t.Fatalf("the pulled-in moved module's templated file is sent as %v: placed, with no text", movedConf)
|
||||
}
|
||||
if keptToken == nil || keptToken["sealed"] == nil {
|
||||
t.Fatalf("the pulled-in module not moved has its secret sent as %v: sealed, as before", keptToken)
|
||||
}
|
||||
if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held {
|
||||
t.Fatalf("the send made the pulled-in handed module's secret in the controller's store (held %v, %v)", held, err)
|
||||
}
|
||||
if _, has := declared.plans["kept-bot"]; has || len(declared.plans) != 1 {
|
||||
t.Fatalf("the send hands %v", declared.plans)
|
||||
}
|
||||
|
||||
record, err := inv.NodeByName(ctx, "laptop")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, inventory.Send{Digest: "first",
|
||||
Provisioning: sentRecordOf(ctx, declared).plans}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inv.SetProvisioningPlans(inv.SentProvisioningPlans)
|
||||
mirror := inv.PlanValuesForTest(t)
|
||||
var value inventory.AssignmentMirrored
|
||||
if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil {
|
||||
t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err)
|
||||
}
|
||||
if value.RequiredBy != "desk" || !value.Assigned.IsZero() || value.Provisioning == nil ||
|
||||
len(value.Provisioning.Files) != 1 || value.Provisioning.Files[0].Path != "/etc/moved-bot/bot.conf" ||
|
||||
len(value.Provisioning.Secrets) != 1 || value.Provisioning.Secrets[0].Name != "token" {
|
||||
t.Fatalf("the pulled-in module is mirrored as %s", mirror["laptop.moved-bot"])
|
||||
}
|
||||
if _, has := mirror["laptop.kept-bot"]; has {
|
||||
t.Fatalf("a pulled-in module not handed is mirrored: %s", mirror["laptop.kept-bot"])
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
)
|
||||
|
||||
// A send keeps, beside each handed module's plan, what requires a module only pulled in (issue 765), so the
|
||||
// assignments mirror gives that plan a value of its own; an assigned module's plan is kept as the bytes it was.
|
||||
func TestASendKeepsWhatRequiresAPulledInModule(t *testing.T) {
|
||||
plan := catalogue.Plan{Templates: catalogue.PlanTemplates{Source: "http://store:5000/v2/notify/files",
|
||||
Digest: "sha256:" + strings.Repeat("d", 64)},
|
||||
Files: []catalogue.PlanFile{{ID: "rc", Path: "/home/op/rc", Template: "rc", OwnerAtRuntime: "owned"}}}
|
||||
r := sentRecordOf(t.Context(), sendable{plans: map[string]catalogue.Plan{"dunst": plan, "notify": plan},
|
||||
requiredBy: map[string]string{"notify": "desk"}})
|
||||
kept := string(r.plans)
|
||||
want := `{"dunst":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` +
|
||||
strings.Repeat("d", 64) + `"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}]},` +
|
||||
`"notify":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` + strings.Repeat("d", 64) +
|
||||
`"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}],"required-by":"desk"}}`
|
||||
if kept != want {
|
||||
t.Fatalf("the send keeps\n%s\nwant\n%s", kept, want)
|
||||
}
|
||||
}
|
||||
|
||||
// **A module only pulled in by another is handed on a machine that can take it** (issue 765): on a contract-11
|
||||
// node running a client, a moved module that nothing assigned there, required by one that is, has its files and its
|
||||
// own secret handed like an assigned one's, no secret of it made or sealed by the controller, and its plan mirrored
|
||||
// under its own key, saying what requires it. A module not moved, pulled in the same way, is sent as before.
|
||||
@@ -338,7 +338,6 @@ func printStatus(asked answers) error {
|
||||
|
||||
// What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not
|
||||
// break "all well"; a record that could not be read does.
|
||||
printLastSends(nodes, asked.lastSent, asked.lastSentUnread)
|
||||
|
||||
if adopted := adoptedNodes(nodes); len(adopted) > 0 {
|
||||
// Said, because nothing forces the flip: a node left adopted is visible here rather than
|
||||
@@ -474,12 +473,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
|
||||
out.pendingUnread = err.Error()
|
||||
err = nil
|
||||
}
|
||||
// And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through
|
||||
// the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said.
|
||||
if out.lastSent, err = inv.LastSends(ctx); err != nil {
|
||||
out.lastSentUnread = err.Error()
|
||||
err = nil
|
||||
}
|
||||
// And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus
|
||||
// to read the log from; a process with none has no log to count.
|
||||
if _, onBus := broker.BusAddress(); onBus == nil {
|
||||
@@ -604,8 +597,7 @@ func (a answers) well() bool {
|
||||
return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 &&
|
||||
len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 &&
|
||||
len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 &&
|
||||
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) &&
|
||||
a.lastSentUnread == ""
|
||||
len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending)
|
||||
}
|
||||
|
||||
// pendingOpen is whether any pending assignment is still to be made. One that ended without being made is
|
||||
|
||||
@@ -196,11 +196,6 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
|
||||
if report.Ordered() {
|
||||
link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
|
||||
}
|
||||
// A send refused for the generation it was composed from is kept on the send it refused, so S20 names
|
||||
// its sender (novox/hq issue 234).
|
||||
if report.OlderGeneration != nil {
|
||||
heardGenerationRefusal(ctx, l.Enrolment.Inventory, report)
|
||||
}
|
||||
// What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its
|
||||
// probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey.
|
||||
if report.Superseded == "" && report.Rekey == nil && report.Node != "" {
|
||||
|
||||
@@ -34,9 +34,6 @@ func TestPlanAndNodeListAnOptionalRequirementLeftUnbound(t *testing.T) {
|
||||
}
|
||||
}
|
||||
const said = "unbound influxdb-api, optional for grafana"
|
||||
if shown := printed(t, func() error { return planCommand(ctx, []string{"laptop"}) }); !strings.Contains(shown, said) {
|
||||
t.Errorf("plan laptop does not say influxdb-api is unbound:\n%s", shown)
|
||||
}
|
||||
if shown := printed(t, func() error { return showNode(ctx, open, "laptop") }); !strings.Contains(shown, said) {
|
||||
t.Errorf("node show laptop does not say influxdb-api is unbound:\n%s", shown)
|
||||
}
|
||||
|
||||
+113
-23
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"github.com/novox/mesh-controller/internal/gosource"
|
||||
"regexp"
|
||||
@@ -127,7 +128,7 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
|
||||
// notNow marks a store that could not be read right now, or plans another controller is working (issue 921), so
|
||||
// the announcement is held rather than lost; anything else is returned as it was.
|
||||
func notNow(err error) error {
|
||||
if inventory.Unreachable(err) || errors.Is(err, inventory.ErrPlansBusy) {
|
||||
if inventory.Unreachable(err) {
|
||||
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
|
||||
}
|
||||
return err
|
||||
@@ -147,8 +148,9 @@ func unsaved(err error) error {
|
||||
// becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go.
|
||||
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
|
||||
// A merge plans nothing here any more: mesh-delivery merges on the Approve, the indexer writes the fingerprints and
|
||||
// the builder builds them (ADR 0539 §5, §17). The controller holds no batch for it.
|
||||
return nil
|
||||
// the builder builds them (ADR 0539 §5, §17). The controller keeps the announcement, for the build of a merge that
|
||||
// moves a module between repositories to be shown the operator's Approve (ADR 0420 §7).
|
||||
return keepForgeMerge(ctx, m)
|
||||
}
|
||||
|
||||
// movesOfMerge is what one merge moves, acted on: every module recorded as built from that repository and
|
||||
@@ -587,26 +589,6 @@ func packagesSaid(read []inventory.ReadRepository, m link.SourceMoved) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// lookedAt is when a merge was last acted on for a module that packages another repository's source: its
|
||||
// newest build, or the newest plan that built it or is still building it, whichever is later. A build asked
|
||||
// after a merge clones that repository with the merge in it, so an older merge is history for it; a plan
|
||||
// that closed without building it looked at nothing.
|
||||
func lookedAt(name string, read []inventory.ReadRepository, plans []inventory.Plan) time.Time {
|
||||
var at time.Time
|
||||
for _, r := range read {
|
||||
if r.Looked.After(at) {
|
||||
at = r.Looked
|
||||
}
|
||||
}
|
||||
for _, p := range plans {
|
||||
s, in := p.Modules[name]
|
||||
if in && (p.Open() || (s != nil && s.State == "built")) && p.Created.After(at) {
|
||||
at = p.Created
|
||||
}
|
||||
}
|
||||
return at
|
||||
}
|
||||
|
||||
// readForPlanning is what each module's build read, as a change is mapped onto it — for a merge's reach, a pull request's
|
||||
// check and the facts alike (novox/hq ADR 0238), so they cannot disagree. No plan overtakes it any more (ADR 0539 §17).
|
||||
func readForPlanning(ctx context.Context, inv *inventory.Inventory) (map[string][]inventory.ReadRepository, error) {
|
||||
@@ -936,3 +918,111 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string {
|
||||
|
||||
// edgeChangeOrder is the edge kind a change order draws.
|
||||
const edgeChangeOrder = "change-order"
|
||||
|
||||
// upgradeCommand says what should happen when a module's current version moves (ADR 0162 §3, ADR
|
||||
// 0235): every module's policy and where it comes from; one module's; or a person's choice for one.
|
||||
func upgradeCommand(ctx context.Context, args []string) error {
|
||||
set := flag.NewFlagSet("upgrade", flag.ContinueOnError)
|
||||
together := set.Bool("together", false,
|
||||
"send every machine running it at once, instead of one machine first")
|
||||
why := set.String("why", "", "why this choice — required for record; kept and said with the policy")
|
||||
positionals, err := parseAround(set, args)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
open, err := openStores(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer open.Close()
|
||||
inv := open.inventory
|
||||
|
||||
if len(positionals) == 0 {
|
||||
all, err := inv.Upgrades(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
names := make([]string, 0, len(all))
|
||||
for n := range all {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
u := all[n]
|
||||
line := fmt.Sprintf("%-28s %-9s %s", n, u.Policy(), u.From)
|
||||
if u.Why != "" {
|
||||
line += ": " + u.Why
|
||||
}
|
||||
fmt.Println(line)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
module := positionals[0]
|
||||
if len(positionals) == 1 {
|
||||
decision, err := inv.UpgradeOf(ctx, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sayUpgrade(module, decision))
|
||||
return nil
|
||||
}
|
||||
|
||||
decision := inventory.Upgrade{Why: strings.TrimSpace(*why), By: link.Caller()}
|
||||
switch positionals[1] {
|
||||
case "roll-out", "roll":
|
||||
decision.RollOut, decision.Together = true, *together
|
||||
case "record":
|
||||
if *together {
|
||||
// Refused rather than ignored: --together only means anything for a roll-out, and
|
||||
// accepting it here would store a preference that never applies and looks like it does.
|
||||
return errors.New("`--together` says how to roll out, so it cannot be given with " +
|
||||
"`record`, which is the choice not to")
|
||||
}
|
||||
if decision.Why == "" {
|
||||
return fmt.Errorf("holding %s back from every merge is a choice a person reads later: --why <text> "+
|
||||
"(novox/hq ADR 0236). Nothing was changed", module)
|
||||
}
|
||||
case "default":
|
||||
if err := inv.ClearUpgradeOf(ctx, module); err != nil {
|
||||
return err
|
||||
}
|
||||
decision, err := inv.UpgradeOf(ctx, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sayUpgrade(module, decision))
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("upgrade <module> roll-out|record|default — not %q", positionals[1])
|
||||
}
|
||||
if err := inv.SetUpgradeOf(ctx, module, decision); err != nil {
|
||||
return err
|
||||
}
|
||||
decision, err = inv.UpgradeOf(ctx, module)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(sayUpgrade(module, decision))
|
||||
return nil
|
||||
}
|
||||
|
||||
func sayUpgrade(module string, u inventory.Upgrade) string {
|
||||
from := " (" + u.From
|
||||
if u.By != "" {
|
||||
from += ", " + u.By
|
||||
}
|
||||
if u.Why != "" {
|
||||
from += ": " + u.Why
|
||||
}
|
||||
from += ")"
|
||||
if !u.RollOut {
|
||||
return fmt.Sprintf("when %s moves, the mesh records it and the machines running it are "+
|
||||
"reported as behind until a person pushes them%s", module, from)
|
||||
}
|
||||
if u.Together {
|
||||
return fmt.Sprintf("when %s moves, every machine running it is sent the new version "+
|
||||
"together%s", module, from)
|
||||
}
|
||||
return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+
|
||||
"the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from)
|
||||
}
|
||||
|
||||
@@ -146,25 +146,6 @@ type machineFacts struct {
|
||||
// asleep says the machine said it would be away and has not said it is back (ADR 0211).
|
||||
func (m machineFacts) asleep() bool { return m.power.Away() }
|
||||
|
||||
type planFacts struct {
|
||||
id, repository, commit string
|
||||
tier, tiers int
|
||||
entered time.Time
|
||||
bound time.Duration
|
||||
waiting string
|
||||
bus bool
|
||||
}
|
||||
|
||||
// waitFacts is one walk waiting for its delivery's word: since its merge opened it.
|
||||
type waitFacts struct {
|
||||
id, repository, commit, awaits string
|
||||
since time.Time
|
||||
// modules are the modules the walk moves, as a person names the delivery.
|
||||
modules []string
|
||||
// merges are the merges it answers (novox/hq ADR 0276), as the deliveries to read are found.
|
||||
merges []inventory.PlanMerge
|
||||
}
|
||||
|
||||
type loopFacts struct {
|
||||
took time.Time
|
||||
pending uint64
|
||||
@@ -415,14 +396,7 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("where the node tools run cannot be read: %w", err)
|
||||
}
|
||||
applies, err := inv.Durations(ctx, inventory.DurationApply, now.Add(-7*24*time.Hour))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the measured applies cannot be read: %w", err)
|
||||
}
|
||||
lastApply := map[string]time.Duration{}
|
||||
for _, d := range applies { // oldest first: the last one read is the newest
|
||||
lastApply[d.Node] = d.Took
|
||||
}
|
||||
var out []machineFacts
|
||||
anyPast := false
|
||||
for _, n := range nodes {
|
||||
|
||||
Reference in New Issue
Block a user