novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests 3 Releases Wiki Activity

Read what a machine has from its haves: reported-and-current, the bus list, a given value's first good start; drop the digest of a send (migration 0110) (release 6, issue 1945 B8)

Browse Source
This commit is contained in:
jschoubben
2026-10-24 17:51:40 +02:00
parent 8f90d3ecd6
commit fe3faaef83
29 changed files with 498 additions and 776 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/adopting_test.go
-7
View File
@@ -83,13 +83,6 @@ func reportsReaching(t *testing.T, open *stores, reachable []link.Reach, held ..
if err != nil {
t.Fatal(err)
}
record, err := open.inventory.NodeByName(ctx, "anchor")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSentForTest(ctx, record.ID, digestOf(body)); err != nil {
t.Fatal(err)
}
if _, err := (link.Enrolment{Inventory: open.inventory}).Heard(ctx, link.Report{
Node: "anchor", Applied: []string{"hello-web.x"}, Declared: digestOf(body),
Firewall: "ufw", Held: held, Reachable: reachable,
cmd/mesh-controller/adoption.go
+1 -1
View File
@@ -615,7 +615,7 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
if !record.Adopted {
return "", fmt.Errorf("%s is converged already; there is nothing to flip", node)
}
reports, err := inv.LastReports(ctx)
reports, err := lastReports(ctx, inv)
if err != nil {
return "", err
}
cmd/mesh-controller/awaiting_push_integration_test.go
-186
View File
@@ -1,186 +0,0 @@
//go:build integration
package main
import (
"errors"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/overlay"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
// The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing
// can be compared about a secret that does not exist (status), and nothing here string-matches.
_, _, err := composeQuestion(ctx, open, "laptop", compose.Reading, false)
var notMade *catalogue.NotMadeError
if !errors.As(err, &notMade) || notMade.Module != "keeper" || notMade.Name != "repository" {
t.Fatalf("a read composition did not say which secret is not made, typed: %v", err)
}
// Foreseen, it composes, names what the push makes, and made nothing.
foreseen, _, err := composeQuestion(ctx, open, "laptop", compose.Foreseeing, false)
if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" {
t.Fatalf("foreseen: %v %v", foreseen.foreseen, err)
}
if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held {
t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err)
}
// Within the bound: nothing at all — no urgent, no warning, nobody notified.
if got := d1(t, open); len(got) != 0 {
t.Fatalf("a machine waiting for a push raised %+v", got)
}
// Past the bound: a warning, not urgent, saying what the push will make.
before := awaitingPushBound
awaitingPushBound = -time.Minute
t.Cleanup(func() { awaitingPushBound = before })
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning ||
!strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") {
t.Fatalf("a machine left un-pushed past the bound: %+v", got)
}
// Pushed: the secret is made and D1 says nothing.
pushedBy(t, open, "laptop")
if got := d1(t, open); len(got) != 0 {
t.Fatalf("a pushed machine still raised %+v", got)
}
}
func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aTalker())
if _, err := assign(ctx, open, "laptop", "talker"); err != nil {
t.Fatal(err)
}
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent ||
!strings.Contains(got[0].Summary, "module issue talker --node laptop") {
t.Fatalf("a push that will be refused was not said urgently: %+v", got)
}
// And a machine whose composition fails for anything else, with a secret waiting beside it, is
// uncomposable for that — the stand-in hides nothing.
register(t, open, aKeeper())
if _, err := assign(ctx, open, "anchor", "keeper"); err != nil {
t.Fatal(err)
}
one, two := rivals()
register(t, open, one)
register(t, open, two)
_, _ = assign(ctx, open, "anchor", "rival-one")
_, _ = assign(ctx, open, "anchor", "rival-two")
keys := map[string]conditions.Severity{}
for _, o := range d1(t, open) {
keys[o.Key()] = o.Severity
}
if keys["machine.anchor.uncomposable"] != conditions.Urgent {
t.Fatalf("a real failure beside a waiting secret: %v", keys)
}
}
func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil {
t.Fatal(err)
}
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil {
t.Fatal(err)
}
got := d1(t, open)
if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") {
t.Fatalf("a given secret under an old key: %+v", got)
}
}
func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
register(t, open, aKeeper())
pushedBy(t, open, "laptop")
sent := time.Now()
since, err := open.inventory.AwaitingSince(ctx, "laptop")
if err != nil || since.After(sent) {
t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent)
}
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
since, err = open.inventory.AwaitingSince(ctx, "laptop")
if err != nil || since.Before(sent.Add(-time.Second)) {
t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err)
}
}
func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) {
open := aMesh(t)
ctx := t.Context()
// What a machine was sent is read from the commits its assignments name (issue 1109): a module is carried by a send only
// when it was built from a commit, and a hand send on a machine with nothing assigned leaves no key to mark unknown.
if err := open.inventory.RegisterModule(ctx, aKeeper(), inventory.Source{BuiltFrom: "c-keeper"}); err != nil {
t.Fatal(err)
}
record, err := open.inventory.NodeByName(ctx, "laptop")
if err != nil {
t.Fatal(err)
}
if err := open.inventory.RecordSentForTest(ctx, record.ID, "carried"); err != nil {
t.Fatal(err)
}
if _, err := assign(ctx, open, "laptop", "keeper"); err != nil {
t.Fatal(err)
}
sends, err := readDeliveries(ctx, open.inventory)
if err != nil {
t.Fatal(err)
}
if sends["laptop"].settled("keeper", time.Now()) {
t.Fatal("a holder assigned and not pushed is expected to answer")
}
if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) {
t.Fatal("a module the machine was sent long ago is not expected to answer")
}
pushedBy(t, open, "laptop")
sends, err = readDeliveries(ctx, open.inventory)
if err != nil {
t.Fatal(err)
}
if sends["laptop"].settled("keeper", time.Now()) {
t.Fatal("pushed a moment ago and not reported, a holder is already expected")
}
if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) {
t.Fatal("pushed past the grace, a holder is not expected")
}
if _, ok := sends["laptop"].carried["keeper"]; !ok {
t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried)
}
pushedAndApplied(t, open, "laptop")
if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) {
t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err)
}
}
cmd/mesh-controller/awaiting_push_test.go
-128
View File
@@ -1,128 +0,0 @@
package main
import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
)
// Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed
// the machine's declaration without making anything, failed on the missing secret, and raised an urgent
// "nothing can be sent to <machine>" — seen live on 2026-10-06, a desktop notification for a machine
// the next push sent to without a word (novox/hq issue 275). D1 now composes as the push would, with a
// stand-in for what the push makes: pending, not broken, and said only past a bound, as a warning.
// aKeeper is a module with an own secret the mesh makes — a backup repository's password.
func aKeeper() catalogue.Manifest {
return catalogue.Manifest{Module: "keeper", Version: "1",
OwnSecrets: catalogue.OwnSecrets{"repository": {Path: "/var/lib/mesh/keeper/repository"}},
Resources: []map[string]any{
{"id": "state", "type": "directory", "path": "/var/lib/mesh/keeper", "mode": "0700"},
}}
}
// pushedBy records a send to a machine as a push does — composed on the send path, so its own secrets
// are made — without a bus to carry it.
func pushedBy(t *testing.T, open *stores, node string) string {
t.Helper()
ctx := t.Context()
plan, settings, err := resolveNode(ctx, open, node)
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(ctx, open, node, plan, settings)
if err != nil {
t.Fatal(err)
}
body, err := declared.Body()
if err != nil {
t.Fatal(err)
}
record, err := open.inventory.NodeByName(ctx, node)
if err != nil {
t.Fatal(err)
}
digest := digestOf(body)
// A push records the build of each module it carries (plan.go, Allocating); a module is carried by a send only when it was
// built from a commit, for what a machine was sent is read from the commits its assignments name (issue 1109).
builds := declared.Builds
if builds == nil {
current, err := open.inventory.CurrentBuilds(ctx)
if err != nil {
t.Fatal(err)
}
builds = map[string]string{}
for _, m := range plan.Modules {
builds[m.Module] = current[m.Module].Commit
}
}
if err := open.inventory.RecordSentForTest(ctx, record.ID, digest); err != nil {
t.Fatal(err)
}
return digest
}
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
func pushedAndApplied(t *testing.T, open *stores, node string) {
t.Helper()
digest := pushedBy(t, open, node)
record, err := open.inventory.NodeByName(t.Context(), node)
if err != nil {
t.Fatal(err)
}
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
t.Fatal(err)
}
}
// d1 runs D1 once.
func d1(t *testing.T, open *stores) []conditions.Observation {
t.Helper()
got, err := probeDeclarations(t.Context(), &doctor{open: open})
if err != nil {
t.Fatal(err)
}
return got
}
// **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is
// waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing.
// **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make.
// A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words.
// A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is
// refused on it, so D1 is too.
// The bound is read from when the machine began waiting: the oldest assignment since its last send.
// **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent
// it and had time to report — never between assign and push.
func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) {
now := time.Now()
sent := now.Add(-time.Minute)
long := now.Add(-time.Hour)
cases := []struct {
name string
send lastSend
want bool
}{
{"never sent", lastSend{}, false},
{"sent without it", lastSend{sent: &long, current: true, carried: map[string]string{"other": "c"}}, false},
{"sent with it, not reported yet", lastSend{sent: &sent, carried: map[string]string{"keeper": "c"}}, false},
{"sent with it and reported", lastSend{sent: &sent, current: true, carried: map[string]string{"keeper": "c"}}, true},
{"sent with it long ago, never reported", lastSend{sent: &long, carried: map[string]string{"keeper": "c"}}, true},
{"sent before builds were kept", lastSend{sent: &long, current: true}, true},
}
for _, c := range cases {
if got := c.send.settled("keeper", now); got != c.want {
t.Errorf("%s: settled %v, want %v", c.name, got, c.want)
}
}
}
// And through the stores: assigned, not in the last send; pushed and reported, carried and settled.
cmd/mesh-controller/bus_step.go
+1 -1
View File
@@ -391,7 +391,7 @@ func probeBusStep(ctx context.Context, d *doctor) ([]conditions.Observation, err
if err != nil {
return nil, err
}
reports, err := inv.LastReports(ctx)
reports, err := lastReports(ctx, inv)
if err != nil {
return nil, err
}
cmd/mesh-controller/doctor.go
+1 -1
View File
@@ -79,7 +79,7 @@ var probeRegistry = []probe{
{ID: "D1", Asserts: "every machine's declaration composes, and passes the node-engine's validation; no module " +
"assigned to a machine is left out of its declaration unsaid",
From: "issues 236, 263, 275, 380", Kind: "declaration-refused",
Raises: []string{kindAwaitingPush, kindLeftOut, kindNeedsOperator}, Phase: 1,
Raises: []string{kindLeftOut, kindNeedsOperator}, Phase: 1,
run: probeDeclarations},
{ID: "D2", Asserts: "every holder of the mesh's resolver answers a machine name for IPv4, and NODATA for IPv6",
From: "issue 262", Kind: "resolver-wrong", Phase: 1, run: probeResolvers},
cmd/mesh-controller/given.go
+76 -15
View File
@@ -3,6 +3,8 @@ package main
import (
"context"
"encoding/json"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/nats-io/nats.go/jetstream"
"log"
"strings"
"sync"
@@ -13,9 +15,8 @@ import (
// A value given by hand lives only until the module's first good start (novox/hq ADR 0228).
//
// The serving controller hears every report; a clean one about the declaration a machine was last
// sent is the signal the store asks about (inventory.ReplaceGivenAfterStart). What it replaced is
// sent at once, said in the log and stated on the bus as the controller seat's `secret-replaced`,
// The serving controller watches every have; a module's have that is running and healthy is the signal the store asks about
// (inventory.ReplaceGivenAfterStart). What it replaced is composed at once, said in the log and stated on the bus as the controller seat's `secret-replaced`,
// so a replacement is never silent. **Not in the hand-act log**: nobody acted by hand, and that log
// is read as the count of repairs a healer is wanted for.
@@ -41,29 +42,32 @@ var givenEvents link.Bus
// same rows, and the store's claim makes one of them the replacer, but the sends need not race.
var replacing sync.Map
// startedWell says a report is a machine's clean account of a declaration: everything applied,
// nothing failed or refused. Whether it is the declaration last sent is the store's to answer.
func startedWell(report link.Report) bool {
return report.Declared != "" && report.Refused == "" && len(report.Failed) == 0 && report.Applied != nil
// startedOn is the signal of a module's first good start: its have is running and healthy, as of the time the node-engine
// wrote it. The zero time when it is not.
func startedOn(h sdkdelivery.Have, written time.Time) time.Time {
if h.Phase == sdkdelivery.PhaseRunning && h.Health == sdkdelivery.HealthOK {
return written
}
return time.Time{}
}
const givenWhy = "a value given by hand lives only until its module's first good start under the mesh (novox/hq ADR 0228)"
// replaceGiven replaces what the report makes due, sends the machines, and says so.
func replaceGiven(ctx context.Context, open *stores, report link.Report) {
if _, busy := replacing.LoadOrStore(report.Node, true); busy {
// replaceGiven replaces what a module's good start makes due, composes the machines, and says so.
func replaceGiven(ctx context.Context, open *stores, node string, started map[string]time.Time) {
if _, busy := replacing.LoadOrStore(node, true); busy {
return
}
defer replacing.Delete(report.Node)
replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, report.Node, report.Declared)
defer replacing.Delete(node)
replaced, err := open.inventory.ReplaceGivenAfterStart(ctx, node, started)
if err != nil {
log.Printf("a value given by hand on %s could not be replaced after its module started: %v", report.Node, err)
log.Printf("a value given by hand on %s could not be replaced after its module started: %v", node, err)
}
for _, r := range replaced {
said := SecretReplaced{Node: report.Node, Module: r.Module, Name: r.Name, Given: r.Given.UTC(),
said := SecretReplaced{Node: node, Module: r.Module, Name: r.Name, Given: r.Given.UTC(),
Sent: r.Machines, Why: givenWhy}
log.Printf("replaced %q of %s on %s, given %s, with a value the mesh made: %s; composing %s",
r.Name, r.Module, report.Node, r.Given.UTC().Format(time.RFC3339), givenWhy, strings.Join(r.Machines, ", "))
r.Name, r.Module, node, r.Given.UTC().Format(time.RFC3339), givenWhy, strings.Join(r.Machines, ", "))
// Composed (novox/hq ADR 0539 clause 5): the compositions that use it get a new revision, which each node's
// want then names.
if _, err := composeModules(ctx, open, r.Machines); err != nil {
@@ -90,3 +94,60 @@ func stateReplaced(ctx context.Context, said SecretReplaced) {
log.Printf("could not say that %s's %q was replaced: %v", said.Module, said.Name, err)
}
}
// watchHavesForGiven replaces the given values of a module on a node each time its have is running and healthy, until ctx
// ends: never on the watch's own goroutine.
func watchHavesForGiven(ctx context.Context, open *stores) {
bus, err := aBus()
if err != nil {
log.Printf("given values: the haves are not watched: %v", err)
return
}
js, err := jetstream.New(bus.Conn())
if err != nil {
bus.Close()
return
}
kv, err := js.KeyValue(ctx, sdkdelivery.HaveBucket)
if err != nil {
bus.Close()
log.Printf("given values: the haves are not watched: %v", err)
return
}
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
if err != nil {
bus.Close()
log.Printf("given values: the haves are not watched: %v", err)
return
}
go func() {
defer bus.Close()
defer w.Stop()
for {
select {
case <-ctx.Done():
return
case e, ok := <-w.Updates():
if !ok {
return
}
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
node, module, isNode, err := sdkdelivery.ParseHaveKey(e.Key())
if err != nil || isNode {
continue
}
have, err := sdkdelivery.DecodeHave(e.Value())
if err != nil {
continue
}
at := startedOn(have, e.Created())
if at.IsZero() {
continue
}
inBackground(func() { replaceGiven(ctx, open, node, map[string]time.Time{module: at}) })
}
}
}()
}
cmd/mesh-controller/given_test.go
+18 -15
View File
@@ -1,10 +1,11 @@
package main
import (
"time"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"strings"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// A rotation asked through the seat carries why to the command, which records it in the hand-act
@@ -27,22 +28,24 @@ func TestARotationThroughTheSeatCarriesWhy(t *testing.T) {
}
}
// Only a clean account of a declaration is a good start; a refusal, a failure or a bare word that
// the machine is there is not (novox/hq ADR 0228).
func TestAGoodStartIsACleanAccountOfADeclaration(t *testing.T) {
// Only a have that is running and healthy is a good start, at the time it was written; any other is none (novox/hq ADR 0228,
// ADR 0539 §9).
func TestAGoodStartIsAHaveRunningAndHealthy(t *testing.T) {
at := time.Date(2026, 10, 24, 17, 0, 0, 0, time.UTC)
for _, c := range []struct {
report link.Report
good bool
have sdkdelivery.Have
good bool
}{
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"container:letta"}}, true},
{link.Report{Node: "anchor", Declared: "d", Applied: []string{}}, true},
{link.Report{Node: "anchor"}, false},
{link.Report{Node: "anchor", Applied: []string{"x"}}, false},
{link.Report{Node: "anchor", Declared: "d", Applied: []string{"x"}, Failed: map[string]string{"y": "no"}}, false},
{link.Report{Node: "anchor", Declared: "d", Refused: "older"}, false},
{sdkdelivery.Have{Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}, true},
{sdkdelivery.Have{Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthFailed}, false},
{sdkdelivery.Have{Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthWaits}, false},
{sdkdelivery.Have{Phase: sdkdelivery.PhaseApplying, Health: sdkdelivery.HealthOK}, false},
{sdkdelivery.Have{Phase: sdkdelivery.PhaseFailed, Health: sdkdelivery.HealthFailed}, false},
{sdkdelivery.Have{Phase: sdkdelivery.PhaseHeld, Health: sdkdelivery.HealthOK}, false},
} {
if got := startedWell(c.report); got != c.good {
t.Errorf("%+v: a good start = %v, want %v", c.report, got, c.good)
got := startedOn(c.have, at)
if good := !got.IsZero(); good != c.good || good && !got.Equal(at) {
t.Errorf("%+v: started at %v, want a good start = %v", c.have, got, c.good)
}
}
}
cmd/mesh-controller/haves.go
+117
View File
@@ -0,0 +1,117 @@
package main
import (
"context"
"errors"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/inventory"
)
// What a machine has, against what it is wanted to have (novox/hq ADR 0539 §1, §9). Nothing is sent, so nothing records that a
// machine was sent anything: whether a machine has "acted on the current words" is read from its haves, the node-engine's
// own bucket, beside the assigner's wants. A node is current when every module wanted on it is had at the wanted fingerprint
// and composition, applied (not failed, not held, not still applying), and every module wanted removed is gone. A node that
// nothing is wanted on has nothing to apply and is current.
// currentNodes is, for each node a want names, whether it has everything wanted of it. Pure.
func currentNodes(wants map[string]sdkdelivery.Want, haves map[string]sdkdelivery.Have) map[string]bool {
out := map[string]bool{}
for key, w := range wants {
node, _, err := sdkdelivery.ParseWantKey(key)
if err != nil {
continue // the node's own want: its composition is the node-engine's own
}
if _, seen := out[node]; !seen {
out[node] = true
}
h, had := haves[key]
switch {
case w.Removed:
if had && h.Phase != sdkdelivery.PhaseRemoved {
out[node] = false
}
case !had, h.FP != w.FP, h.Composed != w.Composed, h.Phase != sdkdelivery.PhaseRunning:
out[node] = false
}
}
return out
}
// readWantsAndHaves reads the assigner's wants and the node-engines' haves from the bus; a bucket not there yet is empty.
func readWantsAndHaves(ctx context.Context, js jetstream.JetStream) (map[string]sdkdelivery.Want, map[string]sdkdelivery.Have, error) {
wants := map[string]sdkdelivery.Want{}
haves := map[string]sdkdelivery.Have{}
each := func(bucket string, take func(key string, value []byte)) error {
kv, err := js.KeyValue(ctx, bucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
return nil
}
if err != nil {
return err
}
keys, err := kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil
}
if err != nil {
return err
}
for _, key := range keys {
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
continue
}
if err != nil {
return err
}
take(key, e.Value())
}
return nil
}
if err := each(sdkdelivery.AssignerWantsBucket, func(key string, value []byte) {
if w, err := sdkdelivery.DecodeWant(value); err == nil {
wants[key] = w
}
}); err != nil {
return nil, nil, err
}
if err := each(sdkdelivery.HaveBucket, func(key string, value []byte) {
if h, err := sdkdelivery.DecodeHave(value); err == nil {
haves[key] = h
}
}); err != nil {
return nil, nil, err
}
return wants, haves, nil
}
// lastReports is inventory.LastReports with Current read from the haves: every machine's last report, and whether the machine
// has everything the assigner wants of it. A process with no bus cannot tell, and leaves Current false.
func lastReports(ctx context.Context, inv *inventory.Inventory) ([]inventory.Reported, error) {
reports, err := inv.LastReports(ctx)
if err != nil {
return nil, err
}
bus, err := aBus()
if err != nil {
return reports, nil // empty-on-error: no bus to read the haves from; Current stays false, which is "not known to be"
}
defer bus.Close()
js, err := jetstream.New(bus.Conn())
if err != nil {
return nil, err
}
wants, haves, err := readWantsAndHaves(ctx, js)
if err != nil {
return nil, err
}
current := currentNodes(wants, haves)
for i := range reports {
c, wanted := current[reports[i].Node]
reports[i].Current = c || !wanted && reports[i].At != nil
}
return reports, nil
}
cmd/mesh-controller/haves_test.go
+102
View File
@@ -0,0 +1,102 @@
package main
import (
"errors"
"strings"
"testing"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
)
// **A machine is current when it has what is wanted of it** (novox/hq ADR 0539 §1, §9, issue 1945): every module wanted on it is
// had at the wanted fingerprint and composition, running, and every module wanted removed is gone. Read from the wants and the
// haves; there is no send to compare a report with.
func TestAMachineIsCurrentWhenItHasWhatIsWantedOfIt(t *testing.T) {
fp := func(c string) string { return sdkdelivery.SourcePrefix2 + strings.Repeat(c, 64) }
want := func(f, composed string) sdkdelivery.Want { return sdkdelivery.Want{V: 1, FP: f, Composed: composed} }
have := func(f, composed string, phase sdkdelivery.Phase) sdkdelivery.Have {
return sdkdelivery.Have{V: 1, FP: f, Composed: composed, Phase: phase, Health: sdkdelivery.HealthOK}
}
key := func(node, module string) string {
k, err := sdkdelivery.WantKey(node, module)
if err != nil {
t.Fatal(err)
}
return k
}
for _, c := range []struct {
what string
wants map[string]sdkdelivery.Want
haves map[string]sdkdelivery.Have
current map[string]bool
}{
{"everything had as wanted",
map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d1")},
map[string]sdkdelivery.Have{key("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseRunning)},
map[string]bool{"anchor": true}},
{"a module not had yet",
map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d1")},
nil, map[string]bool{"anchor": false}},
{"an older fingerprint",
map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("b"), "d1")},
map[string]sdkdelivery.Have{key("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseRunning)},
map[string]bool{"anchor": false}},
{"the same fingerprint, an older composition",
map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d2")},
map[string]sdkdelivery.Have{key("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseRunning)},
map[string]bool{"anchor": false}},
{"still applying",
map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d1")},
map[string]sdkdelivery.Have{key("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseApplying)},
map[string]bool{"anchor": false}},
{"held for a backup that failed",
map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d1")},
map[string]sdkdelivery.Have{key("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseHeld)},
map[string]bool{"anchor": false}},
{"a module wanted removed that is gone",
map[string]sdkdelivery.Want{key("anchor", "web"): {V: 1, Removed: true}},
map[string]sdkdelivery.Have{key("anchor", "web"): have("", "", sdkdelivery.PhaseRemoved)},
map[string]bool{"anchor": true}},
{"a module wanted removed that still runs",
map[string]sdkdelivery.Want{key("anchor", "web"): {V: 1, Removed: true}},
map[string]sdkdelivery.Have{key("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseRunning)},
map[string]bool{"anchor": false}},
{"one machine behind does not hold another back",
map[string]sdkdelivery.Want{key("anchor", "web"): want(fp("a"), "d1"), key("laptop", "web"): want(fp("a"), "d1")},
map[string]sdkdelivery.Have{key("anchor", "web"): have(fp("a"), "d1", sdkdelivery.PhaseRunning)},
map[string]bool{"anchor": true, "laptop": false}},
} {
got := currentNodes(c.wants, c.haves)
for node, want := range c.current {
if got[node] != want {
t.Errorf("%s: %s is current %v, want %v", c.what, node, got[node], want)
}
}
}
if got := currentNodes(nil, nil); len(got) != 0 {
t.Errorf("no want names a node, yet %v", got)
}
}
// **The bus's machine is behind while its have does not name the composition carrying the user list** (ADR 0539 §15): the
// digest of the bus module's composition written now against the one its have names.
func TestTheBusMachineIsBehindUntilItsHaveNamesTheCompositionCarryingTheUserList(t *testing.T) {
have := sdkdelivery.Have{Composed: "d2", Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK}
for _, c := range []struct {
what string
digest string
err error
have sdkdelivery.Have
had bool
behind bool
}{
{"its have names the composition written now", "d2", nil, have, true, false},
{"its have names an older composition (a new user list)", "d3", nil, have, true, true},
{"it has no have of the module yet", "d2", nil, sdkdelivery.Have{}, false, true},
{"the composition cannot be read: not known is behind", "", errors.New("no composition"), have, true, true},
} {
if got := listBehind(c.digest, c.err, c.have, c.had); got != c.behind {
t.Errorf("%s: behind %v, want %v", c.what, got, c.behind)
}
}
}
cmd/mesh-controller/helpers_kept_test.go
+34
View File
@@ -48,3 +48,37 @@ func aContainer(name string) catalogue.Manifest {
return catalogue.Manifest{Module: name, Version: "1",
Resources: []map[string]any{{"id": name, "type": "container", "image": name}}}
}
// pushedBy composes a machine's declaration as the compositions are made, so its own secrets are made, and answers its digest.
// Nothing records a send: there is none (ADR 0539 §5).
func pushedBy(t *testing.T, open *stores, node string) string {
t.Helper()
ctx := t.Context()
plan, settings, err := resolveNode(ctx, open, node)
if err != nil {
t.Fatal(err)
}
declared, err := declarationFor(ctx, open, node, plan, settings)
if err != nil {
t.Fatal(err)
}
body, err := declared.Body()
if err != nil {
t.Fatal(err)
}
return digestOf(body)
}
// pushedAndApplied is pushedBy, and the machine reporting it applied that declaration.
func pushedAndApplied(t *testing.T, open *stores, node string) {
t.Helper()
digest := pushedBy(t, open, node)
record, err := open.inventory.NodeByName(t.Context(), node)
if err != nil {
t.Fatal(err)
}
if _, err := open.inventory.RecordDoing(t.Context(), record.ID, inventory.Doing{
Outcome: inventory.OutcomeApplied, Declared: digest}); err != nil {
t.Fatal(err)
}
}
cmd/mesh-controller/plain_words.go
-7
View File
@@ -94,13 +94,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
"the mesh until it does. A machine asleep is normal.", m, m),
Resolved: "Tools on " + m + " answer again"}
}),
kindAwaitingPush: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: m + " waits for new instructions",
Explanation: fmt.Sprintf("%s's new instructions go out with the next push. Nothing is broken meanwhile.",
conditions.Capital(m)),
Resolved: m + " got its new instructions"}
}),
"declaration-refused": worded(func(o conditions.Observation) words {
if o.Scope == conditions.ScopeMesh {
return words{Headline: "No machine can get new instructions",
cmd/mesh-controller/probes.go
+3 -30
View File
@@ -34,13 +34,6 @@ import (
// wrong now as observations, or an error when it could not tell — never "nothing" for "could not
// look" (ADR 0227 rule 4).
// awaitingPushBound is how long a machine may have something only its next push makes — an own
// secret of a module assigned since its last push — before that is said (novox/hq issue 275).
var awaitingPushBound = 30 * time.Minute
// kindAwaitingPush is D1's kind for a machine waiting for a push past awaitingPushBound.
const kindAwaitingPush = "awaiting-push"
// probeDeclarations is D1: every machine's declaration composes, and the node-engine's own validator
// (mesh-host's `validate`, the package the host runs) takes it.
//
@@ -80,7 +73,7 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
if err != nil && !unresolvable(err) {
return nil, fmt.Errorf("%s cannot be worked out: %w", n.Name, err)
}
var problems, foreseen []string
var problems []string
// Each module the next send leaves out of this machine's declaration (novox/hq issue 380), judged from this
// resolution as the send judges it: a finding, never a refusal.
if err == nil {
@@ -89,7 +82,6 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
if err == nil && gensErr == nil {
var declared sendable
if declared, _, err = composeQuestion(ctx, open, n.Name, compose.Foreseeing, false); err == nil {
foreseen = declared.foreseen
// With the order it was last sent, so the validator reads the envelope a machine is sent
// — its epoch included (novox/hq to-be 45 §6).
var serr error
@@ -119,30 +111,11 @@ func probeDeclarations(ctx context.Context, d *doctor) ([]conditions.Observation
Summary: fmt.Sprintf("%s's node-engine would refuse its declaration whole: %d problem(s), the first: %s",
n.Name, len(problems), problems[0]),
Said: strings.Join(problems, "; ")})
case len(foreseen) > 0:
since, err := open.inventory.AwaitingSince(ctx, n.Name)
if err != nil {
return nil, err // the store, not the machine: the probe could not run
}
if waited := time.Since(since); waited > awaitingPushBound {
out = append(out, awaitingPush(n.Name, foreseen, since, waited))
}
}
}
return out, nil
}
// awaitingPush is D1's warning for a machine whose declaration composes only once its next push has
// made what it names, past awaitingPushBound (novox/hq issue 275).
func awaitingPush(node string, foreseen []string, since time.Time, waited time.Duration) conditions.Observation {
made := strings.Join(foreseen, ", ")
return conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Token: kindAwaitingPush,
Kind: kindAwaitingPush, Machine: node, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s has waited %s for a push: its declaration composes once the next push makes %s — "+
"`push %s` sends it", node, waited.Round(time.Minute), made, node),
Said: fmt.Sprintf("waiting since %s; the next push makes %s", since.UTC().Format(time.RFC3339), made)}
}
// probeResolvers is D2: every holder of the mesh's resolver answers each machine's name with its
// address for IPv4, and with no address and no error for IPv6 — NODATA, not NXDOMAIN, which musl
// takes as final (issue 262).
@@ -600,7 +573,7 @@ func (d lastSend) settled(module string, now time.Time) bool {
// readDeliveries is every machine's last send, by name, from the records a send and a report keep.
func readDeliveries(ctx context.Context, inv *inventory.Inventory) (map[string]lastSend, error) {
reports, err := inv.LastReports(ctx)
reports, err := lastReports(ctx, inv)
if err != nil {
return nil, err
}
@@ -1087,7 +1060,7 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
if err != nil {
return nil, err
}
reports, err := inv.LastReports(ctx)
reports, err := lastReports(ctx, inv)
if err != nil {
return nil, err
}
cmd/mesh-controller/push.go
+42 -20
View File
@@ -6,6 +6,8 @@ import (
"encoding/hex"
"errors"
"fmt"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/conditions"
"log"
"os"
@@ -254,6 +256,8 @@ func serve(ctx context.Context) (err error) {
// And the modules the catalogue does not hold yet, registered from the builder's first done key of each, woken by
// the builder's and the indexer's writes (novox/hq ADR 0501 §6, issue 1851): nothing asks a first build.
watchNewModules(ctx, open)
// And a value given by hand replaced when its module's have first runs healthy (novox/hq ADR 0228, ADR 0539 §9).
watchHavesForGiven(ctx, open)
// And says what it keeps, and keeps what every module says it keeps (novox/hq ADR 0495, issue 1790): a changed word
// sends the machine holding the bus its new user list.
if err := announceOwnStates(ctx, server.JetStream()); err != nil {
@@ -350,35 +354,53 @@ func brokerListState(ctx context.Context, open *stores, names []string) (busList
return busListState{Machine: h.Node}, nil
}
}
// **What cannot be worked out is behind** (the review of hq issue 353). The list it would be sent cannot be
// composed, so whether it carries the grants a send relies on is not known; it is taken as behind, so a push
// sends that machine first and says why it cannot (`plan` says it too), and a rollout that needs the list
// refuses rather than sending code whose grants the bus may not hold. It said "not behind" until then: a
// send went ahead as if the bus held a list nobody could compose.
list, composeErr := "", error(nil)
plan, _, err := resolveNode(ctx, open, h.Node)
// **What cannot be worked out is behind** (the review of hq issue 353): the composition carrying the user list is not
// written, so whether the bus's machine runs it is not known; taken as behind, never "not behind".
// The user list is part of the bus module's composition on the machine holding it (ADR 0539 §15), so the machine is
// behind while its have does not name the digest of the composition written now.
fps, err := currentFingerprints(ctx, inv)
if err != nil {
composeErr = err
} else if list, _, err = busUserList(ctx, inv, plan.Modules); err != nil {
composeErr = err
return busListState{Machine: h.Node, Behind: true, Uncomposed: err}, nil
}
sent, err := inv.SentBusUsers(ctx, h.Node)
bus, err := aBus()
if err != nil {
return busListState{}, err
}
return busListState{Machine: h.Node, Behind: listBehind(list, composeErr, sent), Uncomposed: composeErr}, nil
defer bus.Close()
js, err := jetstream.New(bus.Conn())
if err != nil {
return busListState{}, err
}
digest, digestErr := composedDigestOf(ctx, js, h.Node, h.Module, fps[h.Module])
have, had, err := haveOn(ctx, js, h.Node, h.Module)
if err != nil {
return busListState{}, err
}
return busListState{Machine: h.Node, Behind: listBehind(digest, digestErr, have, had), Uncomposed: digestErr}, nil
}
// listBehind is whether the bus's machine is behind: the list composed now is not the one last sent, or it
// could not be composed at all — not known is behind, never "not behind".
func listBehind(list string, composeErr error, sentDigest string) bool {
return composeErr != nil || userListBehind(list, sentDigest)
// composedDigestOf is the digest of the composition the controller wrote for a module on a node at a fingerprint.
func composedDigestOf(ctx context.Context, js jetstream.JetStream, node, module, fp string) (string, error) {
digests, err := js.KeyValue(ctx, sdkdelivery.ControllerComposedBucket)
if err != nil {
return "", err
}
key, err := sdkdelivery.ComposedKey(node, module, fp)
if err != nil {
return "", err
}
e, err := digests.Get(ctx, key)
if err != nil {
return "", fmt.Errorf("the composition %s is not written: %w", key, err)
}
v, err := sdkdelivery.DecodeComposedDigest(e.Value())
return v.Digest, err
}
// userListBehind is whether the user list composed now is not the one last sent, by its digest. An
// empty list composed is never behind: there is nothing for it to carry.
func userListBehind(now, sentDigest string) bool {
return now != "" && digestOf([]byte(now)) != sentDigest
// listBehind is whether the bus's machine is behind: its have does not name the composition written now, or the
// composition could not be read at all.
func listBehind(digest string, digestErr error, have sdkdelivery.Have, had bool) bool {
return digestErr != nil || !had || have.Composed != digest
}
// digestOf is what the mesh compares to answer "has this machine been sent what it should be".
cmd/mesh-controller/rollout.go
+18 -8
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"os"
"strings"
"time"
@@ -170,7 +171,7 @@ func readinessOf(ctx context.Context, inv *inventory.Inventory) (broker.Readines
// The machine that holds the bus seat is the one that would be sent the user list.
if m.BusUsers != "" && m.ClaimsSeat("mesh-broker") {
state.Holder = n.Name
state.AccountsComposed = wasSentTheUserList(ctx, inv, n.Name)
state.AccountsComposed = wasGivenTheUserList(ctx, n.Name, module)
}
// A module that never speaks needs no credential, so it is not counted as missing one.
if !speaksOnTheBus(m) {
@@ -196,14 +197,23 @@ func speaksOnTheBus(m catalogue.Manifest) bool {
len(m.DefinesSeats) > 0 || len(m.Uses) > 0 || len(m.Claims) > 0
}
// wasSentTheUserList says whether the machine holding the bus has had a declaration since the user
// list became part of one.
// wasGivenTheUserList says whether the machine holding the bus has a composition of the bus module applied since the
// user list became part of one: its have names a composition at all.
//
// Read from what the mesh recorded sending rather than asked of the machine: a machine that is away
// has still been sent it, and this question is about whether the mesh did its part.
func wasSentTheUserList(ctx context.Context, inv *inventory.Inventory, node string) bool {
digest, err := inv.Outstanding(ctx, node)
return err == nil && strings.TrimSpace(digest) != ""
// Read from the node-engine's have rather than asked of the machine: a machine that is away has still been given it, and
// this question is about whether the mesh did its part.
func wasGivenTheUserList(ctx context.Context, node, module string) bool {
bus, err := aBus()
if err != nil {
return false
}
defer bus.Close()
js, err := jetstream.New(bus.Conn())
if err != nil {
return false
}
have, had, err := haveOn(ctx, js, node, module)
return err == nil && had && have.Composed != ""
}
// notReadyOf is the readiness reasoning, named here so a test can reach it without the command's
cmd/mesh-controller/status.go
+1 -1
View File
@@ -489,7 +489,7 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
out.heals = countHeals(heals)
}
out.reported, err = inv.LastReports(ctx)
out.reported, err = lastReports(ctx, inv)
if err != nil {
return answers{}, err
}
cmd/mesh-controller/status_summary.go
-5
View File
@@ -207,11 +207,6 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e
report.Node, herr)
}
}
if err == nil && l.open != nil && startedWell(report) {
// Off the report's path: replacing a given value sends the machine, and a report waits for
// nothing it caused (novox/hq ADR 0228).
go replaceGiven(context.WithoutCancel(ctx), l.open, report)
}
// And, once the move of the secrets into the vault has started, what is left of this machine's (design 57's V3,
// issue 676): a machine asleep when it started is moved when it is next heard from. Off the report's path.
if err == nil && l.open != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" {
cmd/mesh-controller/watchdogs.go
+1 -1
View File
@@ -380,7 +380,7 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory
if err != nil {
return nil, fmt.Errorf("the machines cannot be read: %w", err)
}
reports, err := inv.LastReports(ctx)
reports, err := lastReports(ctx, inv)
if err != nil {
return nil, fmt.Errorf("what each machine last reported cannot be read: %w", err)
}