novox/mesh-controller
18
0
Fork 0
Code Issues Pull Requests 3 Releases Wiki Activity

Delete the push and the declaration: the send stub and node holds, link declare and its order, the sent readers, and the sweep judged by compositions (release 6, issue 1945 B8)

Browse Source
This commit is contained in:
jschoubben
2026-10-24 17:12:35 +02:00
parent 66650e315c
commit f56458da32
57 changed files with 228 additions and 2587 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-controller/acts.go
-12
View File
@@ -58,13 +58,6 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio
// Every line of the answer comes from the one resolution after the write (novox/hq issue 237):
// what the reads before it would say about a machine left out is not the answer's.
ctx = leftOutSaidBy(ctx, io.Discard)
// Held while it is recorded, so it cannot land between a converge's preview and its flip and
// be taken without ever having been previewed (novox/hq ADR 0100).
ctx, release, err := holdNodes(ctx, open, []string{node})
if err != nil {
return "", err
}
defer release()
// **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue
// 325): a build in flight keeps the assignment pending, a module known and not built is said with how to
// build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act
@@ -244,11 +237,6 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string)
}
// One resolution after the write, as assign's answer (novox/hq issue 237).
ctx = leftOutSaidBy(ctx, io.Discard)
ctx, release, err := holdNodes(ctx, open, []string{node})
if err != nil {
return "", err
}
defer release()
shelf, err := open.inventory.Catalogue(ctx)
if err != nil {
return "", err
cmd/mesh-controller/adoption.go
-16
View File
@@ -608,16 +608,6 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s
if filter == "" {
filter = DefaultFilter
}
if yes {
// Held from the checks to the send, so no push composed before the flip is sent after it
// and returns the node to adopted.
held, release, err := holdNodes(ctx, open, []string{node})
if err != nil {
return "", err
}
defer release()
ctx = held
}
record, err := inv.NodeByName(ctx, node)
if err != nil {
return "", err
@@ -1031,12 +1021,6 @@ func adopt(ctx context.Context, open *stores, node string) (string, error) {
if record.Adopted {
return "", fmt.Errorf("%s is adopted already", node)
}
held, release, err := holdNodes(ctx, open, []string{node})
if err != nil {
return "", err
}
defer release()
ctx = held
if err := inv.SetAdopted(ctx, node, true); err != nil {
return "", err
}
cmd/mesh-controller/asker_test.go
-29
View File
@@ -306,35 +306,6 @@ func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) {
}
}
func TestEachAnswerCallsExactlyItsVerb(t *testing.T) {
plan := "plan-1791454185265004861"
waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent,
Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.",
Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)}
module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning,
Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.",
Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart",
Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove,
Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}}
for _, tc := range []struct {
c conditions.Condition
label string
want string
}{
{waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`},
{waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`},
{module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`},
} {
r := newAskerRig(t)
r.open = []conditions.Condition{tc.c}
_ = r.a.reconcile(context.Background())
answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label))
if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) {
t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want)
}
}
}
func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) {
r := newAskerRig(t)
r.open = []conditions.Condition{unitsCondition()}
cmd/mesh-controller/bindings.go
-15
View File
@@ -54,21 +54,6 @@ func otherMachines(machine string, nodes ...string) []string {
return out
}
// reportKept says every move a machine's resolution refused, on the push composing it, and raises its
// condition at once where this process keeps the conditions: a push is when a person is looking.
func reportKept(ctx context.Context, plan catalogue.Resolution) {
for _, k := range plan.Kept {
fmt.Printf("%s: the mesh %s\n", plan.Node, k)
if conditionsFrom != nil {
o := keptObservation(k)
o.Source = probeBindingsID
if _, err := conditionsFrom.Observe(ctx, o); err != nil {
fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err)
}
}
}
}
// probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last
// sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin
// moves it (said, so the data goes first), and never anything else.
cmd/mesh-controller/bindings_test.go
-15
View File
@@ -76,18 +76,3 @@ func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) {
}
}
// A push says the move it refused, and raises its condition at once.
func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) {
k, _ := withConditionsInMemory(t)
reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop",
Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"},
Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}})
open, err := k.Open(t.Context())
if err != nil {
t.Fatal(err)
}
if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent ||
open[0].Source != probeBindingsID {
t.Fatalf("raised %+v", open)
}
}
cmd/mesh-controller/compose_input.go
-3
View File
@@ -250,9 +250,6 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento
}
// Every node-engine of release 6 reads a module per resource and a build per module.
out.ReadsModules = true
if out.SentBuilds, out.SentKnown, err = inv.SentBuilds(ctx, n.Name); err != nil {
return out, err
}
stated, has, err := inv.HealthOf(ctx, n.Name)
if err != nil {
return out, err
cmd/mesh-controller/compose_sdk.go
-6
View File
@@ -39,12 +39,6 @@ func withInputCache(ctx context.Context) context.Context {
return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}})
}
// freshInput is ctx for one composition whose Input is read afresh and kept only until the composition makes something: a
// send that is composed after another one made and recorded things (a push's machines, one after the other).
func freshInput(ctx context.Context) context.Context {
return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}})
}
// keptSignature is what the Input read depends on in ctx: which recorded modules each machine keeps at the build it runs
// (recorded_kept.go), so an Input read for a person's push is never taken for one read for a gated send.
func keptSignature(ctx context.Context) string {
cmd/mesh-controller/container_groups_test.go
-25
View File
@@ -1,9 +1,6 @@
package main
import (
"io"
"os"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
@@ -18,22 +15,6 @@ const n8nWithAGroup = `{"module":"n8n","version":"1",
"image":"n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0",
"groups":["${config:media-gid}"]}]}`
// printedBy runs reportUnhostable and returns what it printed.
func printedBy(t *testing.T, node string, plan catalogue.Resolution) string {
t.Helper()
stdout := os.Stdout
r, w, err := os.Pipe()
if err != nil {
t.Fatal(err)
}
os.Stdout = w
reportUnhostable(node, plan)
w.Close()
os.Stdout = stdout
out, _ := io.ReadAll(r)
return string(out)
}
// 1638/AC1: a node that reports the capability gets the module; one that does not has it held, the capability named.
func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing.T) {
m, err := catalogue.ParseManifest([]byte(n8nWithAGroup))
@@ -48,9 +29,6 @@ func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing
if err != nil || len(got.Unhostable) != 0 || len(got.Modules) != 1 {
t.Fatalf("a node that reads groups: %v, held %+v", err, got.Unhostable)
}
if out := printedBy(t, "ace", got); out != "" {
t.Errorf("a node that reads groups was told: %q", out)
}
got, err = catalogue.Resolve(shelf, []string{"n8n"}, old, catalogue.World{})
if err != nil {
@@ -59,7 +37,4 @@ func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing
if len(got.Modules) != 0 || len(got.Unhostable) != 1 {
t.Fatalf("n8n was sent to a node that does not report container-groups: hosted %d, held %+v", len(got.Modules), got.Unhostable)
}
if out := printedBy(t, "g14", got); !strings.Contains(out, "n8n") || !strings.Contains(out, "container-groups") {
t.Errorf("the hold does not name the module and the capability: %q", out)
}
}
cmd/mesh-controller/declaration_files_integration_test.go
+7 -31
View File
@@ -15,7 +15,6 @@ import (
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/link"
"github.com/novox/mesh-controller/internal/testbus"
)
@@ -34,14 +33,6 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T)
t.Fatal(err)
}
defer conn.Close()
nc, err := conn.JetStream()
if err != nil {
t.Fatal(err)
}
if _, err := nc.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"},
MaxMsgsPerSubject: 1}); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
api, err := jetstream.New(conn)
@@ -61,19 +52,8 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T)
resources = append(resources, aFileResource("big-"+string(rune('a'+i)), contentOf("big"+string(rune('a'+i)), 1200<<10)))
}
resources = append(resources, aFileResource("small", "tiny\n"))
inline := sendable{Resources: resources}
byDigest := sendable{fileReferences: true, Resources: resources}
bus := link.OverNATS{Conn: conn, JS: nc}
body, err := inline.Body()
if err != nil {
t.Fatal(err)
}
if err := link.Declare(ctx, bus, sigOf{}, "novox", body, 5*time.Second); err == nil ||
!strings.Contains(err.Error(), "1048576") || strings.Contains(err.Error(), "maximum payload exceeded") {
t.Fatalf("the inline declaration of more than 3 MiB: %v", err)
}
bucket, err := openBucket(ctx, conn, "novox")
if err != nil {
t.Fatal(err)
@@ -92,18 +72,14 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T)
if err != nil {
t.Fatal(err)
}
if err := link.Declare(ctx, bus, sigOf{}, "novox", refBody, 5*time.Second); err != nil {
t.Fatalf("the declaration by reference: %v", err)
// What the body names is exactly the three objects.
namedSizes, err := namedObjects(refBody)
if err != nil || len(namedSizes) != 3 {
t.Fatalf("what the body names: %v %v", namedSizes, err)
}
// What the stream holds for the node is the reference form, and it names exactly the three objects.
latest, err := latestDeclarations(ctx, conn, []string{"novox", "ace"})
if err != nil || len(latest) != 1 {
t.Fatalf("the latest declarations: %v %v", len(latest), err)
}
named, err := namedByLatest("novox", latest["novox"])
if err != nil || len(named) != 3 {
t.Fatalf("what the latest declaration names: %v %v", named, err)
named := map[string]bool{}
for name := range namedSizes {
named[name] = true
}
novoxBucket, err := broker.DeclarationFilesBucketOf("novox")
if err != nil {
cmd/mesh-controller/declaration_files_test.go
+35 -13
View File
@@ -12,6 +12,7 @@ import (
"time"
sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
@@ -55,7 +56,7 @@ func wireSizeOf(t *testing.T, s sendable) int {
t.Fatal(err)
}
// The signature the controller's key makes is 64 bytes.
msg, err := link.Envelope(body, make([]byte, 64))
msg, err := json.Marshal(link.Signed{Declaration: body, Signature: make([]byte, 64)})
if err != nil {
t.Fatal(err)
}
@@ -306,22 +307,43 @@ func TestAnObjectIsRemovedADayAfterNoLatestDeclarationNamesIt(t *testing.T) { //
}
}
// What a node's latest declaration names is read from the signed envelope as stored on the bus; an unreadable one is an error and
// never "names none", which would mark an object a node needs.
func TestTheObjectsALatestDeclarationNamesAreReadFromTheBus(t *testing.T) {
s := sendable{fileReferences: true, Resources: []map[string]any{aFileResource("a", contentOf("a", 70<<10))}}
body, _ := s.Body()
msg, _ := link.Envelope(body, []byte("sig"))
named, err := namedByLatest("novox", msg)
name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(contentOf("a", 70<<10))))
// What a node's compositions name is read from the values of its bucket `composed-<node>`: the files each carries by digest
// (ADR 0539 §4). An unreadable one is an error and never "names none", which would mark an object a node needs.
func TestTheObjectsACompositionNamesAreReadFromItsValue(t *testing.T) {
content := contentOf("a", 70<<10)
digest := sdkdecl.DigestOf([]byte(content))
ref, _ := json.Marshal(map[string]any{"id": "m.f", "type": "file", "module": "m", "path": "/etc/m", "digest": digest,
"size": len(content)})
small := json.RawMessage(`{"id":"m.g","type":"file","module":"m","path":"/etc/g","content":"x"}`)
fp := sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64)
value, err := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "novox", Module: "m", FP: fp,
Resources: []json.RawMessage{ref, small}})
if err != nil {
t.Fatal(err)
}
named, err := namedByCompositions("novox", [][]byte{value})
name, _ := sdkdecl.ObjectName(digest)
if err != nil || len(named) != 1 || !named[name] {
t.Fatalf("%v %v", named, err)
}
if none, err := namedByLatest("ace", []byte(`{"declaration":"e30=","signature":""}`)); err != nil || len(none) != 0 {
t.Fatalf("a declaration naming none: %v %v", none, err)
none, _ := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "ace", Module: "m", FP: fp, Resources: []json.RawMessage{small}})
if got, err := namedByCompositions("ace", [][]byte{none}); err != nil || len(got) != 0 {
t.Fatalf("a composition naming none: %v %v", got, err)
}
if _, err := namedByLatest("novox", []byte("not json")); err == nil {
t.Fatal("an unreadable latest declaration was read as naming none")
if got, err := namedByCompositions("ace", nil); err != nil || len(got) != 0 {
t.Fatalf("a node with no composition: %v %v", got, err)
}
if _, err := namedByCompositions("novox", [][]byte{[]byte("not json")}); err == nil {
t.Fatal("an unreadable composition was read as naming none")
}
// A node's objects are those of every composition it keeps: the five newest fingerprints of a module name their own.
other := sdkdecl.DigestOf([]byte(contentOf("b", 80<<10)))
ref2, _ := json.Marshal(map[string]any{"id": "m.f", "type": "file", "module": "m", "path": "/etc/m", "digest": other, "size": 80 << 10})
older, _ := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "novox", Module: "m", FP: sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64),
Resources: []json.RawMessage{ref2}})
both, err := namedByCompositions("novox", [][]byte{value, older})
if err != nil || len(both) != 2 {
t.Fatalf("two fingerprints of one module name %v (%v), want both objects", both, err)
}
}
cmd/mesh-controller/declaration_room.go
-55
View File
@@ -2,65 +2,10 @@ package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"strings"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/link"
)
// **A declaration is one bus message, and the bus's largest message is a cliff** (novox/hq issue 1689). On 2026-10-21
// the control node's declaration passed it by 2 KB and nothing reached that node until two assignments were undone.
// So the size is measured at every send: a warning condition from 75% of the limit (ADR 0486 §8, issue 1698: it was 90% while every file travelled inline), urgent over it, cleared when
// the declaration is back under 75%. The limit is the connection's own, not a constant.
// kindDeclarationNearLimit is a declaration at or near the bus's largest message.
const kindDeclarationNearLimit = "declaration-near-limit"
// declarationRoomSource is the condition's source: the sends.
const declarationRoomSource = "sends"
func declarationRoomObservation(node string, size, limit int64) conditions.Observation {
o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindDeclarationNearLimit,
Token: kindDeclarationNearLimit, Machine: node, Severity: conditions.Warning, Source: declarationRoomSource}
if size > limit {
o.Severity = conditions.Urgent
}
o.Summary = fmt.Sprintf("%s's declaration is %d bytes on the wire, %d%% of the bus's largest message of %d bytes",
node, size, size*100/limit, limit)
if size > limit {
o.Summary += ": over it, so it cannot be sent"
}
return o
}
func init() {
link.DeclarationMeasured = func(ctx context.Context, node string, size, limit int64) {
settleDeclarationRoom(ctx, conditionsFrom, node, size, limit)
}
}
// settleDeclarationRoom raises or clears the condition of a declaration's size. A nil keeper (a tool, not the
// serving controller) says nothing.
func settleDeclarationRoom(ctx context.Context, keeper *conditions.Keeper, node string, size, limit int64) {
if keeper == nil || limit <= 0 {
return
}
obs := declarationRoomObservation(node, size, limit)
if float64(size) < link.NearTheLimit*float64(limit) {
if _, err := keeper.Clear(ctx, obs.Key(), fmt.Sprintf("%s's declaration is %d bytes, under the limit's margin", node, size)); err != nil {
fmt.Printf("the condition %s could not be cleared: %v\n", obs.Key(), err)
}
return
}
fmt.Printf("NEAR THE LIMIT: %s\n", obs.Summary)
if _, err := keeper.Observe(ctx, obs); err != nil {
fmt.Printf("the condition %s could not be raised: %v\n", obs.Key(), err)
}
}
// compactKept writes the vault's export compact. The composition (the SDK's keptFile) writes it indented, and its 334
// sealed secrets were 164,763 bytes of it against 138,474 compact: a fifth of the declaration's room, in whitespace,
// base64-encoded again on the wire. Every reader parses JSON, so the spacing is nobody's. A resource that is not
cmd/mesh-controller/declaration_room_test.go
-71
View File
@@ -1,71 +0,0 @@
package main
import (
"strings"
"testing"
"time"
"github.com/novox/mesh-controller/internal/conditions"
)
// The vault's export is sent compact (issue 1689): same document, no indentation, one rewrite.
func TestTheVaultsExportIsSentCompactAndNothingElseIsTouched(t *testing.T) {
indented := "{\n \"export\": 1,\n \"kept\": [\n {\n \"name\": \"a\"\n }\n ]\n}\n"
other := "{\n \"x\": 1\n}\n"
in := []map[string]any{
{"id": "mesh-vault.kept", "type": "file", "path": "/srv/vault/export.json", "mode": "0600", "content": indented},
{"id": "mesh-vault.config", "type": "file", "path": "/srv/vault/config.json", "content": other},
{"id": "other.kept", "type": "file", "path": "/srv/other/notes.json", "content": other},
}
out := compactKept(in)
want := "{\"export\":1,\"kept\":[{\"name\":\"a\"}]}\n"
if out[0]["content"] != want || out[0]["mode"] != "0600" || out[0]["path"] != "/srv/vault/export.json" {
t.Fatalf("the export is %q (%v), expected %q with its mode and path kept", out[0]["content"], out[0], want)
}
if out[1]["content"] != other || out[2]["content"] != other {
t.Fatalf("a file that is not the export was rewritten: %v %v", out[1], out[2])
}
if in[0]["content"] != indented {
t.Fatalf("the composed resource was changed in place")
}
if again := compactKept(out); again[0]["content"] != want {
t.Fatalf("compacting twice changed it: %q", again[0]["content"])
}
}
// From 75% of the bus's largest message (ADR 0486 §8, issue 1698) the declaration is a warning; over it, urgent; back under 75%, cleared.
func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *testing.T) {
store := conditions.NewInMemory()
k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store,
Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }})
t.Cleanup(func() { k.Close(t.Context()) })
const limit = 1000
key := declarationRoomObservation("novox", 0, limit).Key()
read := func() (conditions.Condition, bool) {
t.Helper()
c, found, err := conditions.ReadOne(t.Context(), store, key)
if err != nil {
t.Fatal(err)
}
return c, found
}
settleDeclarationRoom(t.Context(), k, "novox", 749, limit)
if _, found := read(); found {
t.Fatal("a declaration at 74.9% raised a condition")
}
settleDeclarationRoom(t.Context(), k, "novox", 750, limit)
c, found := read()
if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "750 bytes") ||
!strings.Contains(c.Summary, "1000") || c.Subject.Machine != "novox" {
t.Fatalf("at 75%% the condition is %v %+v", found, c)
}
settleDeclarationRoom(t.Context(), k, "novox", 1001, limit)
if c, _ := read(); c.Severity != conditions.Urgent {
t.Fatalf("over the limit the condition is %q, expected urgent", c.Severity)
}
settleDeclarationRoom(t.Context(), k, "novox", 700, limit)
if _, found := read(); found {
t.Fatal("the condition stayed after the declaration fell under 75%")
}
}
cmd/mesh-controller/declaration_sweep.go
+56 -38
View File
@@ -9,23 +9,22 @@ import (
"strings"
"time"
sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// **An object is removed a day after no node's latest declaration names it** (novox/hq ADR 0486 §3, issue 1696), so a node that is
// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest
// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else
// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
// **An object is removed a day after no composition of the node names it** (novox/hq ADR 0486 §3, ADR 0539 §4, issue 1696), so a
// node that is asleep and applies its wants later still finds every object they name. What a node's compositions name is read
// from its bucket `composed-<node>`, where the controller keeps each module's five newest fingerprints and the node's own: nothing
// else is kept of them. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed);
// found named again, that is erased; found unnamed for a day, it is removed. Each node has a bucket of its own, judged against that
// node's latest declaration alone. Objects are immutable and regenerable from the
// controller's state, so a removal loses nothing.
// node's compositions alone. Objects are immutable and regenerable from the controller's state, so a removal loses nothing.
// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration.
// declarationUnnamed is the key of an object's metadata holding when it was first found named by no composition of the node.
const declarationUnnamed = "unnamed-since"
// sweepEvery is how often the bucket is judged: the removal is exact to within this.
@@ -92,41 +91,62 @@ func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[st
return r, nil
}
// namedByLatest is the name of every object this latest declaration of a node names. A body that cannot be read is an error, never
// "names none": an object the node still needs would be marked for removal.
func namedByLatest(node string, signed []byte) (map[string]bool, error) {
var s link.Signed
if err := json.Unmarshal(signed, &s); err != nil {
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
}
names, err := namedObjects(s.Declaration)
if err != nil {
return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err)
}
named := make(map[string]bool, len(names))
for name := range names {
named[name] = true
// namedByCompositions is the name of every object the compositions of a node name: the files over 64 KiB each carries by
// digest (ADR 0539 §4). A composition that cannot be read is an error, never "names none": an object the node still needs
// would be marked for removal.
func namedByCompositions(node string, values [][]byte) (map[string]bool, error) {
named := map[string]bool{}
for _, value := range values {
c, err := sdkdelivery.DecodeComposed(value)
if err != nil {
return nil, fmt.Errorf("a composition of %s is not readable: %w", node, err)
}
body, err := json.Marshal(map[string]any{"resources": c.Resources})
if err != nil {
return nil, err
}
names, err := namedObjects(body)
if err != nil {
return nil, fmt.Errorf("a composition of %s is not readable: %w", node, err)
}
for name := range names {
named[name] = true
}
}
return named, nil
}
// latestDeclarations are the last signed declarations of the nodes, by node, as the stream NODES holds them. A node with none has
// no entry.
func latestDeclarations(ctx context.Context, conn *nats.Conn, nodes []string) (map[string][]byte, error) {
js, err := conn.JetStream(nats.Context(ctx))
// keptCompositionsOf are the values of every composition in a node's bucket `composed-<node>`: what the controller keeps there
// are each module's five newest fingerprints and the node's own (ADR 0539 §4). A node with no bucket has none.
func keptCompositionsOf(ctx context.Context, js jetstream.JetStream, node string) ([][]byte, error) {
bucket, err := sdkdelivery.ComposedBucketFor(node)
if err != nil {
return nil, err
}
out := map[string][]byte{}
for _, node := range nodes {
msg, err := js.GetLastMsg("NODES", link.DeclareSubject(node), nats.Context(ctx))
if errors.Is(err, nats.ErrMsgNotFound) {
kv, err := js.KeyValue(ctx, bucket)
if errors.Is(err, jetstream.ErrBucketNotFound) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("the compositions of %s: %w", node, err)
}
keys, err := kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("the compositions of %s: %w", node, err)
}
var out [][]byte
for _, key := range keys {
e, err := kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
continue
}
if err != nil {
return nil, fmt.Errorf("the latest declaration of %s: %w", node, err)
return nil, fmt.Errorf("the composition %s: %w", key, err)
}
out[node] = msg.Data
out = append(out, e.Value())
}
return out, nil
}
@@ -259,15 +279,13 @@ func sweepNode(ctx context.Context, conn *nats.Conn, node string) (filesSweep, e
return filesSweep{}, fmt.Errorf("the bucket cannot be read: %w", err)
}
defer lockFilesOf(node)()
latest, err := latestDeclarations(working, conn, []string{node})
values, err := keptCompositionsOf(working, js, node)
if err != nil {
return filesSweep{}, err
}
named := map[string]bool{}
if body, ok := latest[node]; ok {
if named, err = namedByLatest(node, body); err != nil {
return filesSweep{}, err
}
named, err := namedByCompositions(node, values)
if err != nil {
return filesSweep{}, err
}
return sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now())
}
cmd/mesh-controller/dependents_test.go
-42
View File
@@ -1,42 +0,0 @@
package main
import (
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
)
// A merge that rebuilds a base rebuilds what stands on it, through every layer, and nothing else
// (novox/hq issue 186): the runtime image moving means every module built on it moves too, and a
// module built on one of those moves as well.
func TestAMergeOfABaseTakesWhatStandsOnItAlong(t *testing.T) {
entry := func(name string) inventory.Entry {
return inventory.Entry{Manifest: catalogue.Manifest{Module: name}}
}
entries := []inventory.Entry{entry("mesh-tools"), entry("shop"), entry("shop-plugin"), entry("postgres"), entry("unrelated")}
against := map[string][]string{
"shop": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
"shop-plugin": {catalogue.ArtifactStoreScheme + "shop/runtime@sha256:b"},
"postgres": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"},
"unrelated": {catalogue.ArtifactStoreScheme + "alpine/base@sha256:c"},
}
got := dependentsOf([]inventory.Entry{entry("mesh-tools")}, entries, against)
var names []string
for _, e := range got {
names = append(names, e.Manifest.Module)
}
want := map[string]bool{"shop": true, "shop-plugin": true, "postgres": true}
if len(names) != len(want) {
t.Fatalf("rebuilt %v; wanted exactly the three that stand on the runtime, directly or through shop", names)
}
for _, n := range names {
if !want[n] {
t.Fatalf("%s was rebuilt and stands on nothing that moved (%v)", n, names)
}
}
// Nothing moved: nothing follows.
if more := dependentsOf(nil, entries, against); len(more) != 0 {
t.Fatalf("with nothing moved, %d module(s) were rebuilt", len(more))
}
}
cmd/mesh-controller/facts.go
-9
View File
@@ -8,7 +8,6 @@ import (
"flag"
"fmt"
"os"
"slices"
"sort"
"strings"
"sync"
@@ -18,7 +17,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/compose"
snapshot "git.novox.be/novox/mesh-sdk/go/facts"
"github.com/novox/mesh-controller/internal/artifacts"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
@@ -315,13 +313,6 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot
return snapshot.Facts{}, err
}
m.Assigned = assigned
sent, known, err := inv.SentBuilds(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
}
if known && slices.Contains(assigned, broker.RuntimeModule) {
m.NodeTools = sent[broker.RuntimeModule]
}
pins, err := inv.PinsFor(ctx, n.Name)
if err != nil {
return snapshot.Facts{}, err
cmd/mesh-controller/grant_follows_binding_integration_test.go
-5
View File
@@ -105,11 +105,6 @@ func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) {
if got := declared.Received["store"]["postgres-database"]; len(got) != 0 {
t.Fatalf("the anchor's store is still told about %+v", got)
}
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") ||
!strings.Contains(said, "cleanup delete") {
t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said)
}
// The store it is bound to: granted.
grants, _, unbound, err = grantsOf(ctx, open, "laptop")
cmd/mesh-controller/helpers_kept_test.go
+5
View File
@@ -43,3 +43,8 @@ func claimingTheSeat(module string) inventory.Entry {
return inventory.Entry{Manifest: catalogue.Manifest{Module: module,
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
}
func aContainer(name string) catalogue.Manifest {
return catalogue.Manifest{Module: name, Version: "1",
Resources: []map[string]any{{"id": name, "type": "container", "image": name}}}
}
cmd/mesh-controller/identity_bound_integration_test.go
-5
View File
@@ -71,11 +71,6 @@ func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) {
if strings.Join(granted, ",") != "files" {
t.Fatalf("the object store grants %v; files and only files fit", granted)
}
said := printed(t, func() error { reportLeftOut("anchor", declared); return nil })
if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) ||
!strings.Contains(said, "left out of anchor's grants") {
t.Fatalf("the push does not say whom it leaves out:\n%s", said)
}
// And `status` names it, and does not call the mesh well while it stands.
asked, err := theThreeQuestions(ctx, open)
cmd/mesh-controller/image_missing.go
+1 -5
View File
@@ -126,12 +126,8 @@ func withoutMissingImages(ctx context.Context, open *stores, node string, s send
if err != nil || address == "" {
return s
}
sent, known, err := inv.SentBuilds(ctx, node)
if err != nil || !known {
sent = nil
}
store := artifacts.Store{Address: address}
found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, sent)
found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, nil)
for _, m := range found {
fmt.Printf("%s: %s is left out of the declaration: its image %s is not in the artifact store\n", node, m.Module, m.Image)
}
cmd/mesh-controller/kept_helpers.go
-19
View File
@@ -13,7 +13,6 @@ import (
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
"regexp"
"strconv"
"strings"
"time"
)
@@ -232,21 +231,3 @@ func refusedWord(failed string) string {
// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which
// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words).
var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`)
// durationSetting reads a duration as a setting says it: "90s", "10m", or a number of seconds.
func durationSetting(v any) (time.Duration, bool) {
switch t := v.(type) {
case string:
if d, err := time.ParseDuration(strings.TrimSpace(t)); err == nil && d >= 0 {
return d, true
}
if n, err := strconv.ParseFloat(strings.TrimSpace(t), 64); err == nil && n >= 0 {
return time.Duration(n * float64(time.Second)), true
}
case float64:
if t >= 0 {
return time.Duration(t * float64(time.Second)), true
}
}
return 0, false
}
cmd/mesh-controller/merge_unsaved_test.go
-26
View File
@@ -1,26 +0,0 @@
package main
import (
"errors"
"testing"
"github.com/novox/mesh-controller/internal/link"
)
// **A merge whose plan could not be saved is held, never taken** (novox/hq ADR 0407 decision 3, issue 1092): the merge is
// carried until its plan is saved, and the bus acknowledges the announcement only when the controller answers without
// ErrTryAgain (link.Server.sourceMoved), so a failed save of any kind must answer it.
func TestAFailureToSaveAMergesPlanIsHeldAndNotTaken(t *testing.T) {
cause := errors.New("the plan for novox/mesh-catalog is not written: the plan was written by somebody else since it was read")
held := unsaved(cause)
if !errors.Is(held, link.ErrTryAgain) || !errors.Is(held, cause) {
t.Fatalf("a failed save answers %v", held)
}
if unsaved(nil) != nil {
t.Fatal("a save that worked answers an error")
}
// Not the same as notNow, which holds a merge only for a store that is away.
if errors.Is(notNow(cause), link.ErrTryAgain) {
t.Fatal("notNow holds every failure; the premise of this test is gone")
}
}
cmd/mesh-controller/pending.go
-7
View File
@@ -411,13 +411,6 @@ func editDistance(a, b string) int {
// refusal. Returns what it said, or nothing when the row no longer waited.
func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignment, by string) string {
inv := open.inventory
ctx, release, err := holdNodes(ctx, open, []string{p.Node})
if err != nil {
// Left waiting: the next tick tries again.
return fmt.Sprintf("the pending assignment of %s to %s waits: %s could not be held: %v", p.Module, p.Node,
p.Node, err)
}
defer release()
claimed, err := inv.ClaimPending(ctx, p.ID)
if err != nil {
return fmt.Sprintf("the pending assignment of %s to %s could not be taken for making: %v", p.Module, p.Node, err)
cmd/mesh-controller/person_wait_test.go
+79
View File
@@ -1,6 +1,10 @@
package main
import (
"strings"
"testing"
"github.com/novox/mesh-controller/internal/conditions"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
@@ -19,3 +23,78 @@ func userUnit(module, account string) inventory.ResourceHealth {
Target: module + "-daemon.service", Account: account, State: link.StateUnhealthy,
Reason: "failed in the account's own service manager (exit-code)"}
}
// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as
// before, so that a fault is never excused.
func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) {
notInGroup := relogin("lights", "operator")
notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why"
systemUnit := userUnit("lights", "operator")
systemUnit.Account = ""
otherAccount := userUnit("lights", "guest")
container := inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights",
State: link.StateUnhealthy, Reason: "down"}
otherModule := userUnit("sound", "operator")
starting := userUnit("lights", "operator")
starting.State = link.StateStarting
byTarget := relogin("lights", "operator")
byTarget.Account = "" // an account said by its target alone is still that account
for _, c := range []struct {
name string
rs []inventory.ResourceHealth
waits bool
}{
{"the account alone", []inventory.ResourceHealth{relogin("lights", "operator")}, true},
{"the account and its unit in that account's manager", []inventory.ResourceHealth{relogin("lights", "operator"),
userUnit("lights", "operator")}, true},
{"an account named by its target", []inventory.ResourceHealth{byTarget, userUnit("lights", "operator")}, true},
{"another module's unit is not this module's", []inventory.ResourceHealth{relogin("lights", "operator"), otherModule}, true},
{"a unit still starting is no fault", []inventory.ResourceHealth{relogin("lights", "operator"), starting}, true},
{"a unit and no account", []inventory.ResourceHealth{userUnit("lights", "operator")}, false},
{"an account not in its group", []inventory.ResourceHealth{notInGroup, userUnit("lights", "operator")}, false},
{"a unit whose manager is not said", []inventory.ResourceHealth{relogin("lights", "operator"), systemUnit}, false},
{"a unit in another account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), otherAccount}, false},
{"a container beside the wait", []inventory.ResourceHealth{relogin("lights", "operator"), container}, false},
} {
said, waits := personWait("lights", "laptop", c.rs)
if waits != c.waits {
t.Errorf("%s: waits %v; want %v", c.name, waits, c.waits)
continue
}
if waits && !strings.HasPrefix(said, "relogin needed on laptop: lights waits for a new login of operator") {
t.Errorf("%s: said %q", c.name, said)
}
}
}
// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers
// no button, since nothing but their own new login can do it.
func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) {
o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
relogin("openrazer", "operator"), userUnit("openrazer", "operator")})
plainExample(t, o, "openrazer waits for a new login on g14",
"Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+
"group it needs. You logged in before that, so openrazer cannot run until you log in again. Its "+
"update is in place and nothing was undone.")
if len(o.Actions) != 0 || o.Needs == "" {
t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions)
}
// **Not "your account" when it is not the operator's** (issue 318 review).
other := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{
relogin("openrazer", "guest"), userUnit("openrazer", "guest")})
plainExample(t, other, "openrazer waits for a new login on g14",
"Needs you: have the account it names log out of g14 completely and log in again, or restart g14. Openrazer "+
"put an account on g14 in a group it needs. That account logged in before that, so openrazer cannot run "+
"until it logs in again. Its update is in place and nothing was undone.")
if unknown := reloginObservation("openrazer", "g14", "…", "", []inventory.ResourceHealth{relogin("openrazer", "operator")}); strings.Contains(unknown.Explanation, "your account") {
t.Errorf("an operator not known is still told it is their account: %q", unknown.Explanation)
}
// The kind's own wording, for a condition raised without words, names the module whole, dots and all,
// and never claims the account is the operator's.
w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "razer.lights.g14", Machine: "g14"})
if w.Headline != "razer.lights waits for a new login on g14" || w.Needs == "" || len(w.Actions) != 0 ||
strings.Contains(w.Explanation, "your account") {
t.Errorf("the kind's wording: %+v", w)
}
}
cmd/mesh-controller/plain_words.go
-67
View File
@@ -202,21 +202,6 @@ var plainWordings = map[string]func(conditions.Observation) words{
"send after send, so %s is sent them inside the instructions instead, which may make them too big.", m, m),
Resolved: "Resolved: the big files for " + m + " reach the bus again"}
}),
kindDeclarationNearLimit: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
if o.Severity == conditions.Urgent {
return words{Headline: m + " cannot be sent its instructions",
Needs: "make the instructions for " + m + " smaller, for example by taking a module off it.",
Explanation: fmt.Sprintf("The instructions for %s have grown past the largest message the bus takes, so "+
"nothing new reaches %s until they are smaller.", m, m),
Resolved: "Resolved: the instructions for " + m + " fit the bus again"}
}
return words{Headline: "The instructions for " + m + " are nearly too big",
Needs: "do not add more to " + m + " until this is dealt with; the details give the size.",
Explanation: fmt.Sprintf("The instructions for %s are close to the largest message the bus takes. "+
"One more module or secret may stop anything new from reaching %s.", m, m),
Resolved: "Resolved: the instructions for " + m + " have room again"}
}),
kindBindingKept: worded(func(o conditions.Observation) words {
m := machineOr(o, "a machine")
return words{Headline: "A module's data source is held on " + m,
@@ -675,14 +660,6 @@ func humanDuration(d time.Duration) string {
return fmt.Sprintf("%d days", int(d.Hours()/24))
}
// repoName is a repository as a person says it: its name without its owner.
func repoName(repository string) string {
if i := strings.LastIndex(repository, "/"); i >= 0 {
return repository[i+1:]
}
return repository
}
// namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more".
func namesWords(names []string, most int) string {
switch {
@@ -696,50 +673,6 @@ func namesWords(names []string, most int) string {
return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most)
}
// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else
// their count and the repository.
func deliveryWhat(modules []string, repository string) string {
if len(modules) > 0 && len(modules) <= 2 {
return namesWords(modules, 2)
}
if len(modules) > 2 {
return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository))
}
return repoName(repository)
}
// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery".
func deliveryName(modules []string, repository string) string {
if len(modules) > 0 && len(modules) <= 2 {
return namesWords(modules, 2) + " delivery"
}
return repoName(repository) + " delivery"
}
// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it.
// Start and Stop are also asked of the operator (novox/hq ADR 0259); the condition's own words keep saying
// where they are given without a channel, and the ask's text drops that (askText).
func waitingNeeds(severity conditions.Severity) string {
if severity == conditions.Urgent {
return "start it, or stop it, " + FromMeshMCPServer
}
return ""
}
// waitingActions are the answers to a walk waiting past its urgent bound: start it, or stop it — the plan's
// own verbs, approved by the operator (novox/hq ADR 0259). None before the bound.
func waitingActions(plan string, severity conditions.Severity) []conditions.Action {
if severity != conditions.Urgent || plan == "" {
return nil
}
return []conditions.Action{
{Label: "Start", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
Arguments: map[string]string{"go": plan, "why": "", "cause": conditions.CauseOperatorAnswer}},
{Label: "Stop", Verb: "mesh-controller.plans", Level: conditions.LevelApprove,
Arguments: map[string]string{"stop": plan, "why": "", "cause": conditions.CauseOperatorAnswer}},
}
}
// moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its
// account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it.
// No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258).
cmd/mesh-controller/plan.go
-95
View File
@@ -14,7 +14,6 @@ import (
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// working out what one machine should be.
@@ -123,43 +122,6 @@ func sortedKeysOf(m map[string]string) []string {
return out
}
// reportLeftOut says which of a machine's modules its declaration leaves out and why (novox/hq ADR
// 0163, rule 6), one line each: the machine is told everything else, and is told it was left out.
func reportLeftOut(node string, declared sendable) {
for _, m := range declared.LeftOut {
fmt.Printf("%s: %s left out — what the machine holds for it is kept and its containers are "+
"untouched. %s\n", node, m, declared.leftOutWhy[m])
}
// And whom it serves nothing, because their identity overflows what the provision keeps (ADR
// 0225): the machine is sent everything else, and the consumer is named.
for _, o := range declared.withheld {
fmt.Printf("%s: %s\n", node, o)
}
// And whom it no longer serves because they are bound elsewhere (novox/hq issue 274).
for _, u := range declared.unbound {
fmt.Printf("%s: %s\n", node, u)
}
// And every contribution its holder could not render, which the machine is sent without (novox/hq
// ADR 0255).
for _, u := range declared.unplaced {
fmt.Printf("%s: %s\n", node, u)
}
// And each file only the provisioner's client fills, left out because the module is not handed here.
for _, u := range declared.handedOnly {
fmt.Printf("%s: %s\n", node, u)
}
}
// engineReadsBusy says whether a machine's node-engine reads a container's `busy` (novox/hq ADR 0335), by its
// own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly.
func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) {
stated, had, err := inv.HealthOf(ctx, node)
if err != nil {
return false, err
}
return had && stated.Contract >= link.BusyContract, nil
}
// provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user
// the process of the module running it runs as, empty where no such module is on the machine.
func provisionerAccount(plan catalogue.Resolution) string {
@@ -178,25 +140,6 @@ func provisionerAccount(plan catalogue.Resolution) string {
return ""
}
// busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares
// one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it.
func busyNotSent(modules []catalogue.Manifest, reads bool) []string {
if reads {
return nil
}
var out []string
for _, m := range modules {
for _, r := range m.Resources {
if _, has := r[catalogue.BusyField]; has {
out = append(out, fmt.Sprintf("%s.%v declares busy, and this node-engine does not read it yet (its "+
"statement's contract is below %d): not sent, so the container is replaced at once, busy or not "+
"(novox/hq ADR 0335)", m.Module, r["id"], link.BusyContract))
}
}
}
return out
}
// certificateFor is what the mesh certifies about one machine's internal name.
//
// It reaches across two contexts and reads neither one's store from the other: `inventory` knows
@@ -259,36 +202,6 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str
return issued, authority.Certificate, nil
}
// listensLines is what a person is told about what this module would open, and why — the same
// `why` every listens entry already carries for the firewall it also feeds (novox/hq ADR 0007), so
// deciding whether to assign a module can see what it would open before it opens it, not only
// after. A module with nothing to listen on prints nothing extra, same as today.
func listensLines(m catalogue.Manifest) []string {
var out []string
for _, l := range m.Listens {
if l.Why == "" {
out = append(out, fmt.Sprintf(" listens %d/%s from %s", l.Port, l.At(), l.From))
continue
}
out = append(out, fmt.Sprintf(" listens %d/%s from %s — %s", l.Port, l.At(), l.From, l.Why))
}
return out
}
// shownAs is the heading `plan --show` puts over a resource's content.
//
// **A file written into says so.** Its content is the mesh's part of a file that is otherwise the
// machine's — the keys of a JSON document (novox/hq ADR 0102), the region of a hosts file (issue
// 128). Shown under a bare path it reads as the whole file, and a person checking what a take
// replaces would see a hosts file of a dozen lines where the machine keeps thirty.
func shownAs(r map[string]any) string {
heading := fmt.Sprintf("%v %v", r["id"], r["path"])
if into, ok := r["into"].(string); ok && into != "" {
heading += fmt.Sprintf(" (written into, %s)", into)
}
return heading
}
// servedOnNode is what a module on a node tells a consumer of one of its provisions, with THAT
// node's ports on it: the port the node was given (novox/hq ADR 0100) over the one the mesh
// assigned over the manifest's own, settled with the node's settings layers.
@@ -458,14 +371,6 @@ func (l *unheldLogger) log(node string, unheld []catalogue.Unheld) {
node, len(lines), strings.Join(lines, "\n "))
}
// unheldChange is what an act on one node changed about its unmet seat dependencies, judged over
// its assignments before and after (novox/hq ADR 0207): each dependency now unmet that was not —
// which includes every one of a module just assigned — and each now met that was not. Nothing about
// any other node, and nothing that was already true before the act.
func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string {
return unheldChangeTo(shelf, node, before, after, unheldOver(shelf, node, after))
}
// unheldOver is the node's unmet seat dependencies judged over a set of module names alone.
func unheldOver(shelf map[string]catalogue.Manifest, node string, names []string) []catalogue.Unheld {
var set []catalogue.Manifest
cmd/mesh-controller/plan_test.go
+1 -49
View File
@@ -1,51 +1,3 @@
package main
import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// `plan` tells a person what a module would open and why, from the same `why` every listens
// entry already carries for the firewall (novox/hq ADR 0007) — so deciding whether to assign a
// module does not need reading its manifest first.
func TestListensLinesShowWhatAModuleWouldOpenAndWhy(t *testing.T) {
m := catalogue.Manifest{Module: "minio", Listens: []catalogue.Listening{
{Port: 9000, From: catalogue.FromMesh, Why: "the S3 endpoint"},
{Port: 9001, From: catalogue.FromMesh},
}}
got := listensLines(m)
if len(got) != 2 {
t.Fatalf("two listens entries, got %d: %v", len(got), got)
}
if !strings.Contains(got[0], "9000/tcp") || !strings.Contains(got[0], "the S3 endpoint") {
t.Errorf("the port and its why did not both appear: %q", got[0])
}
if strings.Contains(got[1], "—") {
t.Errorf("a listens entry with no why should not print a dash: %q", got[1])
}
if !strings.Contains(got[1], "9001/tcp") {
t.Errorf("the port still appears without a why: %q", got[1])
}
}
func TestListensLinesAreEmptyForAModuleWithNothingToListenOn(t *testing.T) {
if got := listensLines(catalogue.Manifest{Module: "board"}); len(got) != 0 {
t.Errorf("a module with no listens should print nothing, got %v", got)
}
}
// `plan --show` says when a file is written into rather than over (novox/hq issue 128), or the
// mesh's region of a hosts file reads as the whole file.
func TestAFileWrittenIntoIsShownAsSuch(t *testing.T) {
region := shownAs(map[string]any{
"id": "mesh-wireguard.fact-node-names", "path": "/etc/hosts", "into": "block"})
if region != "mesh-wireguard.fact-node-names /etc/hosts (written into, block)" {
t.Errorf("the region is shown as %q", region)
}
whole := shownAs(map[string]any{"id": "dnsmasq.fact-node-zones", "path": "/etc/mesh-resolver/nodes.conf"})
if strings.Contains(whole, "written into") {
t.Errorf("a whole file is shown as written into: %q", whole)
}
}
import ()
cmd/mesh-controller/probes.go
+1 -28
View File
@@ -608,15 +608,7 @@ func readDeliveries(ctx context.Context, inv *inventory.Inventory) (map[string]l
}
out := map[string]lastSend{}
for _, r := range reports {
builds, known, err := inv.SentBuilds(ctx, r.Node)
if err != nil {
return nil, err
}
d := lastSend{sent: r.Sent, current: r.Current}
if known {
d.carried = builds
}
out[r.Node] = d
out[r.Node] = lastSend{sent: r.Sent, current: r.Current}
}
return out, nil
}
@@ -1121,25 +1113,6 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation,
behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s (built from %s)",
n.HostVersion, strings.Join(hostVersions, " or "), short(current[hostModule].Commit)))
}
assigned, err := inv.Assigned(ctx, n.Name)
if err != nil {
return nil, err
}
tools := current[broker.RuntimeModule].Commit
if tools != "" && slices.Contains(assigned, broker.RuntimeModule) && !rolling[broker.RuntimeModule] {
sent, known, err := inv.SentBuilds(ctx, n.Name)
if err != nil {
return nil, err
}
switch {
case !known:
case !sameCommit(sent[broker.RuntimeModule], tools):
behind = append(behind, fmt.Sprintf("the node tools %s, the mesh holds %s",
short(orNotKnown(sent[broker.RuntimeModule])), short(tools)))
case !applied[n.Name]:
behind = append(behind, "the node tools it was last sent, not yet reported applied")
}
}
if len(behind) > 0 {
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "core-behind",
Machine: n.Name, Severity: conditions.Warning,
cmd/mesh-controller/push.go
-198
View File
@@ -7,33 +7,16 @@ import (
"errors"
"fmt"
"github.com/novox/mesh-controller/internal/conditions"
"io"
"log"
"os"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/compose"
"github.com/novox/mesh-controller/internal/broker"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
// reportUnhostable says which of a node's assigned modules the machine cannot run, once per push.
//
// A module whose declared capability has no detector on the machine is on the wrong machine. It is
// kept out of what the node is sent — the healthy modules beside it still converge — and named here
// so it is neither silently dropped nor a reason the whole node fails to push.
func reportUnhostable(node string, plan catalogue.Resolution) {
for _, u := range plan.Unhostable {
for _, c := range u.Missing {
fmt.Printf("%s not applied — %s\n", node, catalogue.WrongMachine(u.Module, c, node))
}
}
}
// sending it, and holding the link that carries it.
//
// Split out of main.go, which had reached 2,769 lines because appending was always the
@@ -326,95 +309,9 @@ func serve(ctx context.Context) (err error) {
// OverlayCIDRVar is the range the mesh allocates node addresses from.
const OverlayCIDRVar = "MESH_OVERLAY_CIDR"
// waitForApplied blocks until the node reports it applied exactly the declaration just sent, or the
// wait runs out. A report of failure or refusal for that same declaration ends the wait at once —
// there is nothing to wait for, and the reason is the node's own.
func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest string, wait time.Duration) error {
if digest == "" {
return nil // nothing was sent to this node
}
deadline := time.Now().Add(wait)
for {
doing, said, err := inv.DoingOf(ctx, node)
if err != nil {
return err
}
if said && doing.Declared == digest {
switch doing.Outcome {
case inventory.OutcomeApplied:
fmt.Printf("%s applied it\n", node)
return nil
case inventory.OutcomeFailed:
// Each failed resource by id with the machine's own words, so the answer says what to fix
// without a second look (novox/hq issue 629).
var each []string
for _, f := range doing.Failed {
each = append(each, f.ID+": "+f.Error)
}
return fmt.Errorf("%s applied what it was sent but %d resource(s) failed: %s", node, len(doing.Failed),
strings.Join(each, "; "))
case inventory.OutcomeRefused:
return fmt.Errorf("%s refused what it was sent: %s", node, doing.Refused)
}
}
if time.Now().After(deadline) {
return fmt.Errorf("%s did not report applying what it was sent within %s "+
"(it may still be converging; check `status`)", node, wait)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(500 * time.Millisecond):
}
}
}
// errGrants marks a send that stopped because what the machines' modules may do could not be issued
// (novox/hq issue 249). Nothing about the machines is wrong; asked again, it is likely to work, so an
// announcement that hits it is held and asked again.
var errGrants = errors.New("what the machines' modules may do on the bus could not be issued, and code " +
"sent before its grants is refused there")
// grantsRefused is a grant that failed for some machines and not others: their memberships could not
// be issued, by machine, and only those machines are held back.
type grantsRefused struct{ nodes map[string]error }
func (g *grantsRefused) Error() string {
names := make([]string, 0, len(g.nodes))
for n := range g.nodes {
names = append(names, n)
}
sort.Strings(names)
return fmt.Sprintf("the memberships of %s could not be issued; the first: %v",
strings.Join(names, ", "), g.nodes[names[0]])
}
// errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket.
var errFilesNotPut = errors.New("the files it names by digest are not in the bucket")
// brokerFirst is the machines to send in the order a grant needs (novox/hq issue 249): the machine
// holding the bus first — its declaration carries the bus's user list (composeBusUsers), and a
// module's new permissions are refused by the bus until that list says them. Among the machines it
// is moved to the front; not among them, it is added only when it is behind.
func brokerFirst(names []string, holder string, behind bool) []string {
if holder == "" {
return names
}
present := false
rest := make([]string, 0, len(names))
for _, n := range names {
if n == holder {
present = true
continue
}
rest = append(rest, n)
}
if !present && !behind {
return names
}
return append([]string{holder}, rest...)
}
// brokerBehind is the machine holding the bus — the one whose declaration carries the user list —
// and, when it is not among the machines named, whether the user list it would be sent now differs
// from the one it was last sent (novox/hq issue 249).
@@ -484,21 +381,6 @@ func userListBehind(now, sentDigest string) bool {
return now != "" && digestOf([]byte(now)) != sentDigest
}
// couldNotBeResolved is what a push ends with when some machines could not be worked out.
//
// **After the rest have been sent, never instead of sending them.** It is still an error, because
// the mesh is not in the state somebody asked for and a command that exits cleanly having skipped a
// machine is a command that lies. What it must not do is decide anything about the machines beside
// it, which is why it says how many were sent.
func couldNotBeResolved(refusals []string, sent int) error {
if len(refusals) == 0 {
return nil
}
return fmt.Errorf(
"%d node(s) could not be resolved and were not sent. %d other node(s) were:\n\n%s",
len(refusals), sent, strings.Join(refusals, "\n\n"))
}
// digestOf is what the mesh compares to answer "has this machine been sent what it should be".
//
// Over the same bytes that are sent, so the comparison is of the thing itself rather than of
@@ -508,65 +390,6 @@ func digestOf(body []byte) string {
return hex.EncodeToString(sum[:])
}
// wouldSend is the digest of what each machine should be right now.
//
// Machines that do not resolve are left out rather than reported as waiting: "this machine cannot
// be worked out" is a different problem with a different remedy, and `plan` is where it is said.
func wouldSend(ctx context.Context, open *stores,
nodes []inventory.Node) (map[string]string, error) {
return wouldSendFrom(ctx, open, nodes, nil)
}
// planned is one machine's plan as planFor answered it, for a caller that already asked.
type planned struct {
plan catalogue.Resolution
settings catalogue.SettingsBy
}
// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a
// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq
// to-be 45 Phase 0). A machine absent from plans is worked out here.
func wouldSendFrom(ctx context.Context, open *stores,
nodes []inventory.Node, plans map[string]planned) (map[string]string, error) {
if _, err := generatorsOf(ctx, open); err != nil {
return nil, err
}
var err error
out := map[string]string{}
for _, n := range nodes {
known, have := plans[n.Name]
plan, settings := known.plan, known.settings
if !have {
if plan, settings, err = resolveNode(ctx, open, n.Name); err != nil {
continue
}
}
_, _ = plan, settings
declared, _, err := composeQuestion(ctx, open, n.Name, compose.Reading, false)
if err != nil {
continue
}
// Composed with the number the machine was LAST sent, so this is byte for byte what it was
// sent when nothing else changed. A fresh number here would make every machine read as
// behind for ever (novox/hq 04-ISSUES/107).
if declared.Sequence, err = open.inventory.Sequence(ctx, n.ID); err != nil {
return nil, err
}
// And the epoch it was last sent under, for the same reason: a new holder of the lease is not a
// change of the machine (novox/hq to-be 45 §6).
if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil {
return nil, err
}
body, err := declared.Body()
if err != nil {
return nil, err
}
out[n.Name] = digestOf(body)
}
return out, nil
}
// raiseTheBus asserts the streams and consumers the mesh's own traffic needs.
//
// **Every start, and it says what it did.** The objects are the mesh's, created by nothing else —
@@ -695,27 +518,6 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok
// without a bus.
var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) }
// reportUnheldPushed says what a push's machines lack of the seats their modules depend on
// (novox/hq ADR 0207): every line for a machine the push named, since that is the machine somebody
// is looking at, and one line per machine otherwise — a list per machine across the mesh is the
// hundred lines that buried the one that mattered. Nothing for a machine that lacks nothing.
func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[string][]catalogue.Unheld) {
for _, node := range asked {
lines := unheld[node]
if len(lines) == 0 {
continue
}
if named {
fmt.Fprintf(w, "\n%s has %d unmet seat dependenc(ies) (novox/hq ADR 0207):\n", node, len(lines))
for _, u := range lines {
fmt.Fprintf(w, " %s\n", u)
}
continue
}
fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines))
}
}
// schemaReach is the highest migration this build carries for the inventory's store (novox/hq issue 352).
func schemaReach() (int, error) {
migrations, err := inventory.Migrations()
cmd/mesh-controller/registry_verbs.go
+8
View File
@@ -441,3 +441,11 @@ func imagesCommand(ctx context.Context, args []string) error {
}
return nil
}
// sentResource is what the store kept of one resource a machine was last sent, as far as `images` reads it: the container's
// image. Nothing writes the summary any more (ADR 0539 §5); a machine's images are read from its compositions.
type sentResource struct {
ID string `json:"id"`
Type string `json:"type"`
Image string `json:"image,omitempty"`
}
cmd/mesh-controller/registry_verbs_test.go
-11
View File
@@ -229,14 +229,3 @@ func TestImagesAreEveryContainerImageOfTheDeclarationAndOfWhatWasSent(t *testing
t.Errorf("with nothing sent yet: %+v %v", none, err)
}
}
func TestTheSentSummaryKeepsAContainersImage(t *testing.T) {
summary, err := summarize([]byte(`{"resources":[{"id":"web.server","type":"container","image":"x@sha256:aa"},
{"id":"web.state","type":"directory","path":"/srv"}]}`))
if err != nil {
t.Fatal(err)
}
if summary[0].Image != "x@sha256:aa" || summary[1].Image != "" {
t.Errorf("summarized as %+v", summary)
}
}
cmd/mesh-controller/send_stub.go
-23
View File
@@ -1,23 +0,0 @@
package main
import "context"
// What was once a send is no longer one (ADR 0539 §5, §8): the controller writes the compositions that changed instead
// (composeModules), and each machine follows its wants. The calls below stand where a send was asked until the part of the
// controller that asked it writes compositions in its place; the subtask that replaces the last caller deletes this file.
// sendTo sends nothing; see sendToEach.
func sendTo(ctx context.Context, open *stores, names []string) error {
_, err := sendToEach(ctx, open, names)
return err
}
// sendToEach answers the machines it was asked for, and sends them nothing.
func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) {
return names, nil
}
// holdNodes held machines from composing to sending; nothing is sent, so it holds nothing.
func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) {
return ctx, func() {}, nil
}
cmd/mesh-controller/sendable_integration_test.go
-9
View File
@@ -80,15 +80,6 @@ func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) {
t.Fatalf("taken is not a list on the wire, even empty: %s", body)
}
// The digest the mesh compares is the digest of what is sent: status and push agree.
would, err := wouldSend(ctx, open, mustNodes(t, open))
if err != nil {
t.Fatal(err)
}
if would["anchor"] != digestOf(body) {
t.Fatal("the digest the mesh compares is not of the declaration push sends")
}
// Taking the module moves its resources out of untaken.
if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil {
t.Fatal(err)
cmd/mesh-controller/signals.go
-23
View File
@@ -274,29 +274,6 @@ func watchHeartbeats(f *signalFacts) []conditions.Observation {
return out
}
func watchReports(f *signalFacts) []conditions.Observation {
var out []conditions.Observation
for _, m := range f.machines {
if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() {
continue
}
if m.silent(f.now) {
continue // silent: S1 says it, and a silent machine reports nothing
}
bound := max(reportAtLeast, 3*m.lastApply)
waited := f.now.Sub(m.sentAt)
if waited <= bound {
continue
}
out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name,
Kind: "sent-not-reported", Machine: m.name, Severity: conditions.Warning,
Summary: fmt.Sprintf("%s was sent a declaration at %s and has not reported applying it (bound %s)",
m.name, m.sentAt.UTC().Format("2006-01-02 15:04 MST"), ago(bound)),
Said: fmt.Sprintf("waiting %s for the report of the declaration sent", ago(waited))})
}
return out
}
func watchLoop(f *signalFacts) []conditions.Observation {
if f.loop.pending == 0 {
return nil
cmd/mesh-controller/signals_test.go
+1 -1
View File
@@ -402,7 +402,7 @@ func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) {
f.machines[0].lastHeard = now.Add(-2 * time.Hour)
f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false
f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)}
if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 {
if got := append(watchHeartbeats(f), watchTools(f)...); len(got) != 0 {
t.Fatalf("a sleeping machine raised %+v", got)
}
f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)}
cmd/mesh-controller/status.go
-21
View File
@@ -438,7 +438,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
// ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer
// the private network is built from and should say nothing else; a machine that does not
// resolve is already in refused, and is passed over here.
plans := map[string]planned{}
for _, n := range out.nodes {
plan, settings, err := resolveNode(ctx, open, n.Name)
if err != nil {
@@ -447,7 +446,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
}
return answers{}, err
}
plans[n.Name] = planned{plan, settings}
// And which of its modules a push leaves out (novox/hq issue 380), judged as the push judges it.
if left := leftOutOf(plan, settings, n.Adopted); len(left) > 0 {
if out.leftOut == nil {
@@ -491,25 +489,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) {
out.heals = countHeals(heals)
}
// And which machines are not running what the mesh would send them. The same question as a
// module being behind its source, one level down: that one says the catalogue is out of date,
// this one says a machine is — and only the second has anybody's change waiting in it.
//
// **One machine that cannot be resolved must not take the answer away from every other**
// (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub
// is the blocked machine has no hub — which used to come back here as a refusal, so `status`
// said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The
// reason is kept and reported as data; every question that does not depend on it is still
// answered.
would, err := wouldSendFrom(ctx, open, out.nodes, plans)
if err != nil {
out.network = err.Error()
would = map[string]string{}
}
out.waiting, err = inv.Waiting(ctx, would)
if err != nil {
return answers{}, err
}
out.reported, err = inv.LastReports(ctx)
if err != nil {
return answers{}, err
cmd/mesh-controller/status_summary.go
-7
View File
@@ -185,14 +185,7 @@ type nudgingListener struct {
}
func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) {
// A declaration refused as older than the one the machine holds is counted by its writer — the
// controller epoch it claimed (novox/hq to-be 45 §6, S13) — and so is what the machine's own count
// says it refused beyond the refusals heard.
now := time.Now()
if report.StaleRefusalOf() {
link.StaleRefusals.Refused(link.Refusal{Writer: link.WriterEpoch(report.Epoch), Epoch: report.Epoch,
Receiver: report.Node, At: now})
}
if report.Ordered() {
link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now)
}
cmd/mesh-controller/unheld_report_test.go
-83
View File
@@ -1,83 +0,0 @@
package main
import (
"bytes"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// An act says what it changed about the node it acted on, and nothing about the rest of the mesh
// (novox/hq ADR 0207): after the seat dependencies shipped, every `push <node>` and `assign` printed
// every node's unmet dependencies, a hundred lines around the one about the module just assigned.
func aContainer(name string) catalogue.Manifest {
return catalogue.Manifest{Module: name, Version: "1",
Resources: []map[string]any{{"id": name, "type": "container", "image": name}}}
}
func TestAnAssignmentThatMeetsADependencySaysSo(t *testing.T) {
shelf := map[string]catalogue.Manifest{
"web": aContainer("web"),
"docker": {Module: "docker", Claims: []catalogue.Claim{{Name: catalogue.ContainerRuntimeSeat}},
Resources: []map[string]any{{"id": "d", "type": "container", "image": "dind"}}},
}
lines := unheldChange(shelf, "laptop", []string{"web"}, []string{"web", "docker"})
if len(lines) != 1 || !strings.Contains(lines[0], "web on laptop now has "+catalogue.ContainerRuntimeSeat+" held") {
t.Errorf("meeting a dependency said %v", lines)
}
// Taking the dependent off says nothing: the dependency went with its module.
if lines := unheldChange(shelf, "laptop", []string{"web"}, nil); len(lines) != 0 {
t.Errorf("unassigning the dependent said %v", lines)
}
}
func TestAPushSaysANamedNodesDependenciesAndCountsTheRest(t *testing.T) {
unheld := map[string][]catalogue.Unheld{
"anchor": {{Node: "anchor", Module: "db", Seat: catalogue.ContainerRuntimeSeat}},
"laptop": {{Node: "laptop", Module: "web", Seat: catalogue.ContainerRuntimeSeat},
{Node: "laptop", Module: "sshd", Seat: catalogue.ServiceManagerSeat}},
}
var named bytes.Buffer
reportUnheldPushed(&named, true, []string{"laptop"}, unheld)
got := named.String()
if !strings.Contains(got, "laptop has 2 unmet") || !strings.Contains(got, "web on laptop") ||
!strings.Contains(got, "sshd on laptop") || strings.Contains(got, "anchor") {
t.Errorf("a named push said:\n%s", got)
}
var all bytes.Buffer
reportUnheldPushed(&all, false, []string{"anchor", "laptop", "quiet"}, unheld)
want := "anchor: 1 unmet seat dependenc(ies) — see `status`\nlaptop: 2 unmet seat dependenc(ies) — see `status`\n"
if all.String() != want {
t.Errorf("a push to every node said:\n%s\nwant\n%s", all.String(), want)
}
}
// The test gives logUnheld its own writer and memory, so a repeat in one process and the
// package's other tests see neither the last run's memory nor a swapped os.Stderr (novox/hq issue 486).
func TestOnlyTheServingControllerLogsEachChange(t *testing.T) {
var out bytes.Buffer
saved, savedOn := unheldLog, logUnheldChanges
unheldLog, logUnheldChanges = newUnheldLogger(&out), false
t.Cleanup(func() { unheldLog, logUnheldChanges = saved, savedOn })
read := func(f func()) string {
out.Reset()
f()
return out.String()
}
u := []catalogue.Unheld{{Node: "n1", Module: "web", Seat: catalogue.ContainerRuntimeSeat}}
if got := read(func() { logUnheld("n1", u) }); got != "" {
t.Errorf("a command logged:\n%s", got)
}
logUnheldChanges = true
if got := read(func() { logUnheld("n1", u) }); !strings.Contains(got, "web on n1") {
t.Errorf("the serving controller did not log a change:\n%s", got)
}
if got := read(func() { logUnheld("n1", u) }); got != "" {
t.Errorf("an unchanged report was logged again:\n%s", got)
}
if got := read(func() { logUnheld("n1", nil) }); !strings.Contains(got, "every seat its modules depend on is held") {
t.Errorf("a report that became none was not logged:\n%s", got)
}
}
cmd/mesh-controller/unseen.go
-282
View File
@@ -1,18 +1,8 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"reflect"
"sort"
"strings"
"github.com/novox/mesh-controller/internal/inventory"
)
// No change to a machine takes effect unseen (novox/hq ADR 0217, to-be 44).
@@ -71,275 +61,3 @@ func leaves(m map[string]any, prefix string) map[string]any {
}
return out
}
// ---- 2. what a push will change -----------------------------------------------------------------
// sentResource is what is kept of one resource a machine was sent: enough to say what changed and
// whether a container's data moved, and nothing that could carry a secret — a file's content is
// kept as a digest, never as text (the record lands in the store's own backups).
type sentResource struct {
ID string `json:"id"`
Type string `json:"type"`
// Digest is of the whole resource as it was sent.
Digest string `json:"digest"`
// Fields is each field's digest, so a change can be named by field.
Fields map[string]string `json:"fields"`
// Volumes is a container's mounts as sent — paths, which are not secrets, and what a moved
// data directory is read from.
Volumes []string `json:"volumes,omitempty"`
// Image is a container's image as sent — a reference, not a secret — so the machine can be told
// which images its last declaration named, and keep them (novox/hq ADR 0251, `images`). Absent in
// a summary kept before it was.
Image string `json:"image,omitempty"`
}
// summarize is what is kept of a declaration body: its resources, in the order sent.
func summarize(body []byte) ([]sentResource, error) {
var envelope struct {
Resources []map[string]any `json:"resources"`
}
if err := json.Unmarshal(body, &envelope); err != nil {
return nil, fmt.Errorf("a declaration that is not one: %w", err)
}
out := make([]sentResource, 0, len(envelope.Resources))
for _, r := range envelope.Resources {
s := sentResource{ID: fmt.Sprint(r["id"]), Type: fmt.Sprint(r["type"]), Digest: digestValue(r),
Fields: map[string]string{}}
for k, v := range r {
s.Fields[k] = digestValue(v)
}
if s.Type == "container" {
if image, ok := r["image"].(string); ok {
s.Image = image
}
if vols, ok := r["volumes"].([]any); ok {
for _, v := range vols {
s.Volumes = append(s.Volumes, fmt.Sprint(v))
}
}
}
out = append(out, s)
}
return out, nil
}
func digestValue(v any) string {
raw, _ := json.Marshal(v)
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:8])
}
// changedResource is one resource sent differently, with the fields that differ.
type changedResource struct {
ID, Type string
Fields []string
}
// sentDiff is what would be sent now against what was sent last, by resource id.
type sentDiff struct {
Added, Removed []string
Changed []changedResource
}
func (d sentDiff) empty() bool {
return len(d.Added) == 0 && len(d.Removed) == 0 && len(d.Changed) == 0
}
func diffSent(before, after []sentResource) sentDiff {
was := map[string]sentResource{}
for _, r := range before {
was[r.ID] = r
}
var d sentDiff
seen := map[string]bool{}
for _, r := range after {
seen[r.ID] = true
old, had := was[r.ID]
if !had {
d.Added = append(d.Added, r.ID)
continue
}
if old.Digest == r.Digest {
continue
}
c := changedResource{ID: r.ID, Type: r.Type}
for k, v := range r.Fields {
if old.Fields[k] != v {
c.Fields = append(c.Fields, k)
}
}
for k := range old.Fields {
if _, kept := r.Fields[k]; !kept {
c.Fields = append(c.Fields, k)
}
}
sort.Strings(c.Fields)
d.Changed = append(d.Changed, c)
}
for _, r := range before {
if !seen[r.ID] {
d.Removed = append(d.Removed, r.ID)
}
}
sort.Strings(d.Added)
sort.Strings(d.Removed)
sort.Slice(d.Changed, func(a, b int) bool { return d.Changed[a].ID < d.Changed[b].ID })
return d
}
// writeDiff says a diff the way a person reads one: what is added, removed and changed, and a
// changed container's fields — a container sent differently is a container recreated.
func writeDiff(w io.Writer, node string, d sentDiff, moves []dataMove) {
if d.empty() {
fmt.Fprintf(w, "%s: nothing would change — it was last sent exactly this\n", node)
return
}
fmt.Fprintf(w, "%s would change:\n", node)
for _, id := range d.Added {
fmt.Fprintf(w, " + %s\n", id)
}
for _, id := range d.Removed {
fmt.Fprintf(w, " - %s\n", id)
}
for _, c := range d.Changed {
what := "changed"
if c.Type == "container" {
what = "recreated"
}
fmt.Fprintf(w, " ~ %s %s: %s\n", c.ID, what, strings.Join(c.Fields, ", "))
}
writeMoves(w, node, moves)
}
// ---- 3. a running module's data moving ------------------------------------------------------------
// dataMove is a container keeping a mount at the same place inside it with a different directory
// on the machine behind it: its data moving — or, as on 2026-10-05, a module about to start on an
// empty directory because the placement that pointed at its data was lost.
type dataMove struct {
Container, Inside, From, To string
}
// Module is the module the container belongs to: resource ids are `<module>.<id>`.
func (m dataMove) Module() string {
module, _, _ := strings.Cut(m.Container, ".")
return module
}
// movesIn is every data move between what a machine was sent and what it would be sent. A new
// container, a mount added or dropped and a changed image are not moves.
func movesIn(before, after []sentResource) []dataMove {
was := map[string]sentResource{}
for _, r := range before {
if r.Type == "container" {
was[r.ID] = r
}
}
var out []dataMove
for _, r := range after {
old, had := was[r.ID]
if r.Type != "container" || !had {
continue
}
from := mountSources(old.Volumes)
for inside, to := range mountSources(r.Volumes) {
if prev, mounted := from[inside]; mounted && prev != to {
out = append(out, dataMove{Container: r.ID, Inside: inside, From: prev, To: to})
}
}
}
sort.Slice(out, func(a, b int) bool {
if out[a].Container != out[b].Container {
return out[a].Container < out[b].Container
}
return out[a].Inside < out[b].Inside
})
return out
}
// mountSources is a container's mounts by the place inside it: `source:inside[:options]`.
func mountSources(volumes []string) map[string]string {
out := map[string]string{}
for _, v := range volumes {
parts := strings.SplitN(v, ":", 3)
if len(parts) < 2 {
continue
}
out[parts[1]] = parts[0]
}
return out
}
func writeMoves(w io.Writer, node string, moves []dataMove) {
for _, m := range moves {
fmt.Fprintf(w, " ! %s: %s moves from %s to %s\n", m.Container, m.Inside, m.From, m.To)
}
}
// heldMoves is the moves in a push to one machine that the operator has not said to send (`--move
// <module>`); empty when there is nothing to hold, including a machine never sent anything before.
func heldMoves(ctx context.Context, inv *inventory.Inventory, node string, body []byte, allowed map[string]bool) ([]dataMove, error) {
prev, err := inv.SentSummary(ctx, node)
if err != nil || len(prev) == 0 {
return nil, err
}
var before []sentResource
if err := json.Unmarshal(prev, &before); err != nil {
// A record this version cannot read compares with nothing: holding every push for it would
// stop the mesh on a format, which is not what is at stake. Said, so it is not passed over.
fmt.Fprintf(os.Stderr, "%s: what it was last sent is kept in a form this version does not read, "+
"so whether its data would move is not known: %v\n", node, err)
// empty-on-error: said above; the send replaces the unreadable record with one this version reads
return nil, nil
}
after, err := summarize(body)
if err != nil {
return nil, err
}
var held []dataMove
for _, m := range movesIn(before, after) {
if !allowed[m.Module()] {
held = append(held, m)
}
}
return held, nil
}
// sayDataHeld tells the operator why a machine was not sent, and how to send it.
func sayDataHeld(w io.Writer, node string, moves []dataMove) {
modules := map[string]bool{}
for _, m := range moves {
modules[m.Module()] = true
}
var names []string
for m := range modules {
names = append(names, m)
}
sort.Strings(names)
fmt.Fprintf(w, "held %s: a running module's data would move (novox/hq ADR 0217)\n", node)
writeMoves(w, node, moves)
fmt.Fprintf(w, " if that is meant: push %s --move %s\n", node, strings.Join(names, ","))
}
// writePlanDiff says what sending body to a machine would change against what it was last sent.
func writePlanDiff(ctx context.Context, inv *inventory.Inventory, node string, body []byte) error {
after, err := summarize(body)
if err != nil {
return err
}
prev, err := inv.SentSummary(ctx, node)
if err != nil {
return err
}
if len(prev) == 0 {
fmt.Printf("%s: what it was last sent is not kept yet — the first push from this version keeps "+
"it; %d resource(s) would be sent\n", node, len(after))
return nil
}
var before []sentResource
if err := json.Unmarshal(prev, &before); err != nil {
return fmt.Errorf("%s: what it was last sent is kept in a form this version does not read: %w", node, err)
}
writeDiff(os.Stdout, node, diffSent(before, after), movesIn(before, after))
return nil
}
cmd/mesh-controller/unseen_test.go
-175
View File
@@ -1,175 +0,0 @@
package main
import (
"bytes"
"reflect"
"strings"
"testing"
)
// novox/hq ADR 0217: no change to a machine takes effect unseen.
// The layer of 2026-10-05: a media server's node layer, and the one that replaced it, which kept
// `places` and lost three of its four directories and every other key.
var before = map[string]any{
"puid": 1000.0, "pgid": 1000.0,
"expose": map[string]any{"32400": "anywhere"},
"places": map[string]any{
"config": map[string]any{"path": "/srv/media/config", "owner": "1000:1000"},
"data": map[string]any{"path": "/srv/media/data", "owner": "1000:1000"},
"transcode": map[string]any{"path": "/srv/media/temp", "owner": "1000:1000"},
},
}
func TestASettingThatKeepsAKeyAndLosesItsEntriesIsSaidToRemoveThem(t *testing.T) {
after := map[string]any{"places": map[string]any{
"previews": map[string]any{"path": "/srv/pool/previews", "owner": "1000:1000"},
}}
added, changed, removed := settingsChange(before, after)
if !reflect.DeepEqual(added, []string{"places.previews.owner", "places.previews.path"}) {
t.Errorf("added %v", added)
}
if len(changed) != 0 {
t.Errorf("changed %v", changed)
}
for _, lost := range []string{"expose.32400", "pgid", "places.config.path", "places.data.owner", "puid"} {
found := false
for _, r := range removed {
found = found || r == lost
}
if !found {
t.Errorf("removing %s was not said: %v", lost, removed)
}
}
}
func TestASettingThatOnlyAddsOrChangesRemovesNothing(t *testing.T) {
after := map[string]any{
"puid": 1001.0, "pgid": 1000.0,
"expose": map[string]any{"32400": "anywhere"},
"places": map[string]any{
"config": map[string]any{"path": "/srv/media/config", "owner": "1000:1000"},
"data": map[string]any{"path": "/srv/media/data", "owner": "1000:1000"},
"transcode": map[string]any{"path": "/srv/media/temp", "owner": "1000:1000"},
"previews": map[string]any{"path": "/srv/pool/previews", "owner": "1000:1000"},
},
}
_, changed, removed := settingsChange(before, after)
if len(removed) != 0 {
t.Errorf("an additive set was said to remove %v", removed)
}
if !reflect.DeepEqual(changed, []string{"puid"}) {
t.Errorf("changed %v", changed)
}
}
// What was sent, as a push would send it: a media server's container and a file with a secret.
func declaration(configFrom string, extra ...string) []byte {
vols := `"` + configFrom + `:/config", "/srv/media/data:/data"`
for _, e := range extra {
vols += `, "` + e + `"`
}
return []byte(`{"declaration":1,"sequence":7,"resources":[
{"id":"plex.server","type":"container","image":"plex@sha256:aa","volumes":[` + vols + `]},
{"id":"plex.env","type":"file","path":"/srv/media/env","content":"TOKEN=the-secret-itself"}]}`)
}
func summarized(t *testing.T, body []byte) []sentResource {
t.Helper()
s, err := summarize(body)
if err != nil {
t.Fatal(err)
}
return s
}
func TestWhatIsKeptOfASendHoldsNoFileContent(t *testing.T) {
s := summarized(t, declaration("/srv/media/config"))
var kept bytes.Buffer
for _, r := range s {
kept.WriteString(r.ID + r.Type + r.Digest + strings.Join(r.Volumes, ","))
for k, v := range r.Fields {
kept.WriteString(k + v)
}
}
if strings.Contains(kept.String(), "the-secret-itself") {
t.Fatal("a file's content was kept in the record of what was sent")
}
if len(s) != 2 || s[0].Volumes[0] != "/srv/media/config:/config" {
t.Fatalf("summary %+v", s)
}
}
func TestADiffOfAnUnchangedMachineIsEmptyAndAChangedContainerNamesItsField(t *testing.T) {
was := summarized(t, declaration("/srv/media/config"))
if d := diffSent(was, summarized(t, declaration("/srv/media/config"))); !d.empty() {
t.Fatalf("an unchanged machine differs: %+v", d)
}
d := diffSent(was, summarized(t, declaration("/var/lib/plex/config")))
if len(d.Changed) != 1 || d.Changed[0].ID != "plex.server" || !reflect.DeepEqual(d.Changed[0].Fields, []string{"volumes"}) {
t.Fatalf("diff %+v", d)
}
var out bytes.Buffer
writeDiff(&out, "home", d, movesIn(was, summarized(t, declaration("/var/lib/plex/config"))))
for _, want := range []string{"~ plex.server recreated: volumes", "! plex.server: /config moves from /srv/media/config to /var/lib/plex/config"} {
if !strings.Contains(out.String(), want) {
t.Errorf("diff does not say %q:\n%s", want, out.String())
}
}
}
// The incident: the configuration mount would move to an empty default directory.
func TestARunningContainersDataMovingIsAMoveAndAnAddedMountIsNot(t *testing.T) {
was := summarized(t, declaration("/srv/media/config"))
moves := movesIn(was, summarized(t, declaration("/var/lib/plex/config")))
want := []dataMove{{Container: "plex.server", Inside: "/config", From: "/srv/media/config", To: "/var/lib/plex/config"}}
if !reflect.DeepEqual(moves, want) {
t.Fatalf("moves %+v", moves)
}
if moves[0].Module() != "plex" {
t.Errorf("module %q", moves[0].Module())
}
// A mount added — the previews of 2026-10-05 — is not a move.
if m := movesIn(was, summarized(t, declaration("/srv/media/config", "/srv/pool/previews:/config/Media"))); len(m) != 0 {
t.Errorf("an added mount was held as a move: %+v", m)
}
// Nor is a container the machine never ran.
if m := movesIn(nil, was); len(m) != 0 {
t.Errorf("a first send was held as a move: %+v", m)
}
}
func TestTheVerbsCarryReadReplaceAndMove(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
want []string
}{
{"settings", map[string]any{"module": "plex", "node": "home"}, []string{"settings", "show", "plex", "--node", "home"}},
{"settings", map[string]any{"module": "plex", "history": "true"}, []string{"settings", "show", "plex", "--history"}},
{"settings", map[string]any{"module": "plex", "values": "{}", "replace": "true"}, []string{"settings", "set", "plex", "{}", "--replace"}},
} {
got, err := argvFor(c.verb, c.args)
if err != nil || !reflect.DeepEqual(got, c.want) {
t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want)
}
}
}
// An argument of the guards given where it cannot take effect is refused, not passed over (issue 244):
// a move with no machine named, a replace beside a clear, a history beside values, a diff beside files.
func TestTheGuardsArgumentsAreNotPassedOver(t *testing.T) {
for _, c := range []struct {
verb string
args map[string]any
}{
{"push", map[string]any{"move": "plex", "why": "w"}},
{"settings", map[string]any{"module": "plex", "clear": "true", "replace": "true"}},
{"settings", map[string]any{"module": "plex", "values": "{}", "history": "true"}},
{"plan", map[string]any{"node": "home", "files": "true", "diff": "true"}},
} {
if argv, err := argvFor(c.verb, c.args); err == nil {
t.Errorf("%s %v was taken as %v, and an argument passed over", c.verb, c.args, argv)
}
}
}
cmd/mesh-controller/upgrades.go
+8 -384
View File
@@ -13,7 +13,6 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/novox/mesh-controller/internal/inventory"
"github.com/novox/mesh-controller/internal/link"
)
@@ -43,30 +42,6 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error {
return nil
}
// askAgainOnGrants marks a send that stopped at its grants as one to ask again (novox/hq issue 249):
// an announcement handled by a send whose memberships could not be issued is held and redelivered,
// rather than taken as handled with the machines left on the old version.
func askAgainOnGrants(err error) error {
if err != nil && errors.Is(err, errGrants) && !errors.Is(err, link.ErrTryAgain) {
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
}
return err
}
// readableList names machines the way a sentence does, because this is read by a person deciding
// whether an upgrade went where they expected.
func readableList(names []string) string {
switch len(names) {
case 0:
return "nothing"
case 1:
return names[0]
case 2:
return names[0] + " and " + names[1]
}
return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1]
}
func shortCommit(commit string) string {
if len(commit) > 8 {
return commit[:8]
@@ -74,13 +49,6 @@ func shortCommit(commit string) string {
return commit
}
func isAre(n int) string {
if n == 1 {
return "is"
}
return "are"
}
// Announceable is every build this mesh recorded, in the shape the builder announces one.
//
// **The catalogue asks for this when it starts, and the answer is the graph's foundation**
@@ -125,25 +93,6 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error
return out, nil
}
// notNow marks a store that could not be read right now, or plans another controller is working (issue 921), so
// the announcement is held rather than lost; anything else is returned as it was.
func notNow(err error) error {
if inventory.Unreachable(err) {
return fmt.Errorf("%w: %w", link.ErrTryAgain, err)
}
return err
}
// unsaved marks the failure to save the plan a merge was added to, whatever it was: the merge is carried until that save
// writes it (inventory.AddMerge), so the announcement is held and handed over again, never taken (novox/hq ADR 0407
// decision 3, ADR 0408, issue 1092). The bus acknowledges it only after the save succeeded.
func unsaved(err error) error {
if err == nil {
return nil
}
return fmt.Errorf("%w: the merge is not kept until its plan is saved: %w", link.ErrTryAgain, err)
}
// SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which
// becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go.
func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
@@ -153,82 +102,6 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error {
return keepForgeMerge(ctx, m)
}
// movesOfMerge is what one merge moves, acted on: every module recorded as built from that repository and
// branch is marked as moved to the merge commit; a module the merge deleted is forgotten or said; a new module
// is asked for and sent nowhere. The names returned are what the walk builds, bases first along the graph
// (novox/hq 04-ISSUES/131). Nothing is built or pushed here: the walk asks its tiers. Called when a batch is
// cut, once per repository, with the latest merge of the repository's branch and every file the batch's
// merges of it changed.
func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry,
read map[string][]inventory.ReadRepository) ([]string, error) {
from, packaging, already := mergeCandidates(m, entries, read)
if len(from) == 0 && len(packaging) == 0 {
// "Already built from it" and "nothing reads it" are different facts, and reading the first
// as the second sends somebody looking for a broken trigger when the mesh is up to date.
if already > 0 {
fmt.Printf("%s/%s merged into %s (%.8s); %d module(s) the mesh holds are already built "+
"from it\n", m.Owner, m.Repo, m.Base, m.Commit, already)
return nil, nil
}
fmt.Print(nothingReadsIt(m.Owner+"/"+m.Repo, m))
return nil, nil
}
// Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows
// another branch is not part of this merge, and whoever is waiting for its change should read why.
for _, e := range entries {
if sameRepository(e.Source.Repository, m) && !sourceIs(e.Source, m) {
fmt.Printf(" %s is built from %s/%s and follows %s, not %s; this merge leaves it out\n",
e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base)
}
}
touched, added, _ := touchedBy(from, entries, m, read)
// **A module the merge deleted is not built** (novox/hq ADR 0236): its manifest is gone, so the build
// seat finds nothing saying what it is, and the plan failed on it (`has no module.json at …`) with
// every other module of its tier left unsent. It is forgotten where nothing holds it, said otherwise.
touched, deleted := splitDeleted(touched, m)
for _, e := range deleted {
retireDeleted(ctx, inv, e, m)
}
for _, e := range touched {
if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil {
return nil, notNow(err)
}
}
// **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan
// says so, and assigning it is a person's act, which needs it registered first.
// Nothing asks its first build (novox/hq ADR 0501 §6): the builder builds every fingerprint the indexer writes, and
// the controller registers the module from its first done key (new_modules.go).
var built []string
if len(from) > 0 {
built = added
}
moved := append(append([]inventory.Entry{}, touched...), packaging...)
if len(moved) == 0 {
if len(built) > 0 {
fmt.Printf("%s/%s merged into %s (%.8s); it changed no module the mesh holds, and adds %s: "+
"registered from the builder's done key of each, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit,
strings.Join(built, ", "))
return nil, nil
}
fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+
"built from\n", m.Owner, m.Repo, m.Base, m.Commit)
return nil, nil
}
// Why each is in it (issue 363): the files of its build source the merge changed, or why it is read whole.
for _, e := range moved {
fmt.Printf(" %s: %s\n", e.Manifest.Module, whyMoved(e, read[e.Manifest.Module], m))
}
for _, e := range packaging {
fmt.Printf(" %s reads %s/%s through its build context: its own source record is left where it is\n",
e.Manifest.Module, m.Owner, m.Repo)
}
var names []string
for _, e := range moved {
names = append(names, e.Manifest.Module)
}
return names, nil
}
// actingOnMerges keeps one merge acted on at a time (novox/hq issue 266).
var actingOnMerges sync.Mutex
@@ -292,63 +165,6 @@ func lookedAtCommit(read []inventory.ReadRepository, commit string) bool {
// historyMargin is how far a packaging module's last look is taken back before a merge is history for it.
const historyMargin = time.Minute
// whyMoved is why a merge moves a module, as a plan says it (novox/hq ADR 0267, issue 363): the changed
// files in its build source, or why it is read whole. For a module built from the merged repository or one
// whose build context is that repository; a dependent is in a plan for what it stands on.
func whyMoved(e inventory.Entry, read []inventory.ReadRepository, m link.SourceMoved) string {
if len(m.Paths) == 0 || m.PathsTruncated {
return "read whole: the merge's changed files were not all said"
}
whole := "no build source recorded"
for _, r := range read {
if r.Own && r.Whole != "" {
whole = r.Whole
}
}
held := func(paths []string) []string {
var in []string
for _, p := range m.Paths {
if gosource.SourceHolds(paths, p) {
in = append(in, p)
}
}
return in
}
changed := func(where string, in []string) string {
return fmt.Sprintf("its build source%s changed: %d changed file(s) in it, e.g. %s", where, len(in), in[0])
}
if sameRepository(e.Source.Repository, m) {
if own := ownSource(read); own != nil {
if in := held(own); len(in) > 0 {
return changed("", in)
}
}
dir := strings.Trim(e.Source.Path, "/")
if dir == "" {
return "read whole: " + whole + ", so every file of its repository is its build source"
}
for _, p := range m.Paths {
if inside(p, dir) {
return "read whole: " + whole + ", so its directory is its build source; e.g. " + p
}
}
return "read whole: " + whole
}
for _, r := range read {
if r.Own || !sameRepository(r.Repository, m) || !followsBase(r.Ref, m.Base) {
continue
}
if len(r.Paths) > 0 {
if in := held(r.Paths); len(in) > 0 {
return changed(" in "+m.Owner+"/"+m.Repo, in)
}
continue
}
return "read whole: " + whole + ", so every file of " + m.Owner + "/" + m.Repo + ", which its build context is, is its build source"
}
return "read whole: " + whole
}
// lookedOf is when a module packaging another repository was last looked at, as readForPlanning says.
func lookedOf(read []inventory.ReadRepository) time.Time {
var at time.Time
@@ -360,21 +176,6 @@ func lookedOf(read []inventory.ReadRepository) time.Time {
return at
}
// wouldMove is the modules acting on this merge would move and rebuild — SourceMoved's judgement, made
// without acting (novox/hq issue 266). Empty for a merge already acted on: acting marks each module built
// from the repository as looked at, so the merge then reads as history for it.
//
// **The ones packaging source from it too** (novox/hq ADR 0267): with a module moved only by the files of
// its build source, a merge can move a packaging module and nothing built from the repository, and a missed
// one of those was never acted on. A packaging module's look is its newest build or plan (lookedAt), so a
// merge acted on for it reads as history once its plan is made.
func wouldMove(m link.SourceMoved, entries []inventory.Entry,
read map[string][]inventory.ReadRepository) []inventory.Entry {
from, packaging, _ := mergeCandidates(m, entries, read)
touched, _ := splitDeleted(whatTheMergeTouched(from, entries, m, read), m)
return append(touched, packaging...)
}
// splitDeleted parts the modules a merge touched into those it changed and those whose manifest it
// removed — deleted at their source (ADR 0236).
func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted []inventory.Entry) {
@@ -397,39 +198,6 @@ func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted
return kept, deleted
}
// retireDeleted is what the mesh does with a module deleted at its source: its record says the merge
// was looked at, so it is not acted on again; it is forgotten where nothing holds it; where a machine
// runs it or the mesh holds something for it, that is said, and nothing is built.
func retireDeleted(ctx context.Context, inv *inventory.Inventory, e inventory.Entry, m link.SourceMoved) {
name := e.Manifest.Module
if err := inv.SourceMoved(ctx, name, m.Commit); err != nil {
fmt.Printf(" %s was deleted from %s/%s at %.8s, and that could not be recorded: %v\n", name, m.Owner, m.Repo, m.Commit, err)
}
err := inv.ForgetModule(ctx, name)
switch {
case err == nil:
fmt.Printf(" %s was deleted from %s/%s at %.8s: forgotten, nothing built\n", name, m.Owner, m.Repo, m.Commit)
case errors.Is(err, inventory.ErrStillAssigned) || errors.Is(err, inventory.ErrStillHolds):
fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; the mesh still runs or holds it, so it is "+
"kept until a person unassigns it and `module forget %s`: %v\n", name, m.Owner, m.Repo, m.Commit, name,
firstLine(err.Error()))
default:
fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; forgetting it failed: %v\n", name, m.Owner,
m.Repo, m.Commit, err)
}
}
// nothingReadsIt is what is said of a merge no module the mesh holds reads. **A merge into a branch other than the
// trunk is said to start no delivery** (novox/hq ADR 0401 clause 3, issue 889): every delivery is cut from a
// repository's trunk, and whoever merged into a feature branch should not look for a broken trigger.
func nothingReadsIt(repository string, m link.SourceMoved) string {
if m.Base != "" && m.Base != "main" {
return fmt.Sprintf("%s merged into %s (%.8s), not the trunk of %s; a merge into any other branch starts no "+
"delivery\n", repository, m.Base, m.Commit, repository)
}
return fmt.Sprintf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit)
}
// sourceIs is whether a recorded source is the repository and branch a merge announced. A source on
// the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from.
// An empty recorded ref is the repository's default branch, which is what a merge into the base
@@ -531,79 +299,12 @@ func readsDirectory(e inventory.Entry, p string) bool {
return strings.Trim(e.Source.Path, "/") == "" || inside(p, e.Source.Path)
}
// unchangedOfMerge are the modules a merge changed files for under a reading without recorded build sources, but
// whose recorded build source holds none of them, each with why (novox/hq issue 611): not built, because the build
// the mesh holds is theirs (ADR 0267 §5). That reading — a delivery plan made while another walk built the module
// (issue 606) — would have moved them; the walk says they matched what runs rather than leaving them unnamed.
// A module built from the repository whose directory, or whole repository when built from its root, holds a
// changed file, and a module packaging the repository's branch, are each read so. A merge whose changed files
// were not all said moves every one of them: none is said here. Without acting.
func unchangedOfMerge(m link.SourceMoved, entries []inventory.Entry,
read map[string][]inventory.ReadRepository) map[string]string {
if len(m.Paths) == 0 || m.PathsTruncated {
return nil
}
from, _, _ := mergeCandidates(m, entries, read)
touched, _, _ := touchedBy(from, entries, m, read)
out := map[string]string{}
for _, e := range from {
name := e.Manifest.Module
if ownSource(read[name]) == nil || slices.ContainsFunc(touched, func(t inventory.Entry) bool {
return t.Manifest.Module == name
}) {
continue
}
var in []string
for _, p := range m.Paths {
if readsDirectory(e, p) {
in = append(in, p)
}
}
if len(in) > 0 {
out[name] = fmt.Sprintf("its build source holds none of the %d changed file(s) in its directory (e.g. %s): "+
"the build the mesh holds is its build", len(in), in[0])
}
}
for _, e := range entries {
name := e.Manifest.Module
if sourceIs(e.Source, m) || !packagesSaid(read[name], m) || readsFrom(read[name], m) {
continue
}
out[name] = fmt.Sprintf("its build source in %s/%s holds none of the %d changed file(s) (e.g. %s): "+
"the build the mesh holds is its build", m.Owner, m.Repo, len(m.Paths), m.Paths[0])
}
if len(out) == 0 {
return nil
}
return out
}
// packagesSaid is whether a module's newest trunk build said the build source it read in a merge's repository and
// branch, as a context beside its own.
func packagesSaid(read []inventory.ReadRepository, m link.SourceMoved) bool {
for _, r := range read {
if !r.Own && sameRepository(r.Repository, m) && followsBase(r.Ref, m.Base) && len(r.Paths) > 0 {
return true
}
}
return false
}
// readForPlanning is what each module's build read, as a change is mapped onto it — for a merge's reach, a pull request's
// check and the facts alike (novox/hq ADR 0238), so they cannot disagree. No plan overtakes it any more (ADR 0539 §17).
func readForPlanning(ctx context.Context, inv *inventory.Inventory) (map[string][]inventory.ReadRepository, error) {
return inv.ReadRepositories(ctx)
}
// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: **a
// changed file touches exactly the modules whose build reads it** (novox/hq issue 280, ADR 0238). It is
// touchedBy's first answer; touchedBy is the one place the mesh maps a changed file onto its modules.
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved,
read map[string][]inventory.ReadRepository) []inventory.Entry {
touched, _, _ := touchedBy(candidates, known, m, read)
return touched
}
// touchedBy maps a merge's changed files onto the mesh's modules — **the one place it is done**, for the
// merge handler, the release planner's what-if, the merge gate and a pull request's check alike (novox/hq
// ADR 0238), so planning and gating cannot disagree about what a change touches.
@@ -757,49 +458,6 @@ func inside(path, dir string) bool {
return path == dir || strings.HasPrefix(path, dir+"/")
}
// anyInside is whether any of these changed files is in a directory.
func anyInside(paths []string, dir string) bool {
for _, p := range paths {
if inside(p, dir) {
return true
}
}
return false
}
// standsOnModule is whether an entry's build stood on the named module: by what its newest build
// recorded it was handed (`artifact-store://<module>/<artifact>@…`, the module's own artifact), or
// — for a module registered from a manifest and not yet built — by the base its manifest names.
func standsOnModule(e inventory.Entry, module string, against map[string][]string) bool {
if e.Manifest.Module == module {
return false
}
if e.Manifest.Build != nil {
for _, on := range e.Manifest.Build.On {
if on.Module == module {
return true
}
}
}
prefix := catalogue.ArtifactStoreScheme + module + "/"
for _, ref := range against[e.Manifest.Module] {
if strings.HasPrefix(ref, prefix) {
return true
}
}
return false
}
// standsOn is whether anything in the set is built on the named module's artifacts.
func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool {
for _, e := range entries {
if standsOnModule(e, module, against) {
return true
}
}
return false
}
// isHistory is whether a merge made at mergedAt predates the last time the source was seen. A merge
// with no time on it is taken as news: refusing it would silence a forge that says less.
func isHistory(mergedAt string, seen time.Time) bool {
@@ -813,52 +471,10 @@ func isHistory(mergedAt string, seen time.Time) bool {
return at.Before(seen)
}
// dependentsOf is every catalogued module that stands on one of the moved modules, directly or
// through another dependent, and is not itself among them — in the catalogue's order, so the
// answer is the same each time. A module standing on nothing that moved is left alone: a merge
// rebuilds what it changed and what is built on top of that, not the catalogue.
func dependentsOf(moved, entries []inventory.Entry, against map[string][]string) []inventory.Entry {
bases := map[string]bool{}
for _, e := range moved {
bases[e.Manifest.Module] = true
}
var out []inventory.Entry
taken := map[string]bool{}
for grew := true; grew; {
grew = false
for _, e := range entries {
name := e.Manifest.Module
if bases[name] || taken[name] {
continue
}
for base := range bases {
if standsOnModule(e, base, against) {
taken[name] = true
out = append(out, e)
grew = true
break
}
}
}
for _, e := range out {
bases[e.Manifest.Module] = true
}
}
return out
}
// moduleManifestFile is the file that says what a module is, in its directory (the build seat's
// ManifestName).
const moduleManifestFile = "module.json"
// deletedAtSource is whether a build failed because its module's manifest is not at its source any
// more — the build seat's own words (internal/builder) — which a merge that deleted the module causes
// when its announcer did not say which files went (ADR 0236). Such a module is not a failure of the plan.
func deletedAtSource(failed string) bool {
return strings.Contains(failed, "has no "+moduleManifestFile+" at ") &&
strings.Contains(failed, "so there is nothing saying what it is")
}
// reachOfMerge is what a merge of these changed files into this branch would move, without acting: a pull request's
// reach, as the check reckons it.
func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository,
@@ -1026,3 +642,11 @@ func sayUpgrade(module string, u inventory.Upgrade) string {
return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+
"the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from)
}
// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: touchedBy's first answer
// (novox/hq issue 280, ADR 0238).
func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved,
read map[string][]inventory.ReadRepository) []inventory.Entry {
touched, _, _ := touchedBy(candidates, known, m, read)
return touched
}
internal/inventory/build_lookups.go
-8
View File
@@ -24,11 +24,3 @@ func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[s
}
return i.chainBuildFingerprints(ctx, c, module)
}
// short is a commit as a person reads it.
func short(commit string) string {
if len(commit) > 8 {
return commit[:8]
}
return commit
}
internal/inventory/nodes.go
-11
View File
@@ -903,17 +903,6 @@ type BusyWait struct {
// BusyWaiting is the state of a container left running because it said busy (link.BusyWaiting).
const BusyWaiting = "waiting"
// WaitingFor is the container of a module that waits because it said busy, and whether one does: a
// resource whose id is `<module>.<resource>`.
func WaitingFor(busy []BusyWait, module string) (BusyWait, bool) {
for _, b := range busy {
if b.State == BusyWaiting && strings.HasPrefix(b.ID, module+".") {
return b, true
}
}
return BusyWait{}, false
}
// Waiting says whether any container waits because it said busy.
func (d Doing) Waiting() bool {
for _, b := range d.Busy {
internal/link/bus.go
-18
View File
@@ -26,11 +26,6 @@ type Bus interface {
// nobody is obliged to act (ADR 0041).
PublishEvent(ctx context.Context, key, source, node string, body []byte) error
// PublishDeclaration delivers one node what it should be. Addressed to that node alone: a
// declaration is not an event, and replaying yesterday's is actively harmful
// (design 29 §4, the *state* shape).
PublishDeclaration(ctx context.Context, node string, body []byte) error
// PublishSeatEvent states a fact under a role's own name, for the holder of that role. A
// module's event is addressed to the module; a role's is addressed to the role, so it keeps
// meaning when the holder changes (novox/hq ADR 0121, ADR 0129).
@@ -101,11 +96,6 @@ func SeatEventSubject(seat, event string) string {
return "mesh.seat." + seat + ".event." + event
}
// DeclareSubject is where one node's declaration lands. Last-per-subject on the NODES stream, so
// a node that was away gets exactly the current one and a replayed older one is refused by
// sequence — the wire-level answer to novox/hq issue 107.
func DeclareSubject(node string) string { return "mesh.node." + node + ".declare" }
func (b OverNATS) PublishEvent(ctx context.Context, key, source, node string, body []byte) error {
id, err := eventID()
if err != nil {
@@ -192,14 +182,6 @@ func (b OverNATS) MaxPayload() int64 {
return b.Conn.MaxPayload()
}
func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error {
_, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx))
if err != nil {
return fmt.Errorf("declaring to %s: %w", node, err)
}
return nil
}
// ToolSubject is where a module answers. Derived from the module and the tool, so a caller names
// what it wants rather than where it lives.
func ToolSubject(module, tool string) string { return "mesh.mod." + module + ".tool." + tool }
internal/link/bus_test.go
-10
View File
@@ -12,16 +12,6 @@ import (
//
// go test ./internal/link/ -run TestTheNatsBus (each test on a bus of its own: internal/testbus)
// The subject a declaration lands on is one node's, and nothing else's — the state shape.
func TestADeclarationIsAddressedToOneNode(t *testing.T) {
if got := DeclareSubject("anchor"); got != "mesh.node.anchor.declare" {
t.Fatalf("a declaration would go to %q", got)
}
if DeclareSubject("anchor") == DeclareSubject("laptop") {
t.Fatal("two nodes share a declaration subject, so each would apply the other's")
}
}
// A module cannot emit under another's name: the subject is derived from the source, and the
// server's permissions make that subject the authority.
func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) {
internal/link/declare.go
-94
View File
@@ -1,94 +0,0 @@
package link
import (
"context"
"encoding/json"
"fmt"
"time"
)
// Signer is whatever holds the control plane's signing key.
type Signer interface {
Sign(ctx context.Context, message []byte) ([]byte, error)
}
// ActingGate is what every send passes before it is made (novox/hq to-be 45 §6): whether this process
// may act under the controller's lease. Set by the controller; nil passes every send — a test, a tool.
var ActingGate func(ctx context.Context) error
// Declare sends a node what it should be, signed.
//
// The signature is over the declaration exactly as it is published — the same bytes the node
// verifies. Anything that re-encoded between here and there would produce a signature over
// something else, and the node would refuse a declaration that was genuinely the mesh's.
//
// Published to the node's own queue, which its account alone may read.
func Declare(ctx context.Context, bus Bus, signer Signer, node string,
declaration []byte, timeout time.Duration) error {
if !json.Valid(declaration) {
return fmt.Errorf("refusing to send %s something that is not a declaration", node)
}
// **At the send, not only where it was composed**: a lease lost between the two stops this one.
if ActingGate != nil {
if err := ActingGate(ctx); err != nil {
return fmt.Errorf("%s was not sent its declaration: %w", node, err)
}
}
signature, err := signer.Sign(ctx, declaration)
if err != nil {
return fmt.Errorf("cannot sign a declaration for %s: %w", node, err)
}
body, err := Envelope(declaration, signature)
if err != nil {
return err
}
// **Measured before it is sent** (novox/hq issue 1689): the bare bus error said only "maximum payload
// exceeded", on 2026-10-21 for the control node's declaration, 2 KB over. The limit is the connection's own.
if limit := payloadLimitOf(bus); limit > 0 {
size := int64(len(body))
if DeclarationMeasured != nil {
DeclarationMeasured(ctx, node, size, limit)
}
if size > limit {
return fmt.Errorf("%s's declaration is %d bytes on the wire (%d bytes of declaration, signed and encoded), "+
"over the bus's largest message of %d bytes by %d: it cannot be sent (novox/hq issue 1689)",
node, size, len(declaration), limit, size-limit)
}
}
publish, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
return bus.PublishDeclaration(publish, node, body)
}
// Envelope is the message a declaration goes on the bus as: the declaration and its signature, each as base64 in one JSON
// document. Its length is what the bus's largest message bounds.
func Envelope(declaration, signature []byte) ([]byte, error) {
return json.Marshal(Signed{Declaration: declaration, Signature: signature})
}
// NearTheLimit is the share of the bus's largest message at which a declaration is said to be near it: three quarters, so that
// a declaration which should stay far from the limit raises the condition while there is still room to act (novox/hq ADR 0486
// §8; the first fix of issue 1689 warned at nine tenths while every file was still inline).
const NearTheLimit = 0.75
// DeclarationMeasured is told, at every send of a declaration, its size on the wire and the bus's largest message,
// so the controller raises a condition when it nears the limit and clears it when it is back under (issue 1689).
// Set by the controller; nil in a test or a tool.
var DeclarationMeasured func(ctx context.Context, node string, size, limit int64)
// payloadLimiter is a bus that knows its largest message.
type payloadLimiter interface{ MaxPayload() int64 }
// payloadLimitOf is the bus's largest message in bytes, or 0 when the bus does not say.
func payloadLimitOf(bus Bus) int64 {
if l, ok := bus.(payloadLimiter); ok {
return l.MaxPayload()
}
return 0
}
internal/link/declare_room_integration_test.go
-52
View File
@@ -1,52 +0,0 @@
//go:build integration
package link
import (
"context"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-controller/internal/testbus"
)
// Against a real bus whose largest message is small: the limit is read from the connection, the one that fits goes
// through, the one over is refused by us with its size and the limit, not by the client's bare error (issue 1689).
func TestADeclarationIsMeasuredAgainstTheRealBusLimit(t *testing.T) {
s := testbus.StartWithMaxPayload(t, 2048)
conn, err := nats.Connect(s.ClientURL())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
js, err := conn.JetStream()
if err != nil {
t.Fatal(err)
}
if _, err := js.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"}}); err != nil {
t.Fatal(err)
}
bus := OverNATS{Conn: conn, JS: js}
if bus.MaxPayload() != 2048 {
t.Fatalf("the limit read from the connection is %d, expected 2048", bus.MaxPayload())
}
var told []int64
was := DeclarationMeasured
DeclarationMeasured = func(_ context.Context, _ string, size, limit int64) { told = append(told, size, limit) }
t.Cleanup(func() { DeclarationMeasured = was })
fits := []byte(`{"a":"` + strings.Repeat("x", 600) + `"}`)
if err := Declare(t.Context(), bus, fixedSigner{}, "novox", fits, 5*time.Second); err != nil {
t.Fatalf("a declaration under the limit: %v", err)
}
over := []byte(`{"a":"` + strings.Repeat("x", 1600) + `"}`)
err = Declare(t.Context(), bus, fixedSigner{}, "novox", over, 5*time.Second)
if err == nil || !strings.Contains(err.Error(), "2048 bytes") || strings.Contains(err.Error(), "maximum payload exceeded") {
t.Fatalf("a declaration over the limit: %v", err)
}
if len(told) != 4 || told[1] != 2048 || told[2] <= 2048 {
t.Fatalf("measured %v", told)
}
}
internal/link/declare_room_test.go
-57
View File
@@ -1,57 +0,0 @@
package link
import (
"context"
"strings"
"testing"
"time"
)
type roomBus struct {
Bus
limit int64
sent [][]byte
}
func (b *roomBus) MaxPayload() int64 { return b.limit }
func (b *roomBus) PublishDeclaration(_ context.Context, _ string, body []byte) error {
b.sent = append(b.sent, body)
return nil
}
type fixedSigner struct{}
func (fixedSigner) Sign(context.Context, []byte) ([]byte, error) { return []byte("sig"), nil }
// A declaration is measured as it goes on the wire against the bus's own limit; one over it is refused saying size and
// limit, and what the measure sees is told to whoever raises a condition (issue 1689).
func TestADeclarationOverTheBusLimitIsRefusedSayingSizeAndLimit(t *testing.T) {
var seen []int64
was := DeclarationMeasured
DeclarationMeasured = func(_ context.Context, node string, size, limit int64) {
if node != "novox" || limit != 400 {
t.Errorf("measured %s against %d", node, limit)
}
seen = append(seen, size)
}
t.Cleanup(func() { DeclarationMeasured = was })
bus := &roomBus{limit: 400}
small := []byte(`{"a":"` + strings.Repeat("x", 100) + `"}`)
if err := Declare(t.Context(), bus, fixedSigner{}, "novox", small, time.Second); err != nil || len(bus.sent) != 1 {
t.Fatalf("a small declaration: %v, %d sent", err, len(bus.sent))
}
if int64(len(bus.sent[0])) != seen[0] {
t.Fatalf("measured %d bytes, sent %d", seen[0], len(bus.sent[0]))
}
big := []byte(`{"a":"` + strings.Repeat("x", 300) + `"}`)
err := Declare(t.Context(), bus, fixedSigner{}, "novox", big, time.Second)
if err == nil || !strings.Contains(err.Error(), "400 bytes") || !strings.Contains(err.Error(), "novox") ||
!strings.Contains(err.Error(), "over") || strings.Contains(err.Error(), "maximum payload exceeded") {
t.Fatalf("an oversize declaration: %v", err)
}
if len(bus.sent) != 1 || len(seen) != 2 || seen[1] <= 400 {
t.Fatalf("it was sent (%d) or not measured (%v)", len(bus.sent), seen)
}
}
internal/link/enrolment.go
-49
View File
@@ -11,7 +11,6 @@ import (
"fmt"
"log"
"sort"
"strings"
"time"
"github.com/novox/mesh-controller/internal/broker"
@@ -268,16 +267,6 @@ var _ Enroller = Enrolment{}
// A node states; the owning context writes (novox/hq ADR 0006). What a node says it applied is
// its own account of its own machine, kept as a copy for recovery — so this writes it down and
// decides nothing from it.
// Outstanding is the declaration the mesh last sent a node, so a report about an older one is not
// acted on (design 25 §3, window.go).
//
// **Here rather than on Listener.** A report is recorded by whatever keeps records, and a great
// many things that record reports have no idea what was sent — every test in this package among
// them. So the serving loop asks for this when the listener happens to be able to answer, and
// where it cannot, a report has nothing to be stale against and is simply acted on.
func (e Enrolment) Outstanding(ctx context.Context, node string) (string, error) {
return e.Inventory.Outstanding(ctx, node)
}
func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err error) {
// A store that could not be asked right now is said as such, so the report is kept for
@@ -444,11 +433,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
// arrives; the digest the mesh last sent decides nothing for it. **By digest, from an older
// node-engine**: its report claims no order, and the rule of issue 267 stands for it.
//
// And whether the node-engine reads an epoch in a declaration, from every account it gives: the
// mesh sends one only to a machine that said so, and one rolled back says so no longer.
if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil {
return false, err
}
if report.Ordered() {
account := AccountOf(report)
news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing,
@@ -467,17 +451,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
return false, err
}
} else {
if report.Declared != "" {
sent, err := e.Inventory.Outstanding(ctx, report.Node)
if err != nil {
return false, err
}
if Superseded(report.Declared, sent) {
log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+
"the mesh has sent it %s since", report.Node, short(report.Declared), short(sent))
return false, e.Inventory.Seen(ctx, node.ID)
}
}
// **Whether this is news** is the store's answer: it holds the previous report, and a machine
// that reconciles every minute says the same thing until something changes (novox/hq ADR 0134).
news, err = e.Inventory.RecordDoing(ctx, node.ID, doing)
@@ -493,25 +466,3 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err
}
return news, e.Inventory.RecordOwned(ctx, node.ID, report.Applied)
}
// busyWords is every container waiting because it said busy, in a line (novox/hq ADR 0335).
func busyWords(busy []inventory.BusyWait) string {
var out []string
for _, b := range busy {
if b.State == inventory.BusyWaiting {
out = append(out, fmt.Sprintf("%s since %s, at the latest until %s", b.Container,
b.Since.UTC().Format(time.RFC3339), b.Until.UTC().Format(time.RFC3339)))
}
}
return strings.Join(out, "; ")
}
// replacedAfterWaiting says a report replaced a container that had waited because it was busy.
func replacedAfterWaiting(busy []inventory.BusyWait) bool {
for _, b := range busy {
if b.State != inventory.BusyWaiting {
return true
}
}
return false
}
internal/link/generation_test.go
-48
View File
@@ -1,48 +0,0 @@
package link
import (
"encoding/json"
"testing"
)
// A report's word on the assignment generation (novox/hq issue 234), as mesh-host's report writes it:
// `reads_generation` on every report of a node-engine that reads one, and `older_generation` on a refusal
// of a declaration composed from an older generation than the machine applied.
func TestTheGenerationOnTheWire(t *testing.T) {
raw := []byte(`{"node":"anchor","refused":"older generation","declared":"d12","epoch":57,"sequence":12,` +
`"report_sequence":8,"reads_generation":true,"older_generation":{"generation":38,"applied":40}}`)
var r Report
if err := json.Unmarshal(raw, &r); err != nil {
t.Fatal(err)
}
if !r.ReadsGeneration || r.OlderGeneration == nil || *r.OlderGeneration != (GenerationRefusal{38, 40}) ||
r.Sequence != 12 {
t.Fatalf("a node-engine's generation refusal reads as %+v", r)
}
// Not a refusal by the lease's epoch: S13 counts those, and this one is raised naming its sender.
if r.StaleRefusalOf() {
t.Errorf("a generation refusal reads as a stale writer's")
}
var older Report
if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil {
t.Fatal(err)
}
if older.ReadsGeneration || older.OlderGeneration != nil {
t.Fatalf("a node-engine that never said it reads a generation reads as one that does: %+v", older)
}
}
// 1145/AC3: a report's word that the node-engine reads a module per resource and a build per module, as mesh-host's
// report writes it: `reads_modules`, on every report of a node-engine that reads them.
func TestWhetherTheNodeEngineReadsModulesIsOnItsReport(t *testing.T) {
var reads, older Report
if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":8,"reads_modules":true}`), &reads); err != nil {
t.Fatal(err)
}
if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil {
t.Fatal(err)
}
if !reads.ReadsModules || older.ReadsModules {
t.Fatalf("1145/AC3: reads %v, an older node-engine reads %v", reads.ReadsModules, older.ReadsModules)
}
}
internal/link/heard_order_test.go
-22
View File
@@ -1,7 +1,6 @@
package link_test
import (
"testing"
"time"
"github.com/novox/mesh-controller/internal/link"
@@ -25,27 +24,6 @@ func ordered(declared string, epoch, sequence, report int64, applied ...string)
// A node-engine rolled back to one that orders nothing: its account is taken by the digest rule, the
// order kept is cleared, and it is no longer sent an epoch.
// A stale refusal names its writer by the epoch the refused declaration claimed, and a refusal whose
// own report was lost is still counted from the machine's own tally.
func TestAStaleRefusalIsCountedByItsWriter(t *testing.T) {
count := link.NewRefusalCount()
now := time.Now()
refusal := link.Report{Node: "anchor", Refused: "older", Order: link.Order{Epoch: 41, Sequence: 11},
OlderThan: &link.Order{Epoch: 57, Sequence: 12}, ReportSequence: 9, RefusedOlder: 3}
if !refusal.StaleRefusalOf() {
t.Fatal("a refusal naming the order it holds is not stale")
}
count.Lifetime("anchor", 2, now) // the machine's tally, as the controller first heard it
count.Refused(link.Refusal{Writer: link.WriterEpoch(refusal.Epoch), Epoch: refusal.Epoch, Receiver: "anchor", At: now})
count.Lifetime("anchor", refusal.RefusedOlder, now)
count.Lifetime("anchor", 5, now) // two more refused, their reports lost
got := count.Within(now.Add(-time.Minute))
if len(got) != 2 || got[0].Count != 2 || got[0].Epoch != 0 || got[1].Writer != "the controller of epoch 41" ||
got[1].Count != 1 || got[1].Receivers[0] != "anchor" {
t.Fatalf("the refusals by writer are %+v", got)
}
}
// countFor is how many refusals of a writer this process has heard in the last minute.
func countFor(writer string) int {
for _, w := range link.StaleRefusals.Within(time.Now().Add(-time.Minute)) {
internal/link/order.go
+9 -66
View File
@@ -2,68 +2,22 @@ package link
import "strconv"
// The order a declaration carries, and the order a report about one carries back (novox/hq to-be 45
// §6, ADR 0227 rule 2).
// The order of a node's facts (novox/hq ADR 0539 §9, to-be 45 §6, ADR 0227 rule 2).
//
// **This file is the controller's side of the contract with the node-engine for Phase 2**, and the one
// place it is written here. It mirrors mesh-host internal/link/messages.go (`Order`, and the report's
// `report_sequence`, `older_than`, `refused_older`) field for field and rule for rule, as Report has
// always mirrored the host's report: a key added on one side and not the other is a key the other side
// does not know exists.
//
// **The wire.**
//
// - A declaration's order is two top-level keys of its signed envelope, beside "declaration" and
// "resources": `sequence` (since issue 107) and `epoch` (new). Inside the signed bytes, so a message
// cannot be given a newer order than the controller gave it.
// - A report carries the order of the declaration it is about as the same two top-level keys, beside
// "declared"; `report_sequence`, the node-engine's own number for the report; `older_than`, on a
// refusal of a declaration older than one it applied, the order of the one it holds; and
// `refused_older`, how many it has refused so, ever, on every report.
//
// Every key is optional and absent when zero, and zero is "no order claimed", never "first". So each
// side reads the other's older shape:
//
// - **An older node-engine with this controller.** It decodes a declaration strictly and refuses a key
// it does not know, whole — so `epoch` is sent only to a machine whose latest report carried a
// `report_sequence`, which a node-engine that reads the epoch always does. Until then it is sent the
// sequence alone, as today; its reports carry no report sequence and are judged by the digest they
// name, as today (issue 267).
// - **A newer node-engine with an older controller.** It is sent no epoch, which claims none: it
// refuses nothing by epoch. The keys it adds to a report are fields the older controller ignores.
// - **A controller rolled back to a build without the lease** sends no epoch again and is not refused
// for it: a node-engine refuses by epoch only when both declarations claim one. That gives up the
// epoch's protection while such a build runs — the price of a rollback that cannot strand every
// machine.
//
// **Epoch** is the controller lease's epoch (to-be 45 §6): the lease bucket's revision at which the
// instance that composed the declaration took the lease. It only grows: a later holder's is higher, and
// the controller never issues one at or under the highest it has issued (internal/lease, the floor).
// **Sequence** is the declaration's number for that machine, one higher every send, taken from the
// controller's store before the declaration is composed (issues 107, 204).
// A node-engine says its facts (profile, firewall found, reachable, tunnel, witness, rollbacks, rekey, health) in a report
// on the bus, and orders them itself: `epoch` and `sequence` are the node-engine's own (Writer `node-engine`), and
// `report_sequence` numbers each report it makes. The controller keeps the newest account of a machine and refuses an older
// one, said and counted (rule 2): by epoch, then sequence, then report sequence, each compared only where both claim one.
// Zero is "no order claimed", never "first". The controller sends a machine no declaration to be ordered by: the order here
// is only ever the machine's own.
// Order is where a declaration stands among everything the mesh has sent a machine: the lease epoch it
// was sent under and its sequence. Zero in either claims none.
// Order is where a node-engine's account stands among its own: the epoch and sequence the node-engine counts. Zero in either
// claims none.
type Order struct {
Epoch int64 `json:"epoch,omitempty"`
Sequence int64 `json:"sequence,omitempty"`
}
// Older is the node-engine's rule, as mesh-host states it: a declaration of this order is older than
// one of order `than` the machine applied **only when both claim an epoch** — and then when its epoch
// is lower, or its epoch is the same and its sequence lower (both claimed). A higher epoch is a new lease
// holder and is never older, whatever its sequence. Here so the controller's tests state what the
// machines will do with what it sends.
func (o Order) Older(than Order) bool {
if o.Epoch <= 0 || than.Epoch <= 0 {
return false
}
if o.Epoch != than.Epoch {
return o.Epoch < than.Epoch
}
return o.Sequence > 0 && than.Sequence > 0 && o.Sequence < than.Sequence
}
// Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a Stringer
// promoted onto every report would print each one as its order alone.
func (o Order) Words() string {
@@ -91,17 +45,6 @@ func AccountOf(r Report) Account { return Account{Order: r.Order, ReportSequence
// before (issue 267).
func (r Report) Ordered() bool { return r.ReportSequence > 0 }
// ReadsEpoch says the node-engine that made the report takes an epoch in a declaration: one that orders
// its reports does (mesh-host: "its presence also says this host reads a declaration's epoch").
func (r Report) ReadsEpoch() bool { return r.ReportSequence > 0 }
// StaleRefusalOf says a report refuses a declaration older than one the machine applied: the node-engine
// names the order it holds (`older_than`), or, from a node-engine older than that, says so in the words
// of its sequence refusal.
func (r Report) StaleRefusalOf() bool {
return r.OlderThan != nil || (r.Refused != "" && IsStaleRefusal(r.Refused))
}
// OlderThan is the controller's rule (to-be 45 §6): whether this account is older than the one kept for
// the machine — **by epoch, then sequence, then report sequence**, each compared only where both claim
// one. An account of a declaration of an older epoch is refused; of an older sequence, refused; an older
internal/link/order_test.go
+4 -36
View File
@@ -5,34 +5,8 @@ import (
"testing"
)
// The contract of order.go, case by case: what a node-engine refuses of a declaration (the rule
// mesh-host states, restated so the controller's tests say what the machines do with what it sends),
// what the controller refuses of a report, and the bytes on the wire both ways.
func TestADeclarationIsOlderOnlyWhenBothClaimAnEpoch(t *testing.T) {
for _, c := range []struct {
name string
arriving Order
held Order
older bool
}{
{"a newer sequence of the same epoch", Order{57, 12}, Order{57, 11}, false},
{"the same declaration again (a reconcile)", Order{57, 12}, Order{57, 12}, false},
{"a lower sequence of the same epoch", Order{57, 11}, Order{57, 12}, true},
// Issue 204: a controller that lost its lease goes on sending.
{"an older epoch, whatever its sequence", Order{41, 99}, Order{57, 12}, true},
{"a newer epoch, whatever its sequence", Order{57, 3}, Order{41, 99}, false},
{"no epoch arriving: a rolled-back controller is not stranded", Order{0, 11}, Order{57, 12}, false},
{"no epoch held: what the machine held before any lease", Order{57, 11}, Order{0, 12}, false},
{"no order at all", Order{}, Order{57, 12}, false},
} {
t.Run(c.name, func(t *testing.T) {
if got := c.arriving.Older(c.held); got != c.older {
t.Fatalf("%s against %s: older %v, want %v", c.arriving.Words(), c.held.Words(), got, c.older)
}
})
}
}
// The contract of order.go, case by case: what the controller refuses of a node-engine's facts (the node-engine's own order)
// and the bytes on the wire.
func TestAnAccountIsOlderByEpochThenSequenceThenReportSequence(t *testing.T) {
for _, c := range []struct {
@@ -78,7 +52,7 @@ func TestTheOrderOnTheWire(t *testing.T) {
r.RefusedOlder != 3 {
t.Fatalf("a node-engine's stale refusal reads as %+v", r)
}
if !r.Ordered() || !r.ReadsEpoch() || !r.StaleRefusalOf() {
if !r.Ordered() {
t.Fatalf("a node-engine that orders its reports reads as one that does not: %+v", r)
}
@@ -87,15 +61,9 @@ func TestTheOrderOnTheWire(t *testing.T) {
if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"declared":"d1"}`), &older); err != nil {
t.Fatal(err)
}
if older.Ordered() || older.ReadsEpoch() || older.StaleRefusalOf() {
if older.Ordered() {
t.Fatalf("an older node-engine's report reads as ordered: %+v", older)
}
// Its stale refusal, in the words it has always used, is still one.
older.Refused = "this declaration " + StaleRefusal + ": it is sequence 3"
if !older.StaleRefusalOf() {
t.Fatal("an older node-engine's refusal of an older sequence does not read as stale")
}
// And a report with no order puts no order key on the wire.
raw, err := json.Marshal(Report{Node: "anchor", Declared: "d1"})
if err != nil {
internal/link/receive.go
-12
View File
@@ -121,15 +121,3 @@ type Inbound interface {
// Close lets go of whatever the implementation holds.
Close()
}
// Outstanding answers which declaration the mesh last sent a node — the digest, not the
// declaration.
//
// **Asked before the store is waited on** (design 25 §3): a report about a declaration the mesh
// has already moved past is not worth holding a slot in the window that a current message needs.
// It is a separate interface from Listener rather than a method on it, because a controller that
// only publishes needs neither and something that records reports need not also be able to say
// what was sent.
type Outstanding interface {
Outstanding(ctx context.Context, node string) (string, error)
}
internal/link/receive_nats_integration_test.go
-19
View File
@@ -102,25 +102,6 @@ func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T
})
}
func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) {
js := aBus(t)
store := &sentAndHeardSafely{sent: "d2"}
_, stop := servingOn(t, js, store)
defer stop()
body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}})
if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil {
t.Fatal(err)
}
eventually(t, "the superseded report leaving the stream", func() bool {
info, err := js.Context().StreamInfo("CONTROL")
return err == nil && info.State.Msgs == 0
})
if store.count() != 0 {
t.Fatalf("a report about a superseded declaration was acted on")
}
}
func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) {
js := aBus(t)
s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()},
internal/link/serve.go
+1 -59
View File
@@ -292,26 +292,6 @@ func (s *Server) giveUpAfter() time.Duration {
return s.giveUp
}
// outstanding is the digest of the declaration the mesh last sent a node.
//
// Nothing is guessed when it cannot be answered: a listener that keeps no record of what was sent
// gives a report nothing to be stale against, and a store that cannot be read will hold the report
// anyway, so there is nothing for the check to decide.
func (s *Server) outstanding(ctx context.Context, node string) string {
if node == "" {
return ""
}
asks, ok := s.listener.(Outstanding)
if !ok {
return ""
}
digest, err := asks.Outstanding(ctx, node)
if err != nil {
return ""
}
return digest
}
// toolsHeartbeat records that a machine's node tools were heard from (novox/hq to-be 45 S11), and
// nothing else: in this process's memory, for the watchdog — the next one is a minute away.
func (s *Server) toolsHeartbeat(m Control) {
@@ -343,25 +323,8 @@ func (s *Server) reported(ctx context.Context, m Control) {
what := fmt.Sprintf("%s's report of declaration %s", report.Node, report.Declared)
if s.listener != nil {
// **Asked before the store, not after** (design 25 §3). A report about a declaration the
// mesh has moved past would otherwise wait out a restarting store to be written and then
// overwrite what the node is doing now — and on the bus being built, where the holding is
// the server's, it comes back after the newer was applied whatever the controller does.
outstanding := s.outstanding(ctx, report.Node)
declaredIn := staleAgainst(report)
if report.Ordered() {
// Kept by its order, not by the digest the mesh last sent (novox/hq to-be 45 §6): the
// listener refuses an older account under the one rule, whatever it is about.
declaredIn = ""
}
if Superseded(declaredIn, outstanding) {
s.log.Printf("set aside %s: the mesh has moved past that declaration", what)
_ = m.Took()
return
}
news, err := s.listener.Heard(context.Background(), report)
switch s.decide(ctx, m, what, declaredIn, outstanding, err) {
switch s.decide(ctx, m, what, "", "", err) {
case Hold:
// Held, not settled, while the store cannot take it: the node reports an apply once,
// and a report lost here is a node the mesh never hears from again — the store
@@ -396,27 +359,6 @@ func (s *Server) reported(ctx context.Context, m Control) {
_ = m.Took()
}
// staleAgainst is the declaration a report may be judged stale against, and it is empty for a
// report that is not only an account of an apply.
//
// **A report carries two different things, and only one of them is about a declaration.** What the
// node applied is; what the machine *is* — the tunnel it took over, the ports its own bundle holds,
// what an adopted node found and is keeping, a node moving its overlay key — is not. Those reach
// the mesh on a report because a report is the message a node sends, and nowhere else: a rekey
// dropped as stale is a node whose overlay key never moves, and no retry is coming, because the
// node said it once.
//
// So staleness is asked only of a report that is purely an apply's account. The rest is acted on
// whenever it arrives, which is the behaviour the mesh has had all along.
func staleAgainst(report Report) string {
if report.Rekey != nil || report.Tunnel != nil || len(report.Held) > 0 ||
report.Firewall != "" || len(report.Reachable) > 0 || len(report.Carried) > 0 ||
len(report.Outward) > 0 {
return ""
}
return report.Declared
}
func (s *Server) enrolling(ctx context.Context, m Control) {
reply := EnrolReply{Refusal: "that token cannot be used"}
internal/link/signer.go
+12
View File
@@ -0,0 +1,12 @@
package link
import "context"
// Signer is whatever holds the control plane's signing key.
type Signer interface {
Sign(ctx context.Context, message []byte) ([]byte, error)
}
// ActingGate is what every act passes before it is made (novox/hq to-be 45 §6): whether this process may act under the
// controller's lease. Set by the controller; nil passes every act — a test, a tool.
var ActingGate func(ctx context.Context) error
internal/link/stale_report_test.go
-123
View File
@@ -1,123 +0,0 @@
package link
import (
"context"
"errors"
"testing"
)
// Supersession as a check rather than a memory (design 25 §3).
//
// Holding a message in memory let the controller drop an older report when a newer one for the
// same node arrived. On the bus being built the message belongs to the server and comes back
// whatever happened meanwhile — so the older report is redelivered *after* the newer was applied,
// and acting on it would undo the newer.
//
// The answer was already in the message: a report carries the digest of the declaration it is
// about, so "is this the past?" is a question the message answers.
// sentAndHeard records reports and knows what was last sent, which is the pair the check needs.
type sentAndHeard struct {
sent string
heard []Report
err error
}
func (s *sentAndHeard) Heard(_ context.Context, r Report) (bool, error) {
if s.err != nil {
return false, s.err
}
s.heard = append(s.heard, r)
return true, nil
}
func (s *sentAndHeard) Outstanding(context.Context, string) (string, error) { return s.sent, nil }
// A report about a declaration the mesh has moved past is settled and not acted on. Settled rather
// than dropped, because there is nothing wrong with the message — it is simply the past, and
// redelivering it for ever is worse than letting it go.
func TestAReportAboutASupersededDeclarationIsNotActedOn(t *testing.T) {
store := &sentAndHeard{sent: "d2"}
s, in := serving()
s.listener = store
to := &settled{}
s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1")))
if len(store.heard) != 0 {
t.Fatalf("a report about a superseded declaration was acted on: %+v", store.heard)
}
if !to.acked {
t.Fatalf("a superseded report was not settled, so it comes back for ever: %+v", *to)
}
}
// The report about the declaration that *is* outstanding is acted on, and so is one from a node
// the mesh has no digest for — an older host that says nothing about which declaration it applied
// has nothing to be judged against, and refusing it would silence every node built before reports
// carried the digest.
func TestAReportAboutTheOutstandingDeclarationIsActedOn(t *testing.T) {
for _, c := range []struct{ what, sent, declared string }{
{"the one outstanding", "d2", "d2"},
{"a report that says nothing about which", "d2", ""},
{"a node nothing was ever sent", "", "d1"},
} {
store := &sentAndHeard{sent: c.sent}
s, in := serving()
s.listener = store
to := &settled{}
s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", c.declared)))
if len(store.heard) != 1 || !to.acked {
t.Errorf("%s: was not acted on and acknowledged: heard %+v, settled %+v",
c.what, store.heard, *to)
}
}
}
// **Staleness is decided before the store is waited on**, not after: a redelivery that lost its
// race is not worth holding a slot in the window that a current message needs.
func TestASupersededReportIsNotHeldForTheStore(t *testing.T) {
store := &sentAndHeard{sent: "d2", err: errors.Join(ErrTryAgain, errors.New("starting up"))}
s, in := serving()
s.listener = store
to := &settled{}
s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1")))
if !to.acked || len(in.held) != 0 {
t.Fatalf("a superseded report waited for the store: %+v, %d held", *to, len(in.held))
}
}
// **Half of a report is not about a declaration, and that half is never stale.**
//
// What the machine *is* — the tunnel it took over, the ports its own bundle holds, what an adopted
// node found and is keeping, a node moving its overlay key — reaches the mesh on a report and
// nowhere else. A rekey set aside as stale is a node whose overlay key never moves, and no retry is
// coming, because the node said it once. So a report carrying any of these is acted on whenever it
// arrives, however far the mesh has moved on.
func TestAReportCarryingWhatOnlyTheNodeKnowsIsActedOnHoweverOldItIs(t *testing.T) {
for _, c := range []struct {
what string
report Report
}{
{"a rekey", Report{Node: "anchor", Declared: "d1",
Rekey: &Rekey{Previous: "k1", OverlayKey: "k2"}}},
{"the tunnel it carried", Report{Node: "anchor", Declared: "d1",
Tunnel: &CarriedTunnel{Interface: "wg0", State: "taken"}}},
{"what an adopted node holds", Report{Node: "anchor", Declared: "d1",
Held: []Held{{ID: "conf", Module: "web", Kind: "file"}}}},
{"the firewall it found", Report{Node: "anchor", Declared: "d1", Firewall: "ufw"}},
{"what is reachable on it", Report{Node: "anchor", Declared: "d1",
Reachable: []Reach{{Protocol: "tcp", Port: 443}}}},
{"the ports its own bundle holds", Report{Node: "anchor", Declared: "d1",
Carried: []int{5432}}},
} {
store := &sentAndHeard{sent: "d9"}
s, in := serving()
s.listener = store
to := &settled{}
s.act(context.Background(), in.sends(t, to, KindReport, c.report))
if len(store.heard) != 1 {
t.Errorf("%s was set aside as stale, and the mesh will never hear it again: %+v",
c.what, *to)
}
}
}