From f56458da324dde378f8ff50669c6bf79a12cbdac Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 17:12:35 +0200 Subject: [PATCH] Delete the push and the declaration: the send stub and node holds, link declare and its order, the sent readers, and the sweep judged by compositions (release 6, issue 1945 B8) --- cmd/mesh-controller/acts.go | 12 - cmd/mesh-controller/adoption.go | 16 - cmd/mesh-controller/asker_test.go | 29 -- cmd/mesh-controller/bindings.go | 15 - cmd/mesh-controller/bindings_test.go | 15 - cmd/mesh-controller/compose_input.go | 3 - cmd/mesh-controller/compose_sdk.go | 6 - cmd/mesh-controller/container_groups_test.go | 25 -- .../declaration_files_integration_test.go | 38 +- cmd/mesh-controller/declaration_files_test.go | 48 ++- cmd/mesh-controller/declaration_room.go | 55 --- cmd/mesh-controller/declaration_room_test.go | 71 ---- cmd/mesh-controller/declaration_sweep.go | 94 +++-- cmd/mesh-controller/dependents_test.go | 42 -- cmd/mesh-controller/facts.go | 9 - .../grant_follows_binding_integration_test.go | 5 - cmd/mesh-controller/helpers_kept_test.go | 5 + .../identity_bound_integration_test.go | 5 - cmd/mesh-controller/image_missing.go | 6 +- cmd/mesh-controller/kept_helpers.go | 19 - cmd/mesh-controller/merge_unsaved_test.go | 26 -- cmd/mesh-controller/pending.go | 7 - cmd/mesh-controller/person_wait_test.go | 79 ++++ cmd/mesh-controller/plain_words.go | 67 --- cmd/mesh-controller/plan.go | 95 ----- cmd/mesh-controller/plan_test.go | 50 +-- cmd/mesh-controller/probes.go | 29 +- cmd/mesh-controller/push.go | 198 --------- cmd/mesh-controller/registry_verbs.go | 8 + cmd/mesh-controller/registry_verbs_test.go | 11 - cmd/mesh-controller/send_stub.go | 23 - .../sendable_integration_test.go | 9 - cmd/mesh-controller/signals.go | 23 - cmd/mesh-controller/signals_test.go | 2 +- cmd/mesh-controller/status.go | 21 - cmd/mesh-controller/status_summary.go | 7 - cmd/mesh-controller/unheld_report_test.go | 83 ---- cmd/mesh-controller/unseen.go | 282 ------------- cmd/mesh-controller/unseen_test.go | 175 -------- cmd/mesh-controller/upgrades.go | 392 +----------------- internal/inventory/build_lookups.go | 8 - internal/inventory/nodes.go | 11 - internal/link/bus.go | 18 - internal/link/bus_test.go | 10 - internal/link/declare.go | 94 ----- .../link/declare_room_integration_test.go | 52 --- internal/link/declare_room_test.go | 57 --- internal/link/enrolment.go | 49 --- internal/link/generation_test.go | 48 --- internal/link/heard_order_test.go | 22 - internal/link/order.go | 75 +--- internal/link/order_test.go | 40 +- internal/link/receive.go | 12 - .../link/receive_nats_integration_test.go | 19 - internal/link/serve.go | 60 +-- internal/link/signer.go | 12 + internal/link/stale_report_test.go | 123 ------ 57 files changed, 228 insertions(+), 2587 deletions(-) delete mode 100644 cmd/mesh-controller/declaration_room_test.go delete mode 100644 cmd/mesh-controller/dependents_test.go delete mode 100644 cmd/mesh-controller/merge_unsaved_test.go delete mode 100644 cmd/mesh-controller/send_stub.go delete mode 100644 cmd/mesh-controller/unheld_report_test.go delete mode 100644 cmd/mesh-controller/unseen_test.go delete mode 100644 internal/link/declare.go delete mode 100644 internal/link/declare_room_integration_test.go delete mode 100644 internal/link/declare_room_test.go delete mode 100644 internal/link/generation_test.go create mode 100644 internal/link/signer.go delete mode 100644 internal/link/stale_report_test.go diff --git a/cmd/mesh-controller/acts.go b/cmd/mesh-controller/acts.go index f5c56fd8..3215b1aa 100644 --- a/cmd/mesh-controller/acts.go +++ b/cmd/mesh-controller/acts.go @@ -58,13 +58,6 @@ func assignWith(ctx context.Context, open *stores, node string, opts assignOptio // Every line of the answer comes from the one resolution after the write (novox/hq issue 237): // what the reads before it would say about a machine left out is not the answer's. ctx = leftOutSaidBy(ctx, io.Discard) - // Held while it is recorded, so it cannot land between a converge's preview and its flip and - // be taken without ever having been previewed (novox/hq ADR 0100). - ctx, release, err := holdNodes(ctx, open, []string{node}) - if err != nil { - return "", err - } - defer release() // **A module the catalogue does not hold is looked for further before it is refused** (novox/hq issue // 325): a build in flight keeps the assignment pending, a module known and not built is said with how to // build it, and only a name the mesh never heard of is refused as unknown. Several modules in one act @@ -244,11 +237,6 @@ func unassign(ctx context.Context, open *stores, node string, modules ...string) } // One resolution after the write, as assign's answer (novox/hq issue 237). ctx = leftOutSaidBy(ctx, io.Discard) - ctx, release, err := holdNodes(ctx, open, []string{node}) - if err != nil { - return "", err - } - defer release() shelf, err := open.inventory.Catalogue(ctx) if err != nil { return "", err diff --git a/cmd/mesh-controller/adoption.go b/cmd/mesh-controller/adoption.go index ad5c3b3a..ad679ea5 100644 --- a/cmd/mesh-controller/adoption.go +++ b/cmd/mesh-controller/adoption.go @@ -608,16 +608,6 @@ func converge(ctx context.Context, open *stores, node string, yes bool, digest s if filter == "" { filter = DefaultFilter } - if yes { - // Held from the checks to the send, so no push composed before the flip is sent after it - // and returns the node to adopted. - held, release, err := holdNodes(ctx, open, []string{node}) - if err != nil { - return "", err - } - defer release() - ctx = held - } record, err := inv.NodeByName(ctx, node) if err != nil { return "", err @@ -1031,12 +1021,6 @@ func adopt(ctx context.Context, open *stores, node string) (string, error) { if record.Adopted { return "", fmt.Errorf("%s is adopted already", node) } - held, release, err := holdNodes(ctx, open, []string{node}) - if err != nil { - return "", err - } - defer release() - ctx = held if err := inv.SetAdopted(ctx, node, true); err != nil { return "", err } diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go index 3acbec19..4e732bee 100644 --- a/cmd/mesh-controller/asker_test.go +++ b/cmd/mesh-controller/asker_test.go @@ -306,35 +306,6 @@ func TestAWarrantThatIsNotForItsOwnAskIsRefused(t *testing.T) { } } -func TestEachAnswerCallsExactlyItsVerb(t *testing.T) { - plan := "plan-1791454185265004861" - waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent, - Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.", - Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)} - module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning, - Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.", - Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart", - Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove, - Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}} - for _, tc := range []struct { - c conditions.Condition - label string - want string - }{ - {waiting, "Start", `mesh-controller.plans@ {"cause":"operator-answer","go":"` + plan + `","why":"`}, - {waiting, "Stop", `mesh-controller.plans@ {"cause":"operator-answer","stop":"` + plan + `","why":"`}, - {module, "Restart", `node-service-manager.restart@g14 {"scope":"user","unit":"openrazer-daemon.service"}`}, - } { - r := newAskerRig(t) - r.open = []conditions.Condition{tc.c} - _ = r.a.reconcile(context.Background()) - answerWith(t, r, r.warrantFor(t, tc.c.Key, tc.label)) - if len(r.called) != 1 || !strings.HasPrefix(r.called[0], tc.want) { - t.Errorf("%s: called %v, want %s…", tc.label, r.called, tc.want) - } - } -} - func TestASilenceChosenIsTheControllersOwnAndAnAnswerToAnAsk(t *testing.T) { r := newAskerRig(t) r.open = []conditions.Condition{unitsCondition()} diff --git a/cmd/mesh-controller/bindings.go b/cmd/mesh-controller/bindings.go index aab111f1..64563d50 100644 --- a/cmd/mesh-controller/bindings.go +++ b/cmd/mesh-controller/bindings.go @@ -54,21 +54,6 @@ func otherMachines(machine string, nodes ...string) []string { return out } -// reportKept says every move a machine's resolution refused, on the push composing it, and raises its -// condition at once where this process keeps the conditions: a push is when a person is looking. -func reportKept(ctx context.Context, plan catalogue.Resolution) { - for _, k := range plan.Kept { - fmt.Printf("%s: the mesh %s\n", plan.Node, k) - if conditionsFrom != nil { - o := keptObservation(k) - o.Source = probeBindingsID - if _, err := conditionsFrom.Observe(ctx, o); err != nil { - fmt.Printf("%s: and the condition for it could not be raised: %v\n", plan.Node, err) - } - } - } -} - // probeBindings is D12: every consumer of a provision that keeps its data is bound where it was last // sent, on every machine — the resolver kept it there (said, urgent, until a person pins), or a pin // moves it (said, so the data goes first), and never anything else. diff --git a/cmd/mesh-controller/bindings_test.go b/cmd/mesh-controller/bindings_test.go index 2101e614..7467d485 100644 --- a/cmd/mesh-controller/bindings_test.go +++ b/cmd/mesh-controller/bindings_test.go @@ -76,18 +76,3 @@ func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) { } } -// A push says the move it refused, and raises its condition at once. -func TestAPushSaysAKeptMoveAndRaisesItsCondition(t *testing.T) { - k, _ := withConditionsInMemory(t) - reportKept(t.Context(), catalogue.Resolution{Node: "laptop", Kept: []catalogue.KeptBinding{{Machine: "laptop", - Consumer: "board", Provision: "postgres-database", Bound: catalogue.Chosen{Node: "home", Module: "store"}, - Would: catalogue.Chosen{Node: "anchor", Module: "store"}}}}) - open, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - if len(open) != 1 || open[0].Kind != kindBindingKept || open[0].Severity != conditions.Urgent || - open[0].Source != probeBindingsID { - t.Fatalf("raised %+v", open) - } -} diff --git a/cmd/mesh-controller/compose_input.go b/cmd/mesh-controller/compose_input.go index dd5b22ae..966a4316 100644 --- a/cmd/mesh-controller/compose_input.go +++ b/cmd/mesh-controller/compose_input.go @@ -250,9 +250,6 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento } // Every node-engine of release 6 reads a module per resource and a build per module. out.ReadsModules = true - if out.SentBuilds, out.SentKnown, err = inv.SentBuilds(ctx, n.Name); err != nil { - return out, err - } stated, has, err := inv.HealthOf(ctx, n.Name) if err != nil { return out, err diff --git a/cmd/mesh-controller/compose_sdk.go b/cmd/mesh-controller/compose_sdk.go index 82edab7d..408ae511 100644 --- a/cmd/mesh-controller/compose_sdk.go +++ b/cmd/mesh-controller/compose_sdk.go @@ -39,12 +39,6 @@ func withInputCache(ctx context.Context) context.Context { return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}}) } -// freshInput is ctx for one composition whose Input is read afresh and kept only until the composition makes something: a -// send that is composed after another one made and recorded things (a push's machines, one after the other). -func freshInput(ctx context.Context) context.Context { - return context.WithValue(ctx, inputCacheKey{}, &inputCache{by: map[string]*meshInput{}}) -} - // keptSignature is what the Input read depends on in ctx: which recorded modules each machine keeps at the build it runs // (recorded_kept.go), so an Input read for a person's push is never taken for one read for a gated send. func keptSignature(ctx context.Context) string { diff --git a/cmd/mesh-controller/container_groups_test.go b/cmd/mesh-controller/container_groups_test.go index 6fd9f9ba..7b154fc7 100644 --- a/cmd/mesh-controller/container_groups_test.go +++ b/cmd/mesh-controller/container_groups_test.go @@ -1,9 +1,6 @@ package main import ( - "io" - "os" - "strings" "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -18,22 +15,6 @@ const n8nWithAGroup = `{"module":"n8n","version":"1", "image":"n8nio/n8n@sha256:4846eb2f4b874ab04cde7fc1e249d2ddaec66e9aea64439beb2972cfea88e3c0", "groups":["${config:media-gid}"]}]}` -// printedBy runs reportUnhostable and returns what it printed. -func printedBy(t *testing.T, node string, plan catalogue.Resolution) string { - t.Helper() - stdout := os.Stdout - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - os.Stdout = w - reportUnhostable(node, plan) - w.Close() - os.Stdout = stdout - out, _ := io.ReadAll(r) - return string(out) -} - // 1638/AC1: a node that reports the capability gets the module; one that does not has it held, the capability named. func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing.T) { m, err := catalogue.ParseManifest([]byte(n8nWithAGroup)) @@ -48,9 +29,6 @@ func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing if err != nil || len(got.Unhostable) != 0 || len(got.Modules) != 1 { t.Fatalf("a node that reads groups: %v, held %+v", err, got.Unhostable) } - if out := printedBy(t, "ace", got); out != "" { - t.Errorf("a node that reads groups was told: %q", out) - } got, err = catalogue.Resolve(shelf, []string{"n8n"}, old, catalogue.World{}) if err != nil { @@ -59,7 +37,4 @@ func TestAModuleWithGroupsIsHeldOnANodeThatDoesNotReportTheCapability(t *testing if len(got.Modules) != 0 || len(got.Unhostable) != 1 { t.Fatalf("n8n was sent to a node that does not report container-groups: hosted %d, held %+v", len(got.Modules), got.Unhostable) } - if out := printedBy(t, "g14", got); !strings.Contains(out, "n8n") || !strings.Contains(out, "container-groups") { - t.Errorf("the hold does not name the module and the capability: %q", out) - } } diff --git a/cmd/mesh-controller/declaration_files_integration_test.go b/cmd/mesh-controller/declaration_files_integration_test.go index b554e115..82bdad8d 100644 --- a/cmd/mesh-controller/declaration_files_integration_test.go +++ b/cmd/mesh-controller/declaration_files_integration_test.go @@ -15,7 +15,6 @@ import ( "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/testbus" ) @@ -34,14 +33,6 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) t.Fatal(err) } defer conn.Close() - nc, err := conn.JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := nc.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"}, - MaxMsgsPerSubject: 1}); err != nil { - t.Fatal(err) - } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() api, err := jetstream.New(conn) @@ -61,19 +52,8 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) resources = append(resources, aFileResource("big-"+string(rune('a'+i)), contentOf("big"+string(rune('a'+i)), 1200<<10))) } resources = append(resources, aFileResource("small", "tiny\n")) - inline := sendable{Resources: resources} byDigest := sendable{fileReferences: true, Resources: resources} - bus := link.OverNATS{Conn: conn, JS: nc} - body, err := inline.Body() - if err != nil { - t.Fatal(err) - } - if err := link.Declare(ctx, bus, sigOf{}, "novox", body, 5*time.Second); err == nil || - !strings.Contains(err.Error(), "1048576") || strings.Contains(err.Error(), "maximum payload exceeded") { - t.Fatalf("the inline declaration of more than 3 MiB: %v", err) - } - bucket, err := openBucket(ctx, conn, "novox") if err != nil { t.Fatal(err) @@ -92,18 +72,14 @@ func TestADeclarationOfMoreThanThreeMiBIsSentByReferenceOnARealBus(t *testing.T) if err != nil { t.Fatal(err) } - if err := link.Declare(ctx, bus, sigOf{}, "novox", refBody, 5*time.Second); err != nil { - t.Fatalf("the declaration by reference: %v", err) + // What the body names is exactly the three objects. + namedSizes, err := namedObjects(refBody) + if err != nil || len(namedSizes) != 3 { + t.Fatalf("what the body names: %v %v", namedSizes, err) } - - // What the stream holds for the node is the reference form, and it names exactly the three objects. - latest, err := latestDeclarations(ctx, conn, []string{"novox", "ace"}) - if err != nil || len(latest) != 1 { - t.Fatalf("the latest declarations: %v %v", len(latest), err) - } - named, err := namedByLatest("novox", latest["novox"]) - if err != nil || len(named) != 3 { - t.Fatalf("what the latest declaration names: %v %v", named, err) + named := map[string]bool{} + for name := range namedSizes { + named[name] = true } novoxBucket, err := broker.DeclarationFilesBucketOf("novox") if err != nil { diff --git a/cmd/mesh-controller/declaration_files_test.go b/cmd/mesh-controller/declaration_files_test.go index 3577c6db..e96083ce 100644 --- a/cmd/mesh-controller/declaration_files_test.go +++ b/cmd/mesh-controller/declaration_files_test.go @@ -12,6 +12,7 @@ import ( "time" sdkdecl "git.novox.be/novox/mesh-sdk/go/declaration" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" @@ -55,7 +56,7 @@ func wireSizeOf(t *testing.T, s sendable) int { t.Fatal(err) } // The signature the controller's key makes is 64 bytes. - msg, err := link.Envelope(body, make([]byte, 64)) + msg, err := json.Marshal(link.Signed{Declaration: body, Signature: make([]byte, 64)}) if err != nil { t.Fatal(err) } @@ -306,22 +307,43 @@ func TestAnObjectIsRemovedADayAfterNoLatestDeclarationNamesIt(t *testing.T) { // } } -// What a node's latest declaration names is read from the signed envelope as stored on the bus; an unreadable one is an error and -// never "names none", which would mark an object a node needs. -func TestTheObjectsALatestDeclarationNamesAreReadFromTheBus(t *testing.T) { - s := sendable{fileReferences: true, Resources: []map[string]any{aFileResource("a", contentOf("a", 70<<10))}} - body, _ := s.Body() - msg, _ := link.Envelope(body, []byte("sig")) - named, err := namedByLatest("novox", msg) - name, _ := sdkdecl.ObjectName(sdkdecl.DigestOf([]byte(contentOf("a", 70<<10)))) +// What a node's compositions name is read from the values of its bucket `composed-`: the files each carries by digest +// (ADR 0539 §4). An unreadable one is an error and never "names none", which would mark an object a node needs. +func TestTheObjectsACompositionNamesAreReadFromItsValue(t *testing.T) { + content := contentOf("a", 70<<10) + digest := sdkdecl.DigestOf([]byte(content)) + ref, _ := json.Marshal(map[string]any{"id": "m.f", "type": "file", "module": "m", "path": "/etc/m", "digest": digest, + "size": len(content)}) + small := json.RawMessage(`{"id":"m.g","type":"file","module":"m","path":"/etc/g","content":"x"}`) + fp := sdkdelivery.SourcePrefix2 + strings.Repeat("a", 64) + value, err := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "novox", Module: "m", FP: fp, + Resources: []json.RawMessage{ref, small}}) + if err != nil { + t.Fatal(err) + } + named, err := namedByCompositions("novox", [][]byte{value}) + name, _ := sdkdecl.ObjectName(digest) if err != nil || len(named) != 1 || !named[name] { t.Fatalf("%v %v", named, err) } - if none, err := namedByLatest("ace", []byte(`{"declaration":"e30=","signature":""}`)); err != nil || len(none) != 0 { - t.Fatalf("a declaration naming none: %v %v", none, err) + none, _ := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "ace", Module: "m", FP: fp, Resources: []json.RawMessage{small}}) + if got, err := namedByCompositions("ace", [][]byte{none}); err != nil || len(got) != 0 { + t.Fatalf("a composition naming none: %v %v", got, err) } - if _, err := namedByLatest("novox", []byte("not json")); err == nil { - t.Fatal("an unreadable latest declaration was read as naming none") + if got, err := namedByCompositions("ace", nil); err != nil || len(got) != 0 { + t.Fatalf("a node with no composition: %v %v", got, err) + } + if _, err := namedByCompositions("novox", [][]byte{[]byte("not json")}); err == nil { + t.Fatal("an unreadable composition was read as naming none") + } + // A node's objects are those of every composition it keeps: the five newest fingerprints of a module name their own. + other := sdkdecl.DigestOf([]byte(contentOf("b", 80<<10))) + ref2, _ := json.Marshal(map[string]any{"id": "m.f", "type": "file", "module": "m", "path": "/etc/m", "digest": other, "size": 80 << 10}) + older, _ := sdkdelivery.EncodeComposed(sdkdelivery.Composed{Node: "novox", Module: "m", FP: sdkdelivery.SourcePrefix2 + strings.Repeat("b", 64), + Resources: []json.RawMessage{ref2}}) + both, err := namedByCompositions("novox", [][]byte{value, older}) + if err != nil || len(both) != 2 { + t.Fatalf("two fingerprints of one module name %v (%v), want both objects", both, err) } } diff --git a/cmd/mesh-controller/declaration_room.go b/cmd/mesh-controller/declaration_room.go index 2eb0b6df..d7b5b546 100644 --- a/cmd/mesh-controller/declaration_room.go +++ b/cmd/mesh-controller/declaration_room.go @@ -2,65 +2,10 @@ package main import ( "bytes" - "context" "encoding/json" - "fmt" "strings" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" ) -// **A declaration is one bus message, and the bus's largest message is a cliff** (novox/hq issue 1689). On 2026-10-21 -// the control node's declaration passed it by 2 KB and nothing reached that node until two assignments were undone. -// So the size is measured at every send: a warning condition from 75% of the limit (ADR 0486 §8, issue 1698: it was 90% while every file travelled inline), urgent over it, cleared when -// the declaration is back under 75%. The limit is the connection's own, not a constant. - -// kindDeclarationNearLimit is a declaration at or near the bus's largest message. -const kindDeclarationNearLimit = "declaration-near-limit" - -// declarationRoomSource is the condition's source: the sends. -const declarationRoomSource = "sends" - -func declarationRoomObservation(node string, size, limit int64) conditions.Observation { - o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindDeclarationNearLimit, - Token: kindDeclarationNearLimit, Machine: node, Severity: conditions.Warning, Source: declarationRoomSource} - if size > limit { - o.Severity = conditions.Urgent - } - o.Summary = fmt.Sprintf("%s's declaration is %d bytes on the wire, %d%% of the bus's largest message of %d bytes", - node, size, size*100/limit, limit) - if size > limit { - o.Summary += ": over it, so it cannot be sent" - } - return o -} - -func init() { - link.DeclarationMeasured = func(ctx context.Context, node string, size, limit int64) { - settleDeclarationRoom(ctx, conditionsFrom, node, size, limit) - } -} - -// settleDeclarationRoom raises or clears the condition of a declaration's size. A nil keeper (a tool, not the -// serving controller) says nothing. -func settleDeclarationRoom(ctx context.Context, keeper *conditions.Keeper, node string, size, limit int64) { - if keeper == nil || limit <= 0 { - return - } - obs := declarationRoomObservation(node, size, limit) - if float64(size) < link.NearTheLimit*float64(limit) { - if _, err := keeper.Clear(ctx, obs.Key(), fmt.Sprintf("%s's declaration is %d bytes, under the limit's margin", node, size)); err != nil { - fmt.Printf("the condition %s could not be cleared: %v\n", obs.Key(), err) - } - return - } - fmt.Printf("NEAR THE LIMIT: %s\n", obs.Summary) - if _, err := keeper.Observe(ctx, obs); err != nil { - fmt.Printf("the condition %s could not be raised: %v\n", obs.Key(), err) - } -} - // compactKept writes the vault's export compact. The composition (the SDK's keptFile) writes it indented, and its 334 // sealed secrets were 164,763 bytes of it against 138,474 compact: a fifth of the declaration's room, in whitespace, // base64-encoded again on the wire. Every reader parses JSON, so the spacing is nobody's. A resource that is not diff --git a/cmd/mesh-controller/declaration_room_test.go b/cmd/mesh-controller/declaration_room_test.go deleted file mode 100644 index a52b547f..00000000 --- a/cmd/mesh-controller/declaration_room_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" -) - -// The vault's export is sent compact (issue 1689): same document, no indentation, one rewrite. -func TestTheVaultsExportIsSentCompactAndNothingElseIsTouched(t *testing.T) { - indented := "{\n \"export\": 1,\n \"kept\": [\n {\n \"name\": \"a\"\n }\n ]\n}\n" - other := "{\n \"x\": 1\n}\n" - in := []map[string]any{ - {"id": "mesh-vault.kept", "type": "file", "path": "/srv/vault/export.json", "mode": "0600", "content": indented}, - {"id": "mesh-vault.config", "type": "file", "path": "/srv/vault/config.json", "content": other}, - {"id": "other.kept", "type": "file", "path": "/srv/other/notes.json", "content": other}, - } - out := compactKept(in) - want := "{\"export\":1,\"kept\":[{\"name\":\"a\"}]}\n" - if out[0]["content"] != want || out[0]["mode"] != "0600" || out[0]["path"] != "/srv/vault/export.json" { - t.Fatalf("the export is %q (%v), expected %q with its mode and path kept", out[0]["content"], out[0], want) - } - if out[1]["content"] != other || out[2]["content"] != other { - t.Fatalf("a file that is not the export was rewritten: %v %v", out[1], out[2]) - } - if in[0]["content"] != indented { - t.Fatalf("the composed resource was changed in place") - } - if again := compactKept(out); again[0]["content"] != want { - t.Fatalf("compacting twice changed it: %q", again[0]["content"]) - } -} - -// From 75% of the bus's largest message (ADR 0486 §8, issue 1698) the declaration is a warning; over it, urgent; back under 75%, cleared. -func TestADeclarationNearTheBusLimitRaisesAConditionAndClearsWhenSmaller(t *testing.T) { - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, - Teller: &conditions.Told{}, Now: func() time.Time { return time.Date(2026, 10, 21, 14, 0, 0, 0, time.UTC) }}) - t.Cleanup(func() { k.Close(t.Context()) }) - const limit = 1000 - key := declarationRoomObservation("novox", 0, limit).Key() - read := func() (conditions.Condition, bool) { - t.Helper() - c, found, err := conditions.ReadOne(t.Context(), store, key) - if err != nil { - t.Fatal(err) - } - return c, found - } - - settleDeclarationRoom(t.Context(), k, "novox", 749, limit) - if _, found := read(); found { - t.Fatal("a declaration at 74.9% raised a condition") - } - settleDeclarationRoom(t.Context(), k, "novox", 750, limit) - c, found := read() - if !found || c.Severity != conditions.Warning || !strings.Contains(c.Summary, "750 bytes") || - !strings.Contains(c.Summary, "1000") || c.Subject.Machine != "novox" { - t.Fatalf("at 75%% the condition is %v %+v", found, c) - } - settleDeclarationRoom(t.Context(), k, "novox", 1001, limit) - if c, _ := read(); c.Severity != conditions.Urgent { - t.Fatalf("over the limit the condition is %q, expected urgent", c.Severity) - } - settleDeclarationRoom(t.Context(), k, "novox", 700, limit) - if _, found := read(); found { - t.Fatal("the condition stayed after the declaration fell under 75%") - } -} diff --git a/cmd/mesh-controller/declaration_sweep.go b/cmd/mesh-controller/declaration_sweep.go index 33bc2a79..a9b710a8 100644 --- a/cmd/mesh-controller/declaration_sweep.go +++ b/cmd/mesh-controller/declaration_sweep.go @@ -9,23 +9,22 @@ import ( "strings" "time" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) -// **An object is removed a day after no node's latest declaration names it** (novox/hq ADR 0486 §3, issue 1696), so a node that is -// asleep and receives its latest declaration later still finds every object that declaration names. What each node's latest -// declaration names is read from the bus itself, the last message of that node's subject of the stream NODES: nothing else -// is kept of it. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed); +// **An object is removed a day after no composition of the node names it** (novox/hq ADR 0486 §3, ADR 0539 §4, issue 1696), so a +// node that is asleep and applies its wants later still finds every object they name. What a node's compositions name is read +// from its bucket `composed-`, where the controller keeps each module's five newest fingerprints and the node's own: nothing +// else is kept of them. When an object is first found named by none, the time is written in its own metadata (declarationUnnamed); // found named again, that is erased; found unnamed for a day, it is removed. Each node has a bucket of its own, judged against that -// node's latest declaration alone. Objects are immutable and regenerable from the -// controller's state, so a removal loses nothing. +// node's compositions alone. Objects are immutable and regenerable from the controller's state, so a removal loses nothing. -// declarationUnnamed is the key of an object's metadata holding when it was first found named by no node's latest declaration. +// declarationUnnamed is the key of an object's metadata holding when it was first found named by no composition of the node. const declarationUnnamed = "unnamed-since" // sweepEvery is how often the bucket is judged: the removal is exact to within this. @@ -92,41 +91,62 @@ func sweepDeclarationFiles(ctx context.Context, bucket sweptBucket, named map[st return r, nil } -// namedByLatest is the name of every object this latest declaration of a node names. A body that cannot be read is an error, never -// "names none": an object the node still needs would be marked for removal. -func namedByLatest(node string, signed []byte) (map[string]bool, error) { - var s link.Signed - if err := json.Unmarshal(signed, &s); err != nil { - return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err) - } - names, err := namedObjects(s.Declaration) - if err != nil { - return nil, fmt.Errorf("%s's latest declaration is not readable: %w", node, err) - } - named := make(map[string]bool, len(names)) - for name := range names { - named[name] = true +// namedByCompositions is the name of every object the compositions of a node name: the files over 64 KiB each carries by +// digest (ADR 0539 §4). A composition that cannot be read is an error, never "names none": an object the node still needs +// would be marked for removal. +func namedByCompositions(node string, values [][]byte) (map[string]bool, error) { + named := map[string]bool{} + for _, value := range values { + c, err := sdkdelivery.DecodeComposed(value) + if err != nil { + return nil, fmt.Errorf("a composition of %s is not readable: %w", node, err) + } + body, err := json.Marshal(map[string]any{"resources": c.Resources}) + if err != nil { + return nil, err + } + names, err := namedObjects(body) + if err != nil { + return nil, fmt.Errorf("a composition of %s is not readable: %w", node, err) + } + for name := range names { + named[name] = true + } } return named, nil } -// latestDeclarations are the last signed declarations of the nodes, by node, as the stream NODES holds them. A node with none has -// no entry. -func latestDeclarations(ctx context.Context, conn *nats.Conn, nodes []string) (map[string][]byte, error) { - js, err := conn.JetStream(nats.Context(ctx)) +// keptCompositionsOf are the values of every composition in a node's bucket `composed-`: what the controller keeps there +// are each module's five newest fingerprints and the node's own (ADR 0539 §4). A node with no bucket has none. +func keptCompositionsOf(ctx context.Context, js jetstream.JetStream, node string) ([][]byte, error) { + bucket, err := sdkdelivery.ComposedBucketFor(node) if err != nil { return nil, err } - out := map[string][]byte{} - for _, node := range nodes { - msg, err := js.GetLastMsg("NODES", link.DeclareSubject(node), nats.Context(ctx)) - if errors.Is(err, nats.ErrMsgNotFound) { + kv, err := js.KeyValue(ctx, bucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("the compositions of %s: %w", node, err) + } + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("the compositions of %s: %w", node, err) + } + var out [][]byte + for _, key := range keys { + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { continue } if err != nil { - return nil, fmt.Errorf("the latest declaration of %s: %w", node, err) + return nil, fmt.Errorf("the composition %s: %w", key, err) } - out[node] = msg.Data + out = append(out, e.Value()) } return out, nil } @@ -259,15 +279,13 @@ func sweepNode(ctx context.Context, conn *nats.Conn, node string) (filesSweep, e return filesSweep{}, fmt.Errorf("the bucket cannot be read: %w", err) } defer lockFilesOf(node)() - latest, err := latestDeclarations(working, conn, []string{node}) + values, err := keptCompositionsOf(working, js, node) if err != nil { return filesSweep{}, err } - named := map[string]bool{} - if body, ok := latest[node]; ok { - if named, err = namedByLatest(node, body); err != nil { - return filesSweep{}, err - } + named, err := namedByCompositions(node, values) + if err != nil { + return filesSweep{}, err } return sweepDeclarationFiles(working, busSweptBucket{store}, named, time.Now()) } diff --git a/cmd/mesh-controller/dependents_test.go b/cmd/mesh-controller/dependents_test.go deleted file mode 100644 index bc9386a9..00000000 --- a/cmd/mesh-controller/dependents_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package main - -import ( - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A merge that rebuilds a base rebuilds what stands on it, through every layer, and nothing else -// (novox/hq issue 186): the runtime image moving means every module built on it moves too, and a -// module built on one of those moves as well. -func TestAMergeOfABaseTakesWhatStandsOnItAlong(t *testing.T) { - entry := func(name string) inventory.Entry { - return inventory.Entry{Manifest: catalogue.Manifest{Module: name}} - } - entries := []inventory.Entry{entry("mesh-tools"), entry("shop"), entry("shop-plugin"), entry("postgres"), entry("unrelated")} - against := map[string][]string{ - "shop": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"}, - "shop-plugin": {catalogue.ArtifactStoreScheme + "shop/runtime@sha256:b"}, - "postgres": {catalogue.ArtifactStoreScheme + "mesh-tools/runtime@sha256:a"}, - "unrelated": {catalogue.ArtifactStoreScheme + "alpine/base@sha256:c"}, - } - got := dependentsOf([]inventory.Entry{entry("mesh-tools")}, entries, against) - var names []string - for _, e := range got { - names = append(names, e.Manifest.Module) - } - want := map[string]bool{"shop": true, "shop-plugin": true, "postgres": true} - if len(names) != len(want) { - t.Fatalf("rebuilt %v; wanted exactly the three that stand on the runtime, directly or through shop", names) - } - for _, n := range names { - if !want[n] { - t.Fatalf("%s was rebuilt and stands on nothing that moved (%v)", n, names) - } - } - // Nothing moved: nothing follows. - if more := dependentsOf(nil, entries, against); len(more) != 0 { - t.Fatalf("with nothing moved, %d module(s) were rebuilt", len(more)) - } -} diff --git a/cmd/mesh-controller/facts.go b/cmd/mesh-controller/facts.go index 4cb07134..d1985eac 100644 --- a/cmd/mesh-controller/facts.go +++ b/cmd/mesh-controller/facts.go @@ -8,7 +8,6 @@ import ( "flag" "fmt" "os" - "slices" "sort" "strings" "sync" @@ -18,7 +17,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/compose" snapshot "git.novox.be/novox/mesh-sdk/go/facts" "github.com/novox/mesh-controller/internal/artifacts" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -315,13 +313,6 @@ func gatherFacts(ctx context.Context, open *stores, busVersion string) (snapshot return snapshot.Facts{}, err } m.Assigned = assigned - sent, known, err := inv.SentBuilds(ctx, n.Name) - if err != nil { - return snapshot.Facts{}, err - } - if known && slices.Contains(assigned, broker.RuntimeModule) { - m.NodeTools = sent[broker.RuntimeModule] - } pins, err := inv.PinsFor(ctx, n.Name) if err != nil { return snapshot.Facts{}, err diff --git a/cmd/mesh-controller/grant_follows_binding_integration_test.go b/cmd/mesh-controller/grant_follows_binding_integration_test.go index 0aed7741..f8441f2a 100644 --- a/cmd/mesh-controller/grant_follows_binding_integration_test.go +++ b/cmd/mesh-controller/grant_follows_binding_integration_test.go @@ -105,11 +105,6 @@ func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) { if got := declared.Received["store"]["postgres-database"]; len(got) != 0 { t.Fatalf("the anchor's store is still told about %+v", got) } - said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) - if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") || - !strings.Contains(said, "cleanup delete") { - t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said) - } // The store it is bound to: granted. grants, _, unbound, err = grantsOf(ctx, open, "laptop") diff --git a/cmd/mesh-controller/helpers_kept_test.go b/cmd/mesh-controller/helpers_kept_test.go index ac44a2ea..0efe6a8b 100644 --- a/cmd/mesh-controller/helpers_kept_test.go +++ b/cmd/mesh-controller/helpers_kept_test.go @@ -43,3 +43,8 @@ func claimingTheSeat(module string) inventory.Entry { return inventory.Entry{Manifest: catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}} } + +func aContainer(name string) catalogue.Manifest { + return catalogue.Manifest{Module: name, Version: "1", + Resources: []map[string]any{{"id": name, "type": "container", "image": name}}} +} diff --git a/cmd/mesh-controller/identity_bound_integration_test.go b/cmd/mesh-controller/identity_bound_integration_test.go index 01774c33..695cb459 100644 --- a/cmd/mesh-controller/identity_bound_integration_test.go +++ b/cmd/mesh-controller/identity_bound_integration_test.go @@ -71,11 +71,6 @@ func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) { if strings.Join(granted, ",") != "files" { t.Fatalf("the object store grants %v; files and only files fit", granted) } - said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) - if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) || - !strings.Contains(said, "left out of anchor's grants") { - t.Fatalf("the push does not say whom it leaves out:\n%s", said) - } // And `status` names it, and does not call the mesh well while it stands. asked, err := theThreeQuestions(ctx, open) diff --git a/cmd/mesh-controller/image_missing.go b/cmd/mesh-controller/image_missing.go index 181314da..d3c78240 100644 --- a/cmd/mesh-controller/image_missing.go +++ b/cmd/mesh-controller/image_missing.go @@ -126,12 +126,8 @@ func withoutMissingImages(ctx context.Context, open *stores, node string, s send if err != nil || address == "" { return s } - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil || !known { - sent = nil - } store := artifacts.Store{Address: address} - found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, sent) + found := holdBackMissingImages(ctx, node, &s, address, store.HoldsManifest, nil) for _, m := range found { fmt.Printf("%s: %s is left out of the declaration: its image %s is not in the artifact store\n", node, m.Module, m.Image) } diff --git a/cmd/mesh-controller/kept_helpers.go b/cmd/mesh-controller/kept_helpers.go index 46e747a5..c4291b15 100644 --- a/cmd/mesh-controller/kept_helpers.go +++ b/cmd/mesh-controller/kept_helpers.go @@ -13,7 +13,6 @@ import ( "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" "regexp" - "strconv" "strings" "time" ) @@ -232,21 +231,3 @@ func refusedWord(failed string) string { // unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which // the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words). var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`) - -// durationSetting reads a duration as a setting says it: "90s", "10m", or a number of seconds. -func durationSetting(v any) (time.Duration, bool) { - switch t := v.(type) { - case string: - if d, err := time.ParseDuration(strings.TrimSpace(t)); err == nil && d >= 0 { - return d, true - } - if n, err := strconv.ParseFloat(strings.TrimSpace(t), 64); err == nil && n >= 0 { - return time.Duration(n * float64(time.Second)), true - } - case float64: - if t >= 0 { - return time.Duration(t * float64(time.Second)), true - } - } - return 0, false -} diff --git a/cmd/mesh-controller/merge_unsaved_test.go b/cmd/mesh-controller/merge_unsaved_test.go deleted file mode 100644 index c967060f..00000000 --- a/cmd/mesh-controller/merge_unsaved_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -import ( - "errors" - "testing" - - "github.com/novox/mesh-controller/internal/link" -) - -// **A merge whose plan could not be saved is held, never taken** (novox/hq ADR 0407 decision 3, issue 1092): the merge is -// carried until its plan is saved, and the bus acknowledges the announcement only when the controller answers without -// ErrTryAgain (link.Server.sourceMoved), so a failed save of any kind must answer it. -func TestAFailureToSaveAMergesPlanIsHeldAndNotTaken(t *testing.T) { - cause := errors.New("the plan for novox/mesh-catalog is not written: the plan was written by somebody else since it was read") - held := unsaved(cause) - if !errors.Is(held, link.ErrTryAgain) || !errors.Is(held, cause) { - t.Fatalf("a failed save answers %v", held) - } - if unsaved(nil) != nil { - t.Fatal("a save that worked answers an error") - } - // Not the same as notNow, which holds a merge only for a store that is away. - if errors.Is(notNow(cause), link.ErrTryAgain) { - t.Fatal("notNow holds every failure; the premise of this test is gone") - } -} diff --git a/cmd/mesh-controller/pending.go b/cmd/mesh-controller/pending.go index 93757976..1bc0b7ac 100644 --- a/cmd/mesh-controller/pending.go +++ b/cmd/mesh-controller/pending.go @@ -411,13 +411,6 @@ func editDistance(a, b string) int { // refusal. Returns what it said, or nothing when the row no longer waited. func applyPending(ctx context.Context, open *stores, p inventory.PendingAssignment, by string) string { inv := open.inventory - ctx, release, err := holdNodes(ctx, open, []string{p.Node}) - if err != nil { - // Left waiting: the next tick tries again. - return fmt.Sprintf("the pending assignment of %s to %s waits: %s could not be held: %v", p.Module, p.Node, - p.Node, err) - } - defer release() claimed, err := inv.ClaimPending(ctx, p.ID) if err != nil { return fmt.Sprintf("the pending assignment of %s to %s could not be taken for making: %v", p.Module, p.Node, err) diff --git a/cmd/mesh-controller/person_wait_test.go b/cmd/mesh-controller/person_wait_test.go index 7a3d3073..2355abfd 100644 --- a/cmd/mesh-controller/person_wait_test.go +++ b/cmd/mesh-controller/person_wait_test.go @@ -1,6 +1,10 @@ package main import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -19,3 +23,78 @@ func userUnit(module, account string) inventory.ResourceHealth { Target: module + "-daemon.service", Account: account, State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} } + +// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as +// before, so that a fault is never excused. +func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) { + notInGroup := relogin("lights", "operator") + notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why" + systemUnit := userUnit("lights", "operator") + systemUnit.Account = "" + otherAccount := userUnit("lights", "guest") + container := inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights", + State: link.StateUnhealthy, Reason: "down"} + otherModule := userUnit("sound", "operator") + starting := userUnit("lights", "operator") + starting.State = link.StateStarting + byTarget := relogin("lights", "operator") + byTarget.Account = "" // an account said by its target alone is still that account + + for _, c := range []struct { + name string + rs []inventory.ResourceHealth + waits bool + }{ + {"the account alone", []inventory.ResourceHealth{relogin("lights", "operator")}, true}, + {"the account and its unit in that account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), + userUnit("lights", "operator")}, true}, + {"an account named by its target", []inventory.ResourceHealth{byTarget, userUnit("lights", "operator")}, true}, + {"another module's unit is not this module's", []inventory.ResourceHealth{relogin("lights", "operator"), otherModule}, true}, + {"a unit still starting is no fault", []inventory.ResourceHealth{relogin("lights", "operator"), starting}, true}, + {"a unit and no account", []inventory.ResourceHealth{userUnit("lights", "operator")}, false}, + {"an account not in its group", []inventory.ResourceHealth{notInGroup, userUnit("lights", "operator")}, false}, + {"a unit whose manager is not said", []inventory.ResourceHealth{relogin("lights", "operator"), systemUnit}, false}, + {"a unit in another account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), otherAccount}, false}, + {"a container beside the wait", []inventory.ResourceHealth{relogin("lights", "operator"), container}, false}, + } { + said, waits := personWait("lights", "laptop", c.rs) + if waits != c.waits { + t.Errorf("%s: waits %v; want %v", c.name, waits, c.waits) + continue + } + if waits && !strings.HasPrefix(said, "relogin needed on laptop: lights waits for a new login of operator") { + t.Errorf("%s: said %q", c.name, said) + } + } +} + +// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers +// no button, since nothing but their own new login can do it. +func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { + o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{ + relogin("openrazer", "operator"), userUnit("openrazer", "operator")}) + plainExample(t, o, "openrazer waits for a new login on g14", + "Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+ + "group it needs. You logged in before that, so openrazer cannot run until you log in again. Its "+ + "update is in place and nothing was undone.") + if len(o.Actions) != 0 || o.Needs == "" { + t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions) + } + // **Not "your account" when it is not the operator's** (issue 318 review). + other := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{ + relogin("openrazer", "guest"), userUnit("openrazer", "guest")}) + plainExample(t, other, "openrazer waits for a new login on g14", + "Needs you: have the account it names log out of g14 completely and log in again, or restart g14. Openrazer "+ + "put an account on g14 in a group it needs. That account logged in before that, so openrazer cannot run "+ + "until it logs in again. Its update is in place and nothing was undone.") + if unknown := reloginObservation("openrazer", "g14", "…", "", []inventory.ResourceHealth{relogin("openrazer", "operator")}); strings.Contains(unknown.Explanation, "your account") { + t.Errorf("an operator not known is still told it is their account: %q", unknown.Explanation) + } + // The kind's own wording, for a condition raised without words, names the module whole, dots and all, + // and never claims the account is the operator's. + w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "razer.lights.g14", Machine: "g14"}) + if w.Headline != "razer.lights waits for a new login on g14" || w.Needs == "" || len(w.Actions) != 0 || + strings.Contains(w.Explanation, "your account") { + t.Errorf("the kind's wording: %+v", w) + } +} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 06cfc19a..1b04411d 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -202,21 +202,6 @@ var plainWordings = map[string]func(conditions.Observation) words{ "send after send, so %s is sent them inside the instructions instead, which may make them too big.", m, m), Resolved: "Resolved: the big files for " + m + " reach the bus again"} }), - kindDeclarationNearLimit: worded(func(o conditions.Observation) words { - m := machineOr(o, "a machine") - if o.Severity == conditions.Urgent { - return words{Headline: m + " cannot be sent its instructions", - Needs: "make the instructions for " + m + " smaller, for example by taking a module off it.", - Explanation: fmt.Sprintf("The instructions for %s have grown past the largest message the bus takes, so "+ - "nothing new reaches %s until they are smaller.", m, m), - Resolved: "Resolved: the instructions for " + m + " fit the bus again"} - } - return words{Headline: "The instructions for " + m + " are nearly too big", - Needs: "do not add more to " + m + " until this is dealt with; the details give the size.", - Explanation: fmt.Sprintf("The instructions for %s are close to the largest message the bus takes. "+ - "One more module or secret may stop anything new from reaching %s.", m, m), - Resolved: "Resolved: the instructions for " + m + " have room again"} - }), kindBindingKept: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "A module's data source is held on " + m, @@ -675,14 +660,6 @@ func humanDuration(d time.Duration) string { return fmt.Sprintf("%d days", int(d.Hours()/24)) } -// repoName is a repository as a person says it: its name without its owner. -func repoName(repository string) string { - if i := strings.LastIndex(repository, "/"); i >= 0 { - return repository[i+1:] - } - return repository -} - // namesWords lists names as a person says them: "a", "a and b", "a, b and 2 more". func namesWords(names []string, most int) string { switch { @@ -696,50 +673,6 @@ func namesWords(names []string, most int) string { return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most) } -// deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else -// their count and the repository. -func deliveryWhat(modules []string, repository string) string { - if len(modules) > 0 && len(modules) <= 2 { - return namesWords(modules, 2) - } - if len(modules) > 2 { - return fmt.Sprintf("%d modules of %s", len(modules), repoName(repository)) - } - return repoName(repository) -} - -// deliveryName is a delivery's name in a headline: "openrazer delivery", "mesh-catalog delivery". -func deliveryName(modules []string, repository string) string { - if len(modules) > 0 && len(modules) <= 2 { - return namesWords(modules, 2) + " delivery" - } - return repoName(repository) + " delivery" -} - -// waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it. -// Start and Stop are also asked of the operator (novox/hq ADR 0259); the condition's own words keep saying -// where they are given without a channel, and the ask's text drops that (askText). -func waitingNeeds(severity conditions.Severity) string { - if severity == conditions.Urgent { - return "start it, or stop it, " + FromMeshMCPServer - } - return "" -} - -// waitingActions are the answers to a walk waiting past its urgent bound: start it, or stop it — the plan's -// own verbs, approved by the operator (novox/hq ADR 0259). None before the bound. -func waitingActions(plan string, severity conditions.Severity) []conditions.Action { - if severity != conditions.Urgent || plan == "" { - return nil - } - return []conditions.Action{ - {Label: "Start", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, - Arguments: map[string]string{"go": plan, "why": "", "cause": conditions.CauseOperatorAnswer}}, - {Label: "Stop", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, - Arguments: map[string]string{"stop": plan, "why": "", "cause": conditions.CauseOperatorAnswer}}, - } -} - // moduleNeeds is what the operator can do about a module unhealthy on a machine: log in again where its // account's groups wait for it (ADR 0252), restart a failed service, or nothing where the mesh restarts it. // No answer is offered for a restart: a desk click performs only an acknowledgement (ADR 0258). diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index 24cba62e..3d2729bb 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -14,7 +14,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // working out what one machine should be. @@ -123,43 +122,6 @@ func sortedKeysOf(m map[string]string) []string { return out } -// reportLeftOut says which of a machine's modules its declaration leaves out and why (novox/hq ADR -// 0163, rule 6), one line each: the machine is told everything else, and is told it was left out. -func reportLeftOut(node string, declared sendable) { - for _, m := range declared.LeftOut { - fmt.Printf("%s: %s left out — what the machine holds for it is kept and its containers are "+ - "untouched. %s\n", node, m, declared.leftOutWhy[m]) - } - // And whom it serves nothing, because their identity overflows what the provision keeps (ADR - // 0225): the machine is sent everything else, and the consumer is named. - for _, o := range declared.withheld { - fmt.Printf("%s: %s\n", node, o) - } - // And whom it no longer serves because they are bound elsewhere (novox/hq issue 274). - for _, u := range declared.unbound { - fmt.Printf("%s: %s\n", node, u) - } - // And every contribution its holder could not render, which the machine is sent without (novox/hq - // ADR 0255). - for _, u := range declared.unplaced { - fmt.Printf("%s: %s\n", node, u) - } - // And each file only the provisioner's client fills, left out because the module is not handed here. - for _, u := range declared.handedOnly { - fmt.Printf("%s: %s\n", node, u) - } -} - -// engineReadsBusy says whether a machine's node-engine reads a container's `busy` (novox/hq ADR 0335), by its -// own newest statement, for the same reason as engineReadsHealth: an older engine parses strictly. -func engineReadsBusy(ctx context.Context, inv *inventory.Inventory, node string) (bool, error) { - stated, had, err := inv.HealthOf(ctx, node) - if err != nil { - return false, err - } - return had && stated.Contract >= link.BusyContract, nil -} - // provisionerAccount is the account the provisioner's client runs as on this machine (novox/hq ADR 0349 §4): the user // the process of the module running it runs as, empty where no such module is on the machine. func provisionerAccount(plan catalogue.Resolution) string { @@ -178,25 +140,6 @@ func provisionerAccount(plan catalogue.Resolution) string { return "" } -// busyNotSent is, for a node-engine that does not read a container's busy, each container whose module declares -// one: not sent it, so replaced at once while it may hold work (novox/hq ADR 0335). Nothing for one that reads it. -func busyNotSent(modules []catalogue.Manifest, reads bool) []string { - if reads { - return nil - } - var out []string - for _, m := range modules { - for _, r := range m.Resources { - if _, has := r[catalogue.BusyField]; has { - out = append(out, fmt.Sprintf("%s.%v declares busy, and this node-engine does not read it yet (its "+ - "statement's contract is below %d): not sent, so the container is replaced at once, busy or not "+ - "(novox/hq ADR 0335)", m.Module, r["id"], link.BusyContract)) - } - } - } - return out -} - // certificateFor is what the mesh certifies about one machine's internal name. // // It reaches across two contexts and reads neither one's store from the other: `inventory` knows @@ -259,36 +202,6 @@ func certificateFor(ctx context.Context, open *stores, node string) (string, str return issued, authority.Certificate, nil } -// listensLines is what a person is told about what this module would open, and why — the same -// `why` every listens entry already carries for the firewall it also feeds (novox/hq ADR 0007), so -// deciding whether to assign a module can see what it would open before it opens it, not only -// after. A module with nothing to listen on prints nothing extra, same as today. -func listensLines(m catalogue.Manifest) []string { - var out []string - for _, l := range m.Listens { - if l.Why == "" { - out = append(out, fmt.Sprintf(" listens %d/%s from %s", l.Port, l.At(), l.From)) - continue - } - out = append(out, fmt.Sprintf(" listens %d/%s from %s — %s", l.Port, l.At(), l.From, l.Why)) - } - return out -} - -// shownAs is the heading `plan --show` puts over a resource's content. -// -// **A file written into says so.** Its content is the mesh's part of a file that is otherwise the -// machine's — the keys of a JSON document (novox/hq ADR 0102), the region of a hosts file (issue -// 128). Shown under a bare path it reads as the whole file, and a person checking what a take -// replaces would see a hosts file of a dozen lines where the machine keeps thirty. -func shownAs(r map[string]any) string { - heading := fmt.Sprintf("%v %v", r["id"], r["path"]) - if into, ok := r["into"].(string); ok && into != "" { - heading += fmt.Sprintf(" (written into, %s)", into) - } - return heading -} - // servedOnNode is what a module on a node tells a consumer of one of its provisions, with THAT // node's ports on it: the port the node was given (novox/hq ADR 0100) over the one the mesh // assigned over the manifest's own, settled with the node's settings layers. @@ -458,14 +371,6 @@ func (l *unheldLogger) log(node string, unheld []catalogue.Unheld) { node, len(lines), strings.Join(lines, "\n ")) } -// unheldChange is what an act on one node changed about its unmet seat dependencies, judged over -// its assignments before and after (novox/hq ADR 0207): each dependency now unmet that was not — -// which includes every one of a module just assigned — and each now met that was not. Nothing about -// any other node, and nothing that was already true before the act. -func unheldChange(shelf map[string]catalogue.Manifest, node string, before, after []string) []string { - return unheldChangeTo(shelf, node, before, after, unheldOver(shelf, node, after)) -} - // unheldOver is the node's unmet seat dependencies judged over a set of module names alone. func unheldOver(shelf map[string]catalogue.Manifest, node string, names []string) []catalogue.Unheld { var set []catalogue.Manifest diff --git a/cmd/mesh-controller/plan_test.go b/cmd/mesh-controller/plan_test.go index a9e0df65..221e9231 100644 --- a/cmd/mesh-controller/plan_test.go +++ b/cmd/mesh-controller/plan_test.go @@ -1,51 +1,3 @@ package main -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// `plan` tells a person what a module would open and why, from the same `why` every listens -// entry already carries for the firewall (novox/hq ADR 0007) — so deciding whether to assign a -// module does not need reading its manifest first. -func TestListensLinesShowWhatAModuleWouldOpenAndWhy(t *testing.T) { - m := catalogue.Manifest{Module: "minio", Listens: []catalogue.Listening{ - {Port: 9000, From: catalogue.FromMesh, Why: "the S3 endpoint"}, - {Port: 9001, From: catalogue.FromMesh}, - }} - got := listensLines(m) - if len(got) != 2 { - t.Fatalf("two listens entries, got %d: %v", len(got), got) - } - if !strings.Contains(got[0], "9000/tcp") || !strings.Contains(got[0], "the S3 endpoint") { - t.Errorf("the port and its why did not both appear: %q", got[0]) - } - if strings.Contains(got[1], "—") { - t.Errorf("a listens entry with no why should not print a dash: %q", got[1]) - } - if !strings.Contains(got[1], "9001/tcp") { - t.Errorf("the port still appears without a why: %q", got[1]) - } -} - -func TestListensLinesAreEmptyForAModuleWithNothingToListenOn(t *testing.T) { - if got := listensLines(catalogue.Manifest{Module: "board"}); len(got) != 0 { - t.Errorf("a module with no listens should print nothing, got %v", got) - } -} - -// `plan --show` says when a file is written into rather than over (novox/hq issue 128), or the -// mesh's region of a hosts file reads as the whole file. -func TestAFileWrittenIntoIsShownAsSuch(t *testing.T) { - region := shownAs(map[string]any{ - "id": "mesh-wireguard.fact-node-names", "path": "/etc/hosts", "into": "block"}) - if region != "mesh-wireguard.fact-node-names /etc/hosts (written into, block)" { - t.Errorf("the region is shown as %q", region) - } - whole := shownAs(map[string]any{"id": "dnsmasq.fact-node-zones", "path": "/etc/mesh-resolver/nodes.conf"}) - if strings.Contains(whole, "written into") { - t.Errorf("a whole file is shown as written into: %q", whole) - } -} +import () diff --git a/cmd/mesh-controller/probes.go b/cmd/mesh-controller/probes.go index 825a4c12..2bca46ae 100644 --- a/cmd/mesh-controller/probes.go +++ b/cmd/mesh-controller/probes.go @@ -608,15 +608,7 @@ func readDeliveries(ctx context.Context, inv *inventory.Inventory) (map[string]l } out := map[string]lastSend{} for _, r := range reports { - builds, known, err := inv.SentBuilds(ctx, r.Node) - if err != nil { - return nil, err - } - d := lastSend{sent: r.Sent, current: r.Current} - if known { - d.carried = builds - } - out[r.Node] = d + out[r.Node] = lastSend{sent: r.Sent, current: r.Current} } return out, nil } @@ -1121,25 +1113,6 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, behind = append(behind, fmt.Sprintf("the node-engine %s, the mesh holds %s (built from %s)", n.HostVersion, strings.Join(hostVersions, " or "), short(current[hostModule].Commit))) } - assigned, err := inv.Assigned(ctx, n.Name) - if err != nil { - return nil, err - } - tools := current[broker.RuntimeModule].Commit - if tools != "" && slices.Contains(assigned, broker.RuntimeModule) && !rolling[broker.RuntimeModule] { - sent, known, err := inv.SentBuilds(ctx, n.Name) - if err != nil { - return nil, err - } - switch { - case !known: - case !sameCommit(sent[broker.RuntimeModule], tools): - behind = append(behind, fmt.Sprintf("the node tools %s, the mesh holds %s", - short(orNotKnown(sent[broker.RuntimeModule])), short(tools))) - case !applied[n.Name]: - behind = append(behind, "the node tools it was last sent, not yet reported applied") - } - } if len(behind) > 0 { out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: n.Name, Token: "core-behind", Machine: n.Name, Severity: conditions.Warning, diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index e42f3dbf..a4ce877d 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -7,33 +7,16 @@ import ( "errors" "fmt" "github.com/novox/mesh-controller/internal/conditions" - "io" "log" "os" - "sort" "strings" - "time" "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/compose" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) -// reportUnhostable says which of a node's assigned modules the machine cannot run, once per push. -// -// A module whose declared capability has no detector on the machine is on the wrong machine. It is -// kept out of what the node is sent — the healthy modules beside it still converge — and named here -// so it is neither silently dropped nor a reason the whole node fails to push. -func reportUnhostable(node string, plan catalogue.Resolution) { - for _, u := range plan.Unhostable { - for _, c := range u.Missing { - fmt.Printf("%s not applied — %s\n", node, catalogue.WrongMachine(u.Module, c, node)) - } - } -} - // sending it, and holding the link that carries it. // // Split out of main.go, which had reached 2,769 lines because appending was always the @@ -326,95 +309,9 @@ func serve(ctx context.Context) (err error) { // OverlayCIDRVar is the range the mesh allocates node addresses from. const OverlayCIDRVar = "MESH_OVERLAY_CIDR" -// waitForApplied blocks until the node reports it applied exactly the declaration just sent, or the -// wait runs out. A report of failure or refusal for that same declaration ends the wait at once — -// there is nothing to wait for, and the reason is the node's own. -func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest string, wait time.Duration) error { - if digest == "" { - return nil // nothing was sent to this node - } - deadline := time.Now().Add(wait) - for { - doing, said, err := inv.DoingOf(ctx, node) - if err != nil { - return err - } - if said && doing.Declared == digest { - switch doing.Outcome { - case inventory.OutcomeApplied: - fmt.Printf("%s applied it\n", node) - return nil - case inventory.OutcomeFailed: - // Each failed resource by id with the machine's own words, so the answer says what to fix - // without a second look (novox/hq issue 629). - var each []string - for _, f := range doing.Failed { - each = append(each, f.ID+": "+f.Error) - } - return fmt.Errorf("%s applied what it was sent but %d resource(s) failed: %s", node, len(doing.Failed), - strings.Join(each, "; ")) - case inventory.OutcomeRefused: - return fmt.Errorf("%s refused what it was sent: %s", node, doing.Refused) - } - } - if time.Now().After(deadline) { - return fmt.Errorf("%s did not report applying what it was sent within %s "+ - "(it may still be converging; check `status`)", node, wait) - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(500 * time.Millisecond): - } - } -} - -// errGrants marks a send that stopped because what the machines' modules may do could not be issued -// (novox/hq issue 249). Nothing about the machines is wrong; asked again, it is likely to work, so an -// announcement that hits it is held and asked again. -var errGrants = errors.New("what the machines' modules may do on the bus could not be issued, and code " + - "sent before its grants is refused there") - -// grantsRefused is a grant that failed for some machines and not others: their memberships could not -// be issued, by machine, and only those machines are held back. -type grantsRefused struct{ nodes map[string]error } - -func (g *grantsRefused) Error() string { - names := make([]string, 0, len(g.nodes)) - for n := range g.nodes { - names = append(names, n) - } - sort.Strings(names) - return fmt.Sprintf("the memberships of %s could not be issued; the first: %v", - strings.Join(names, ", "), g.nodes[names[0]]) -} - // errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket. var errFilesNotPut = errors.New("the files it names by digest are not in the bucket") -// brokerFirst is the machines to send in the order a grant needs (novox/hq issue 249): the machine -// holding the bus first — its declaration carries the bus's user list (composeBusUsers), and a -// module's new permissions are refused by the bus until that list says them. Among the machines it -// is moved to the front; not among them, it is added only when it is behind. -func brokerFirst(names []string, holder string, behind bool) []string { - if holder == "" { - return names - } - present := false - rest := make([]string, 0, len(names)) - for _, n := range names { - if n == holder { - present = true - continue - } - rest = append(rest, n) - } - if !present && !behind { - return names - } - return append([]string{holder}, rest...) -} - // brokerBehind is the machine holding the bus — the one whose declaration carries the user list — // and, when it is not among the machines named, whether the user list it would be sent now differs // from the one it was last sent (novox/hq issue 249). @@ -484,21 +381,6 @@ func userListBehind(now, sentDigest string) bool { return now != "" && digestOf([]byte(now)) != sentDigest } -// couldNotBeResolved is what a push ends with when some machines could not be worked out. -// -// **After the rest have been sent, never instead of sending them.** It is still an error, because -// the mesh is not in the state somebody asked for and a command that exits cleanly having skipped a -// machine is a command that lies. What it must not do is decide anything about the machines beside -// it, which is why it says how many were sent. -func couldNotBeResolved(refusals []string, sent int) error { - if len(refusals) == 0 { - return nil - } - return fmt.Errorf( - "%d node(s) could not be resolved and were not sent. %d other node(s) were:\n\n%s", - len(refusals), sent, strings.Join(refusals, "\n\n")) -} - // digestOf is what the mesh compares to answer "has this machine been sent what it should be". // // Over the same bytes that are sent, so the comparison is of the thing itself rather than of @@ -508,65 +390,6 @@ func digestOf(body []byte) string { return hex.EncodeToString(sum[:]) } -// wouldSend is the digest of what each machine should be right now. -// -// Machines that do not resolve are left out rather than reported as waiting: "this machine cannot -// be worked out" is a different problem with a different remedy, and `plan` is where it is said. -func wouldSend(ctx context.Context, open *stores, - nodes []inventory.Node) (map[string]string, error) { - return wouldSendFrom(ctx, open, nodes, nil) -} - -// planned is one machine's plan as planFor answered it, for a caller that already asked. -type planned struct { - plan catalogue.Resolution - settings catalogue.SettingsBy -} - -// wouldSendFrom is wouldSend reusing the plans a caller worked out a moment before: resolving a -// machine is most of what `status` costs, and it used to resolve every machine twice (novox/hq -// to-be 45 Phase 0). A machine absent from plans is worked out here. -func wouldSendFrom(ctx context.Context, open *stores, - nodes []inventory.Node, plans map[string]planned) (map[string]string, error) { - - if _, err := generatorsOf(ctx, open); err != nil { - return nil, err - } - var err error - out := map[string]string{} - for _, n := range nodes { - known, have := plans[n.Name] - plan, settings := known.plan, known.settings - if !have { - if plan, settings, err = resolveNode(ctx, open, n.Name); err != nil { - continue - } - } - _, _ = plan, settings - declared, _, err := composeQuestion(ctx, open, n.Name, compose.Reading, false) - if err != nil { - continue - } - // Composed with the number the machine was LAST sent, so this is byte for byte what it was - // sent when nothing else changed. A fresh number here would make every machine read as - // behind for ever (novox/hq 04-ISSUES/107). - if declared.Sequence, err = open.inventory.Sequence(ctx, n.ID); err != nil { - return nil, err - } - // And the epoch it was last sent under, for the same reason: a new holder of the lease is not a - // change of the machine (novox/hq to-be 45 §6). - if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil { - return nil, err - } - body, err := declared.Body() - if err != nil { - return nil, err - } - out[n.Name] = digestOf(body) - } - return out, nil -} - // raiseTheBus asserts the streams and consumers the mesh's own traffic needs. // // **Every start, and it says what it did.** The objects are the mesh's, created by nothing else — @@ -695,27 +518,6 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok // without a bus. var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) } -// reportUnheldPushed says what a push's machines lack of the seats their modules depend on -// (novox/hq ADR 0207): every line for a machine the push named, since that is the machine somebody -// is looking at, and one line per machine otherwise — a list per machine across the mesh is the -// hundred lines that buried the one that mattered. Nothing for a machine that lacks nothing. -func reportUnheldPushed(w io.Writer, named bool, asked []string, unheld map[string][]catalogue.Unheld) { - for _, node := range asked { - lines := unheld[node] - if len(lines) == 0 { - continue - } - if named { - fmt.Fprintf(w, "\n%s has %d unmet seat dependenc(ies) (novox/hq ADR 0207):\n", node, len(lines)) - for _, u := range lines { - fmt.Fprintf(w, " %s\n", u) - } - continue - } - fmt.Fprintf(w, "%s: %d unmet seat dependenc(ies) — see `status`\n", node, len(lines)) - } -} - // schemaReach is the highest migration this build carries for the inventory's store (novox/hq issue 352). func schemaReach() (int, error) { migrations, err := inventory.Migrations() diff --git a/cmd/mesh-controller/registry_verbs.go b/cmd/mesh-controller/registry_verbs.go index c89f695d..b42be444 100644 --- a/cmd/mesh-controller/registry_verbs.go +++ b/cmd/mesh-controller/registry_verbs.go @@ -441,3 +441,11 @@ func imagesCommand(ctx context.Context, args []string) error { } return nil } + +// sentResource is what the store kept of one resource a machine was last sent, as far as `images` reads it: the container's +// image. Nothing writes the summary any more (ADR 0539 §5); a machine's images are read from its compositions. +type sentResource struct { + ID string `json:"id"` + Type string `json:"type"` + Image string `json:"image,omitempty"` +} diff --git a/cmd/mesh-controller/registry_verbs_test.go b/cmd/mesh-controller/registry_verbs_test.go index ec883231..bffcafb8 100644 --- a/cmd/mesh-controller/registry_verbs_test.go +++ b/cmd/mesh-controller/registry_verbs_test.go @@ -229,14 +229,3 @@ func TestImagesAreEveryContainerImageOfTheDeclarationAndOfWhatWasSent(t *testing t.Errorf("with nothing sent yet: %+v %v", none, err) } } - -func TestTheSentSummaryKeepsAContainersImage(t *testing.T) { - summary, err := summarize([]byte(`{"resources":[{"id":"web.server","type":"container","image":"x@sha256:aa"}, - {"id":"web.state","type":"directory","path":"/srv"}]}`)) - if err != nil { - t.Fatal(err) - } - if summary[0].Image != "x@sha256:aa" || summary[1].Image != "" { - t.Errorf("summarized as %+v", summary) - } -} diff --git a/cmd/mesh-controller/send_stub.go b/cmd/mesh-controller/send_stub.go deleted file mode 100644 index b263605e..00000000 --- a/cmd/mesh-controller/send_stub.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import "context" - -// What was once a send is no longer one (ADR 0539 §5, §8): the controller writes the compositions that changed instead -// (composeModules), and each machine follows its wants. The calls below stand where a send was asked until the part of the -// controller that asked it writes compositions in its place; the subtask that replaces the last caller deletes this file. - -// sendTo sends nothing; see sendToEach. -func sendTo(ctx context.Context, open *stores, names []string) error { - _, err := sendToEach(ctx, open, names) - return err -} - -// sendToEach answers the machines it was asked for, and sends them nothing. -func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) { - return names, nil -} - -// holdNodes held machines from composing to sending; nothing is sent, so it holds nothing. -func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) { - return ctx, func() {}, nil -} diff --git a/cmd/mesh-controller/sendable_integration_test.go b/cmd/mesh-controller/sendable_integration_test.go index ff78394f..ac441a8b 100644 --- a/cmd/mesh-controller/sendable_integration_test.go +++ b/cmd/mesh-controller/sendable_integration_test.go @@ -80,15 +80,6 @@ func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) { t.Fatalf("taken is not a list on the wire, even empty: %s", body) } - // The digest the mesh compares is the digest of what is sent: status and push agree. - would, err := wouldSend(ctx, open, mustNodes(t, open)) - if err != nil { - t.Fatal(err) - } - if would["anchor"] != digestOf(body) { - t.Fatal("the digest the mesh compares is not of the declaration push sends") - } - // Taking the module moves its resources out of untaken. if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil { t.Fatal(err) diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index dbcf95b6..18bfea30 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -274,29 +274,6 @@ func watchHeartbeats(f *signalFacts) []conditions.Observation { return out } -func watchReports(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, m := range f.machines { - if m.sentAt.IsZero() || m.reportedCurrent || m.asleep() { - continue - } - if m.silent(f.now) { - continue // silent: S1 says it, and a silent machine reports nothing - } - bound := max(reportAtLeast, 3*m.lastApply) - waited := f.now.Sub(m.sentAt) - if waited <= bound { - continue - } - out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: m.name, - Kind: "sent-not-reported", Machine: m.name, Severity: conditions.Warning, - Summary: fmt.Sprintf("%s was sent a declaration at %s and has not reported applying it (bound %s)", - m.name, m.sentAt.UTC().Format("2006-01-02 15:04 MST"), ago(bound)), - Said: fmt.Sprintf("waiting %s for the report of the declaration sent", ago(waited))}) - } - return out -} - func watchLoop(f *signalFacts) []conditions.Observation { if f.loop.pending == 0 { return nil diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index 08976a0b..80834150 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -402,7 +402,7 @@ func TestAMachineThatSaidItSleepsIsNotSilent(t *testing.T) { f.machines[0].lastHeard = now.Add(-2 * time.Hour) f.machines[0].sentAt, f.machines[0].reportedCurrent = now.Add(-time.Hour), false f.machines[0].power = link.PowerState{State: "sleeping", At: now.Add(-2 * time.Hour)} - if got := append(watchHeartbeats(f), append(watchReports(f), watchTools(f)...)...); len(got) != 0 { + if got := append(watchHeartbeats(f), watchTools(f)...); len(got) != 0 { t.Fatalf("a sleeping machine raised %+v", got) } f.machines[0].power = link.PowerState{State: "woke", At: now.Add(-time.Hour)} diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index 55af4a6f..c28cbe5b 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -438,7 +438,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { // ADR 0207). Each machine resolved again rather than threaded through whoResolves, whose answer // the private network is built from and should say nothing else; a machine that does not // resolve is already in refused, and is passed over here. - plans := map[string]planned{} for _, n := range out.nodes { plan, settings, err := resolveNode(ctx, open, n.Name) if err != nil { @@ -447,7 +446,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { } return answers{}, err } - plans[n.Name] = planned{plan, settings} // And which of its modules a push leaves out (novox/hq issue 380), judged as the push judges it. if left := leftOutOf(plan, settings, n.Adopted); len(left) > 0 { if out.leftOut == nil { @@ -491,25 +489,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { out.heals = countHeals(heals) } - // And which machines are not running what the mesh would send them. The same question as a - // module being behind its source, one level down: that one says the catalogue is out of date, - // this one says a machine is — and only the second has anybody's change waiting in it. - // - // **One machine that cannot be resolved must not take the answer away from every other** - // (novox/hq 04-ISSUES/017's sibling). This reaches the private network, and a mesh whose hub - // is the blocked machine has no hub — which used to come back here as a refusal, so `status` - // said nothing at all and `status --json` emitted prose to stderr and no JSON anywhere. The - // reason is kept and reported as data; every question that does not depend on it is still - // answered. - would, err := wouldSendFrom(ctx, open, out.nodes, plans) - if err != nil { - out.network = err.Error() - would = map[string]string{} - } - out.waiting, err = inv.Waiting(ctx, would) - if err != nil { - return answers{}, err - } out.reported, err = inv.LastReports(ctx) if err != nil { return answers{}, err diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index 40ebc04e..455624cf 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -185,14 +185,7 @@ type nudgingListener struct { } func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, error) { - // A declaration refused as older than the one the machine holds is counted by its writer — the - // controller epoch it claimed (novox/hq to-be 45 §6, S13) — and so is what the machine's own count - // says it refused beyond the refusals heard. now := time.Now() - if report.StaleRefusalOf() { - link.StaleRefusals.Refused(link.Refusal{Writer: link.WriterEpoch(report.Epoch), Epoch: report.Epoch, - Receiver: report.Node, At: now}) - } if report.Ordered() { link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now) } diff --git a/cmd/mesh-controller/unheld_report_test.go b/cmd/mesh-controller/unheld_report_test.go deleted file mode 100644 index e012a161..00000000 --- a/cmd/mesh-controller/unheld_report_test.go +++ /dev/null @@ -1,83 +0,0 @@ -package main - -import ( - "bytes" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// An act says what it changed about the node it acted on, and nothing about the rest of the mesh -// (novox/hq ADR 0207): after the seat dependencies shipped, every `push ` and `assign` printed -// every node's unmet dependencies, a hundred lines around the one about the module just assigned. - -func aContainer(name string) catalogue.Manifest { - return catalogue.Manifest{Module: name, Version: "1", - Resources: []map[string]any{{"id": name, "type": "container", "image": name}}} -} - -func TestAnAssignmentThatMeetsADependencySaysSo(t *testing.T) { - shelf := map[string]catalogue.Manifest{ - "web": aContainer("web"), - "docker": {Module: "docker", Claims: []catalogue.Claim{{Name: catalogue.ContainerRuntimeSeat}}, - Resources: []map[string]any{{"id": "d", "type": "container", "image": "dind"}}}, - } - lines := unheldChange(shelf, "laptop", []string{"web"}, []string{"web", "docker"}) - if len(lines) != 1 || !strings.Contains(lines[0], "web on laptop now has "+catalogue.ContainerRuntimeSeat+" held") { - t.Errorf("meeting a dependency said %v", lines) - } - // Taking the dependent off says nothing: the dependency went with its module. - if lines := unheldChange(shelf, "laptop", []string{"web"}, nil); len(lines) != 0 { - t.Errorf("unassigning the dependent said %v", lines) - } -} - -func TestAPushSaysANamedNodesDependenciesAndCountsTheRest(t *testing.T) { - unheld := map[string][]catalogue.Unheld{ - "anchor": {{Node: "anchor", Module: "db", Seat: catalogue.ContainerRuntimeSeat}}, - "laptop": {{Node: "laptop", Module: "web", Seat: catalogue.ContainerRuntimeSeat}, - {Node: "laptop", Module: "sshd", Seat: catalogue.ServiceManagerSeat}}, - } - var named bytes.Buffer - reportUnheldPushed(&named, true, []string{"laptop"}, unheld) - got := named.String() - if !strings.Contains(got, "laptop has 2 unmet") || !strings.Contains(got, "web on laptop") || - !strings.Contains(got, "sshd on laptop") || strings.Contains(got, "anchor") { - t.Errorf("a named push said:\n%s", got) - } - var all bytes.Buffer - reportUnheldPushed(&all, false, []string{"anchor", "laptop", "quiet"}, unheld) - want := "anchor: 1 unmet seat dependenc(ies) — see `status`\nlaptop: 2 unmet seat dependenc(ies) — see `status`\n" - if all.String() != want { - t.Errorf("a push to every node said:\n%s\nwant\n%s", all.String(), want) - } -} - -// The test gives logUnheld its own writer and memory, so a repeat in one process and the -// package's other tests see neither the last run's memory nor a swapped os.Stderr (novox/hq issue 486). -func TestOnlyTheServingControllerLogsEachChange(t *testing.T) { - var out bytes.Buffer - saved, savedOn := unheldLog, logUnheldChanges - unheldLog, logUnheldChanges = newUnheldLogger(&out), false - t.Cleanup(func() { unheldLog, logUnheldChanges = saved, savedOn }) - read := func(f func()) string { - out.Reset() - f() - return out.String() - } - u := []catalogue.Unheld{{Node: "n1", Module: "web", Seat: catalogue.ContainerRuntimeSeat}} - if got := read(func() { logUnheld("n1", u) }); got != "" { - t.Errorf("a command logged:\n%s", got) - } - logUnheldChanges = true - if got := read(func() { logUnheld("n1", u) }); !strings.Contains(got, "web on n1") { - t.Errorf("the serving controller did not log a change:\n%s", got) - } - if got := read(func() { logUnheld("n1", u) }); got != "" { - t.Errorf("an unchanged report was logged again:\n%s", got) - } - if got := read(func() { logUnheld("n1", nil) }); !strings.Contains(got, "every seat its modules depend on is held") { - t.Errorf("a report that became none was not logged:\n%s", got) - } -} diff --git a/cmd/mesh-controller/unseen.go b/cmd/mesh-controller/unseen.go index 1ed42bcd..53cc2acd 100644 --- a/cmd/mesh-controller/unseen.go +++ b/cmd/mesh-controller/unseen.go @@ -1,18 +1,8 @@ package main import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "os" "reflect" "sort" - "strings" - - "github.com/novox/mesh-controller/internal/inventory" ) // No change to a machine takes effect unseen (novox/hq ADR 0217, to-be 44). @@ -71,275 +61,3 @@ func leaves(m map[string]any, prefix string) map[string]any { } return out } - -// ---- 2. what a push will change ----------------------------------------------------------------- - -// sentResource is what is kept of one resource a machine was sent: enough to say what changed and -// whether a container's data moved, and nothing that could carry a secret — a file's content is -// kept as a digest, never as text (the record lands in the store's own backups). -type sentResource struct { - ID string `json:"id"` - Type string `json:"type"` - // Digest is of the whole resource as it was sent. - Digest string `json:"digest"` - // Fields is each field's digest, so a change can be named by field. - Fields map[string]string `json:"fields"` - // Volumes is a container's mounts as sent — paths, which are not secrets, and what a moved - // data directory is read from. - Volumes []string `json:"volumes,omitempty"` - // Image is a container's image as sent — a reference, not a secret — so the machine can be told - // which images its last declaration named, and keep them (novox/hq ADR 0251, `images`). Absent in - // a summary kept before it was. - Image string `json:"image,omitempty"` -} - -// summarize is what is kept of a declaration body: its resources, in the order sent. -func summarize(body []byte) ([]sentResource, error) { - var envelope struct { - Resources []map[string]any `json:"resources"` - } - if err := json.Unmarshal(body, &envelope); err != nil { - return nil, fmt.Errorf("a declaration that is not one: %w", err) - } - out := make([]sentResource, 0, len(envelope.Resources)) - for _, r := range envelope.Resources { - s := sentResource{ID: fmt.Sprint(r["id"]), Type: fmt.Sprint(r["type"]), Digest: digestValue(r), - Fields: map[string]string{}} - for k, v := range r { - s.Fields[k] = digestValue(v) - } - if s.Type == "container" { - if image, ok := r["image"].(string); ok { - s.Image = image - } - if vols, ok := r["volumes"].([]any); ok { - for _, v := range vols { - s.Volumes = append(s.Volumes, fmt.Sprint(v)) - } - } - } - out = append(out, s) - } - return out, nil -} - -func digestValue(v any) string { - raw, _ := json.Marshal(v) - sum := sha256.Sum256(raw) - return hex.EncodeToString(sum[:8]) -} - -// changedResource is one resource sent differently, with the fields that differ. -type changedResource struct { - ID, Type string - Fields []string -} - -// sentDiff is what would be sent now against what was sent last, by resource id. -type sentDiff struct { - Added, Removed []string - Changed []changedResource -} - -func (d sentDiff) empty() bool { - return len(d.Added) == 0 && len(d.Removed) == 0 && len(d.Changed) == 0 -} - -func diffSent(before, after []sentResource) sentDiff { - was := map[string]sentResource{} - for _, r := range before { - was[r.ID] = r - } - var d sentDiff - seen := map[string]bool{} - for _, r := range after { - seen[r.ID] = true - old, had := was[r.ID] - if !had { - d.Added = append(d.Added, r.ID) - continue - } - if old.Digest == r.Digest { - continue - } - c := changedResource{ID: r.ID, Type: r.Type} - for k, v := range r.Fields { - if old.Fields[k] != v { - c.Fields = append(c.Fields, k) - } - } - for k := range old.Fields { - if _, kept := r.Fields[k]; !kept { - c.Fields = append(c.Fields, k) - } - } - sort.Strings(c.Fields) - d.Changed = append(d.Changed, c) - } - for _, r := range before { - if !seen[r.ID] { - d.Removed = append(d.Removed, r.ID) - } - } - sort.Strings(d.Added) - sort.Strings(d.Removed) - sort.Slice(d.Changed, func(a, b int) bool { return d.Changed[a].ID < d.Changed[b].ID }) - return d -} - -// writeDiff says a diff the way a person reads one: what is added, removed and changed, and a -// changed container's fields — a container sent differently is a container recreated. -func writeDiff(w io.Writer, node string, d sentDiff, moves []dataMove) { - if d.empty() { - fmt.Fprintf(w, "%s: nothing would change — it was last sent exactly this\n", node) - return - } - fmt.Fprintf(w, "%s would change:\n", node) - for _, id := range d.Added { - fmt.Fprintf(w, " + %s\n", id) - } - for _, id := range d.Removed { - fmt.Fprintf(w, " - %s\n", id) - } - for _, c := range d.Changed { - what := "changed" - if c.Type == "container" { - what = "recreated" - } - fmt.Fprintf(w, " ~ %s %s: %s\n", c.ID, what, strings.Join(c.Fields, ", ")) - } - writeMoves(w, node, moves) -} - -// ---- 3. a running module's data moving ------------------------------------------------------------ - -// dataMove is a container keeping a mount at the same place inside it with a different directory -// on the machine behind it: its data moving — or, as on 2026-10-05, a module about to start on an -// empty directory because the placement that pointed at its data was lost. -type dataMove struct { - Container, Inside, From, To string -} - -// Module is the module the container belongs to: resource ids are `.`. -func (m dataMove) Module() string { - module, _, _ := strings.Cut(m.Container, ".") - return module -} - -// movesIn is every data move between what a machine was sent and what it would be sent. A new -// container, a mount added or dropped and a changed image are not moves. -func movesIn(before, after []sentResource) []dataMove { - was := map[string]sentResource{} - for _, r := range before { - if r.Type == "container" { - was[r.ID] = r - } - } - var out []dataMove - for _, r := range after { - old, had := was[r.ID] - if r.Type != "container" || !had { - continue - } - from := mountSources(old.Volumes) - for inside, to := range mountSources(r.Volumes) { - if prev, mounted := from[inside]; mounted && prev != to { - out = append(out, dataMove{Container: r.ID, Inside: inside, From: prev, To: to}) - } - } - } - sort.Slice(out, func(a, b int) bool { - if out[a].Container != out[b].Container { - return out[a].Container < out[b].Container - } - return out[a].Inside < out[b].Inside - }) - return out -} - -// mountSources is a container's mounts by the place inside it: `source:inside[:options]`. -func mountSources(volumes []string) map[string]string { - out := map[string]string{} - for _, v := range volumes { - parts := strings.SplitN(v, ":", 3) - if len(parts) < 2 { - continue - } - out[parts[1]] = parts[0] - } - return out -} - -func writeMoves(w io.Writer, node string, moves []dataMove) { - for _, m := range moves { - fmt.Fprintf(w, " ! %s: %s moves from %s to %s\n", m.Container, m.Inside, m.From, m.To) - } -} - -// heldMoves is the moves in a push to one machine that the operator has not said to send (`--move -// `); empty when there is nothing to hold, including a machine never sent anything before. -func heldMoves(ctx context.Context, inv *inventory.Inventory, node string, body []byte, allowed map[string]bool) ([]dataMove, error) { - prev, err := inv.SentSummary(ctx, node) - if err != nil || len(prev) == 0 { - return nil, err - } - var before []sentResource - if err := json.Unmarshal(prev, &before); err != nil { - // A record this version cannot read compares with nothing: holding every push for it would - // stop the mesh on a format, which is not what is at stake. Said, so it is not passed over. - fmt.Fprintf(os.Stderr, "%s: what it was last sent is kept in a form this version does not read, "+ - "so whether its data would move is not known: %v\n", node, err) - // empty-on-error: said above; the send replaces the unreadable record with one this version reads - return nil, nil - } - after, err := summarize(body) - if err != nil { - return nil, err - } - var held []dataMove - for _, m := range movesIn(before, after) { - if !allowed[m.Module()] { - held = append(held, m) - } - } - return held, nil -} - -// sayDataHeld tells the operator why a machine was not sent, and how to send it. -func sayDataHeld(w io.Writer, node string, moves []dataMove) { - modules := map[string]bool{} - for _, m := range moves { - modules[m.Module()] = true - } - var names []string - for m := range modules { - names = append(names, m) - } - sort.Strings(names) - fmt.Fprintf(w, "held %s: a running module's data would move (novox/hq ADR 0217)\n", node) - writeMoves(w, node, moves) - fmt.Fprintf(w, " if that is meant: push %s --move %s\n", node, strings.Join(names, ",")) -} - -// writePlanDiff says what sending body to a machine would change against what it was last sent. -func writePlanDiff(ctx context.Context, inv *inventory.Inventory, node string, body []byte) error { - after, err := summarize(body) - if err != nil { - return err - } - prev, err := inv.SentSummary(ctx, node) - if err != nil { - return err - } - if len(prev) == 0 { - fmt.Printf("%s: what it was last sent is not kept yet — the first push from this version keeps "+ - "it; %d resource(s) would be sent\n", node, len(after)) - return nil - } - var before []sentResource - if err := json.Unmarshal(prev, &before); err != nil { - return fmt.Errorf("%s: what it was last sent is kept in a form this version does not read: %w", node, err) - } - writeDiff(os.Stdout, node, diffSent(before, after), movesIn(before, after)) - return nil -} diff --git a/cmd/mesh-controller/unseen_test.go b/cmd/mesh-controller/unseen_test.go deleted file mode 100644 index 30bb7413..00000000 --- a/cmd/mesh-controller/unseen_test.go +++ /dev/null @@ -1,175 +0,0 @@ -package main - -import ( - "bytes" - "reflect" - "strings" - "testing" -) - -// novox/hq ADR 0217: no change to a machine takes effect unseen. - -// The layer of 2026-10-05: a media server's node layer, and the one that replaced it, which kept -// `places` and lost three of its four directories and every other key. -var before = map[string]any{ - "puid": 1000.0, "pgid": 1000.0, - "expose": map[string]any{"32400": "anywhere"}, - "places": map[string]any{ - "config": map[string]any{"path": "/srv/media/config", "owner": "1000:1000"}, - "data": map[string]any{"path": "/srv/media/data", "owner": "1000:1000"}, - "transcode": map[string]any{"path": "/srv/media/temp", "owner": "1000:1000"}, - }, -} - -func TestASettingThatKeepsAKeyAndLosesItsEntriesIsSaidToRemoveThem(t *testing.T) { - after := map[string]any{"places": map[string]any{ - "previews": map[string]any{"path": "/srv/pool/previews", "owner": "1000:1000"}, - }} - added, changed, removed := settingsChange(before, after) - if !reflect.DeepEqual(added, []string{"places.previews.owner", "places.previews.path"}) { - t.Errorf("added %v", added) - } - if len(changed) != 0 { - t.Errorf("changed %v", changed) - } - for _, lost := range []string{"expose.32400", "pgid", "places.config.path", "places.data.owner", "puid"} { - found := false - for _, r := range removed { - found = found || r == lost - } - if !found { - t.Errorf("removing %s was not said: %v", lost, removed) - } - } -} - -func TestASettingThatOnlyAddsOrChangesRemovesNothing(t *testing.T) { - after := map[string]any{ - "puid": 1001.0, "pgid": 1000.0, - "expose": map[string]any{"32400": "anywhere"}, - "places": map[string]any{ - "config": map[string]any{"path": "/srv/media/config", "owner": "1000:1000"}, - "data": map[string]any{"path": "/srv/media/data", "owner": "1000:1000"}, - "transcode": map[string]any{"path": "/srv/media/temp", "owner": "1000:1000"}, - "previews": map[string]any{"path": "/srv/pool/previews", "owner": "1000:1000"}, - }, - } - _, changed, removed := settingsChange(before, after) - if len(removed) != 0 { - t.Errorf("an additive set was said to remove %v", removed) - } - if !reflect.DeepEqual(changed, []string{"puid"}) { - t.Errorf("changed %v", changed) - } -} - -// What was sent, as a push would send it: a media server's container and a file with a secret. -func declaration(configFrom string, extra ...string) []byte { - vols := `"` + configFrom + `:/config", "/srv/media/data:/data"` - for _, e := range extra { - vols += `, "` + e + `"` - } - return []byte(`{"declaration":1,"sequence":7,"resources":[ - {"id":"plex.server","type":"container","image":"plex@sha256:aa","volumes":[` + vols + `]}, - {"id":"plex.env","type":"file","path":"/srv/media/env","content":"TOKEN=the-secret-itself"}]}`) -} - -func summarized(t *testing.T, body []byte) []sentResource { - t.Helper() - s, err := summarize(body) - if err != nil { - t.Fatal(err) - } - return s -} - -func TestWhatIsKeptOfASendHoldsNoFileContent(t *testing.T) { - s := summarized(t, declaration("/srv/media/config")) - var kept bytes.Buffer - for _, r := range s { - kept.WriteString(r.ID + r.Type + r.Digest + strings.Join(r.Volumes, ",")) - for k, v := range r.Fields { - kept.WriteString(k + v) - } - } - if strings.Contains(kept.String(), "the-secret-itself") { - t.Fatal("a file's content was kept in the record of what was sent") - } - if len(s) != 2 || s[0].Volumes[0] != "/srv/media/config:/config" { - t.Fatalf("summary %+v", s) - } -} - -func TestADiffOfAnUnchangedMachineIsEmptyAndAChangedContainerNamesItsField(t *testing.T) { - was := summarized(t, declaration("/srv/media/config")) - if d := diffSent(was, summarized(t, declaration("/srv/media/config"))); !d.empty() { - t.Fatalf("an unchanged machine differs: %+v", d) - } - d := diffSent(was, summarized(t, declaration("/var/lib/plex/config"))) - if len(d.Changed) != 1 || d.Changed[0].ID != "plex.server" || !reflect.DeepEqual(d.Changed[0].Fields, []string{"volumes"}) { - t.Fatalf("diff %+v", d) - } - var out bytes.Buffer - writeDiff(&out, "home", d, movesIn(was, summarized(t, declaration("/var/lib/plex/config")))) - for _, want := range []string{"~ plex.server recreated: volumes", "! plex.server: /config moves from /srv/media/config to /var/lib/plex/config"} { - if !strings.Contains(out.String(), want) { - t.Errorf("diff does not say %q:\n%s", want, out.String()) - } - } -} - -// The incident: the configuration mount would move to an empty default directory. -func TestARunningContainersDataMovingIsAMoveAndAnAddedMountIsNot(t *testing.T) { - was := summarized(t, declaration("/srv/media/config")) - moves := movesIn(was, summarized(t, declaration("/var/lib/plex/config"))) - want := []dataMove{{Container: "plex.server", Inside: "/config", From: "/srv/media/config", To: "/var/lib/plex/config"}} - if !reflect.DeepEqual(moves, want) { - t.Fatalf("moves %+v", moves) - } - if moves[0].Module() != "plex" { - t.Errorf("module %q", moves[0].Module()) - } - // A mount added — the previews of 2026-10-05 — is not a move. - if m := movesIn(was, summarized(t, declaration("/srv/media/config", "/srv/pool/previews:/config/Media"))); len(m) != 0 { - t.Errorf("an added mount was held as a move: %+v", m) - } - // Nor is a container the machine never ran. - if m := movesIn(nil, was); len(m) != 0 { - t.Errorf("a first send was held as a move: %+v", m) - } -} - -func TestTheVerbsCarryReadReplaceAndMove(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - want []string - }{ - {"settings", map[string]any{"module": "plex", "node": "home"}, []string{"settings", "show", "plex", "--node", "home"}}, - {"settings", map[string]any{"module": "plex", "history": "true"}, []string{"settings", "show", "plex", "--history"}}, - {"settings", map[string]any{"module": "plex", "values": "{}", "replace": "true"}, []string{"settings", "set", "plex", "{}", "--replace"}}, - } { - got, err := argvFor(c.verb, c.args) - if err != nil || !reflect.DeepEqual(got, c.want) { - t.Errorf("%s %v: %v %v, want %v", c.verb, c.args, got, err, c.want) - } - } -} - -// An argument of the guards given where it cannot take effect is refused, not passed over (issue 244): -// a move with no machine named, a replace beside a clear, a history beside values, a diff beside files. -func TestTheGuardsArgumentsAreNotPassedOver(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - }{ - {"push", map[string]any{"move": "plex", "why": "w"}}, - {"settings", map[string]any{"module": "plex", "clear": "true", "replace": "true"}}, - {"settings", map[string]any{"module": "plex", "values": "{}", "history": "true"}}, - {"plan", map[string]any{"node": "home", "files": "true", "diff": "true"}}, - } { - if argv, err := argvFor(c.verb, c.args); err == nil { - t.Errorf("%s %v was taken as %v, and an argument passed over", c.verb, c.args, argv) - } - } -} diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index 1018cd91..b571cc93 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -13,7 +13,6 @@ import ( "sync" "time" - "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -43,30 +42,6 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error { return nil } -// askAgainOnGrants marks a send that stopped at its grants as one to ask again (novox/hq issue 249): -// an announcement handled by a send whose memberships could not be issued is held and redelivered, -// rather than taken as handled with the machines left on the old version. -func askAgainOnGrants(err error) error { - if err != nil && errors.Is(err, errGrants) && !errors.Is(err, link.ErrTryAgain) { - return fmt.Errorf("%w: %w", link.ErrTryAgain, err) - } - return err -} - -// readableList names machines the way a sentence does, because this is read by a person deciding -// whether an upgrade went where they expected. -func readableList(names []string) string { - switch len(names) { - case 0: - return "nothing" - case 1: - return names[0] - case 2: - return names[0] + " and " + names[1] - } - return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1] -} - func shortCommit(commit string) string { if len(commit) > 8 { return commit[:8] @@ -74,13 +49,6 @@ func shortCommit(commit string) string { return commit } -func isAre(n int) string { - if n == 1 { - return "is" - } - return "are" -} - // Announceable is every build this mesh recorded, in the shape the builder announces one. // // **The catalogue asks for this when it starts, and the answer is the graph's foundation** @@ -125,25 +93,6 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error return out, nil } -// notNow marks a store that could not be read right now, or plans another controller is working (issue 921), so -// the announcement is held rather than lost; anything else is returned as it was. -func notNow(err error) error { - if inventory.Unreachable(err) { - return fmt.Errorf("%w: %w", link.ErrTryAgain, err) - } - return err -} - -// unsaved marks the failure to save the plan a merge was added to, whatever it was: the merge is carried until that save -// writes it (inventory.AddMerge), so the announcement is held and handed over again, never taken (novox/hq ADR 0407 -// decision 3, ADR 0408, issue 1092). The bus acknowledges it only after the save succeeded. -func unsaved(err error) error { - if err == nil { - return nil - } - return fmt.Errorf("%w: the merge is not kept until its plan is saved: %w", link.ErrTryAgain, err) -} - // SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which // becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go. func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { @@ -153,82 +102,6 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { return keepForgeMerge(ctx, m) } -// movesOfMerge is what one merge moves, acted on: every module recorded as built from that repository and -// branch is marked as moved to the merge commit; a module the merge deleted is forgotten or said; a new module -// is asked for and sent nowhere. The names returned are what the walk builds, bases first along the graph -// (novox/hq 04-ISSUES/131). Nothing is built or pushed here: the walk asks its tiers. Called when a batch is -// cut, once per repository, with the latest merge of the repository's branch and every file the batch's -// merges of it changed. -func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry, - read map[string][]inventory.ReadRepository) ([]string, error) { - from, packaging, already := mergeCandidates(m, entries, read) - if len(from) == 0 && len(packaging) == 0 { - // "Already built from it" and "nothing reads it" are different facts, and reading the first - // as the second sends somebody looking for a broken trigger when the mesh is up to date. - if already > 0 { - fmt.Printf("%s/%s merged into %s (%.8s); %d module(s) the mesh holds are already built "+ - "from it\n", m.Owner, m.Repo, m.Base, m.Commit, already) - return nil, nil - } - fmt.Print(nothingReadsIt(m.Owner+"/"+m.Repo, m)) - return nil, nil - } - // Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows - // another branch is not part of this merge, and whoever is waiting for its change should read why. - for _, e := range entries { - if sameRepository(e.Source.Repository, m) && !sourceIs(e.Source, m) { - fmt.Printf(" %s is built from %s/%s and follows %s, not %s; this merge leaves it out\n", - e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base) - } - } - touched, added, _ := touchedBy(from, entries, m, read) - // **A module the merge deleted is not built** (novox/hq ADR 0236): its manifest is gone, so the build - // seat finds nothing saying what it is, and the plan failed on it (`has no module.json at …`) with - // every other module of its tier left unsent. It is forgotten where nothing holds it, said otherwise. - touched, deleted := splitDeleted(touched, m) - for _, e := range deleted { - retireDeleted(ctx, inv, e, m) - } - for _, e := range touched { - if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil { - return nil, notNow(err) - } - } - // **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan - // says so, and assigning it is a person's act, which needs it registered first. - // Nothing asks its first build (novox/hq ADR 0501 §6): the builder builds every fingerprint the indexer writes, and - // the controller registers the module from its first done key (new_modules.go). - var built []string - if len(from) > 0 { - built = added - } - moved := append(append([]inventory.Entry{}, touched...), packaging...) - if len(moved) == 0 { - if len(built) > 0 { - fmt.Printf("%s/%s merged into %s (%.8s); it changed no module the mesh holds, and adds %s: "+ - "registered from the builder's done key of each, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit, - strings.Join(built, ", ")) - return nil, nil - } - fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+ - "built from\n", m.Owner, m.Repo, m.Base, m.Commit) - return nil, nil - } - // Why each is in it (issue 363): the files of its build source the merge changed, or why it is read whole. - for _, e := range moved { - fmt.Printf(" %s: %s\n", e.Manifest.Module, whyMoved(e, read[e.Manifest.Module], m)) - } - for _, e := range packaging { - fmt.Printf(" %s reads %s/%s through its build context: its own source record is left where it is\n", - e.Manifest.Module, m.Owner, m.Repo) - } - var names []string - for _, e := range moved { - names = append(names, e.Manifest.Module) - } - return names, nil -} - // actingOnMerges keeps one merge acted on at a time (novox/hq issue 266). var actingOnMerges sync.Mutex @@ -292,63 +165,6 @@ func lookedAtCommit(read []inventory.ReadRepository, commit string) bool { // historyMargin is how far a packaging module's last look is taken back before a merge is history for it. const historyMargin = time.Minute -// whyMoved is why a merge moves a module, as a plan says it (novox/hq ADR 0267, issue 363): the changed -// files in its build source, or why it is read whole. For a module built from the merged repository or one -// whose build context is that repository; a dependent is in a plan for what it stands on. -func whyMoved(e inventory.Entry, read []inventory.ReadRepository, m link.SourceMoved) string { - if len(m.Paths) == 0 || m.PathsTruncated { - return "read whole: the merge's changed files were not all said" - } - whole := "no build source recorded" - for _, r := range read { - if r.Own && r.Whole != "" { - whole = r.Whole - } - } - held := func(paths []string) []string { - var in []string - for _, p := range m.Paths { - if gosource.SourceHolds(paths, p) { - in = append(in, p) - } - } - return in - } - changed := func(where string, in []string) string { - return fmt.Sprintf("its build source%s changed: %d changed file(s) in it, e.g. %s", where, len(in), in[0]) - } - if sameRepository(e.Source.Repository, m) { - if own := ownSource(read); own != nil { - if in := held(own); len(in) > 0 { - return changed("", in) - } - } - dir := strings.Trim(e.Source.Path, "/") - if dir == "" { - return "read whole: " + whole + ", so every file of its repository is its build source" - } - for _, p := range m.Paths { - if inside(p, dir) { - return "read whole: " + whole + ", so its directory is its build source; e.g. " + p - } - } - return "read whole: " + whole - } - for _, r := range read { - if r.Own || !sameRepository(r.Repository, m) || !followsBase(r.Ref, m.Base) { - continue - } - if len(r.Paths) > 0 { - if in := held(r.Paths); len(in) > 0 { - return changed(" in "+m.Owner+"/"+m.Repo, in) - } - continue - } - return "read whole: " + whole + ", so every file of " + m.Owner + "/" + m.Repo + ", which its build context is, is its build source" - } - return "read whole: " + whole -} - // lookedOf is when a module packaging another repository was last looked at, as readForPlanning says. func lookedOf(read []inventory.ReadRepository) time.Time { var at time.Time @@ -360,21 +176,6 @@ func lookedOf(read []inventory.ReadRepository) time.Time { return at } -// wouldMove is the modules acting on this merge would move and rebuild — SourceMoved's judgement, made -// without acting (novox/hq issue 266). Empty for a merge already acted on: acting marks each module built -// from the repository as looked at, so the merge then reads as history for it. -// -// **The ones packaging source from it too** (novox/hq ADR 0267): with a module moved only by the files of -// its build source, a merge can move a packaging module and nothing built from the repository, and a missed -// one of those was never acted on. A packaging module's look is its newest build or plan (lookedAt), so a -// merge acted on for it reads as history once its plan is made. -func wouldMove(m link.SourceMoved, entries []inventory.Entry, - read map[string][]inventory.ReadRepository) []inventory.Entry { - from, packaging, _ := mergeCandidates(m, entries, read) - touched, _ := splitDeleted(whatTheMergeTouched(from, entries, m, read), m) - return append(touched, packaging...) -} - // splitDeleted parts the modules a merge touched into those it changed and those whose manifest it // removed — deleted at their source (ADR 0236). func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted []inventory.Entry) { @@ -397,39 +198,6 @@ func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted return kept, deleted } -// retireDeleted is what the mesh does with a module deleted at its source: its record says the merge -// was looked at, so it is not acted on again; it is forgotten where nothing holds it; where a machine -// runs it or the mesh holds something for it, that is said, and nothing is built. -func retireDeleted(ctx context.Context, inv *inventory.Inventory, e inventory.Entry, m link.SourceMoved) { - name := e.Manifest.Module - if err := inv.SourceMoved(ctx, name, m.Commit); err != nil { - fmt.Printf(" %s was deleted from %s/%s at %.8s, and that could not be recorded: %v\n", name, m.Owner, m.Repo, m.Commit, err) - } - err := inv.ForgetModule(ctx, name) - switch { - case err == nil: - fmt.Printf(" %s was deleted from %s/%s at %.8s: forgotten, nothing built\n", name, m.Owner, m.Repo, m.Commit) - case errors.Is(err, inventory.ErrStillAssigned) || errors.Is(err, inventory.ErrStillHolds): - fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; the mesh still runs or holds it, so it is "+ - "kept until a person unassigns it and `module forget %s`: %v\n", name, m.Owner, m.Repo, m.Commit, name, - firstLine(err.Error())) - default: - fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; forgetting it failed: %v\n", name, m.Owner, - m.Repo, m.Commit, err) - } -} - -// nothingReadsIt is what is said of a merge no module the mesh holds reads. **A merge into a branch other than the -// trunk is said to start no delivery** (novox/hq ADR 0401 clause 3, issue 889): every delivery is cut from a -// repository's trunk, and whoever merged into a feature branch should not look for a broken trigger. -func nothingReadsIt(repository string, m link.SourceMoved) string { - if m.Base != "" && m.Base != "main" { - return fmt.Sprintf("%s merged into %s (%.8s), not the trunk of %s; a merge into any other branch starts no "+ - "delivery\n", repository, m.Base, m.Commit, repository) - } - return fmt.Sprintf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit) -} - // sourceIs is whether a recorded source is the repository and branch a merge announced. A source on // the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from. // An empty recorded ref is the repository's default branch, which is what a merge into the base @@ -531,79 +299,12 @@ func readsDirectory(e inventory.Entry, p string) bool { return strings.Trim(e.Source.Path, "/") == "" || inside(p, e.Source.Path) } -// unchangedOfMerge are the modules a merge changed files for under a reading without recorded build sources, but -// whose recorded build source holds none of them, each with why (novox/hq issue 611): not built, because the build -// the mesh holds is theirs (ADR 0267 §5). That reading — a delivery plan made while another walk built the module -// (issue 606) — would have moved them; the walk says they matched what runs rather than leaving them unnamed. -// A module built from the repository whose directory, or whole repository when built from its root, holds a -// changed file, and a module packaging the repository's branch, are each read so. A merge whose changed files -// were not all said moves every one of them: none is said here. Without acting. -func unchangedOfMerge(m link.SourceMoved, entries []inventory.Entry, - read map[string][]inventory.ReadRepository) map[string]string { - if len(m.Paths) == 0 || m.PathsTruncated { - return nil - } - from, _, _ := mergeCandidates(m, entries, read) - touched, _, _ := touchedBy(from, entries, m, read) - out := map[string]string{} - for _, e := range from { - name := e.Manifest.Module - if ownSource(read[name]) == nil || slices.ContainsFunc(touched, func(t inventory.Entry) bool { - return t.Manifest.Module == name - }) { - continue - } - var in []string - for _, p := range m.Paths { - if readsDirectory(e, p) { - in = append(in, p) - } - } - if len(in) > 0 { - out[name] = fmt.Sprintf("its build source holds none of the %d changed file(s) in its directory (e.g. %s): "+ - "the build the mesh holds is its build", len(in), in[0]) - } - } - for _, e := range entries { - name := e.Manifest.Module - if sourceIs(e.Source, m) || !packagesSaid(read[name], m) || readsFrom(read[name], m) { - continue - } - out[name] = fmt.Sprintf("its build source in %s/%s holds none of the %d changed file(s) (e.g. %s): "+ - "the build the mesh holds is its build", m.Owner, m.Repo, len(m.Paths), m.Paths[0]) - } - if len(out) == 0 { - return nil - } - return out -} - -// packagesSaid is whether a module's newest trunk build said the build source it read in a merge's repository and -// branch, as a context beside its own. -func packagesSaid(read []inventory.ReadRepository, m link.SourceMoved) bool { - for _, r := range read { - if !r.Own && sameRepository(r.Repository, m) && followsBase(r.Ref, m.Base) && len(r.Paths) > 0 { - return true - } - } - return false -} - // readForPlanning is what each module's build read, as a change is mapped onto it — for a merge's reach, a pull request's // check and the facts alike (novox/hq ADR 0238), so they cannot disagree. No plan overtakes it any more (ADR 0539 §17). func readForPlanning(ctx context.Context, inv *inventory.Inventory) (map[string][]inventory.ReadRepository, error) { return inv.ReadRepositories(ctx) } -// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: **a -// changed file touches exactly the modules whose build reads it** (novox/hq issue 280, ADR 0238). It is -// touchedBy's first answer; touchedBy is the one place the mesh maps a changed file onto its modules. -func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved, - read map[string][]inventory.ReadRepository) []inventory.Entry { - touched, _, _ := touchedBy(candidates, known, m, read) - return touched -} - // touchedBy maps a merge's changed files onto the mesh's modules — **the one place it is done**, for the // merge handler, the release planner's what-if, the merge gate and a pull request's check alike (novox/hq // ADR 0238), so planning and gating cannot disagree about what a change touches. @@ -757,49 +458,6 @@ func inside(path, dir string) bool { return path == dir || strings.HasPrefix(path, dir+"/") } -// anyInside is whether any of these changed files is in a directory. -func anyInside(paths []string, dir string) bool { - for _, p := range paths { - if inside(p, dir) { - return true - } - } - return false -} - -// standsOnModule is whether an entry's build stood on the named module: by what its newest build -// recorded it was handed (`artifact-store:///@…`, the module's own artifact), or -// — for a module registered from a manifest and not yet built — by the base its manifest names. -func standsOnModule(e inventory.Entry, module string, against map[string][]string) bool { - if e.Manifest.Module == module { - return false - } - if e.Manifest.Build != nil { - for _, on := range e.Manifest.Build.On { - if on.Module == module { - return true - } - } - } - prefix := catalogue.ArtifactStoreScheme + module + "/" - for _, ref := range against[e.Manifest.Module] { - if strings.HasPrefix(ref, prefix) { - return true - } - } - return false -} - -// standsOn is whether anything in the set is built on the named module's artifacts. -func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool { - for _, e := range entries { - if standsOnModule(e, module, against) { - return true - } - } - return false -} - // isHistory is whether a merge made at mergedAt predates the last time the source was seen. A merge // with no time on it is taken as news: refusing it would silence a forge that says less. func isHistory(mergedAt string, seen time.Time) bool { @@ -813,52 +471,10 @@ func isHistory(mergedAt string, seen time.Time) bool { return at.Before(seen) } -// dependentsOf is every catalogued module that stands on one of the moved modules, directly or -// through another dependent, and is not itself among them — in the catalogue's order, so the -// answer is the same each time. A module standing on nothing that moved is left alone: a merge -// rebuilds what it changed and what is built on top of that, not the catalogue. -func dependentsOf(moved, entries []inventory.Entry, against map[string][]string) []inventory.Entry { - bases := map[string]bool{} - for _, e := range moved { - bases[e.Manifest.Module] = true - } - var out []inventory.Entry - taken := map[string]bool{} - for grew := true; grew; { - grew = false - for _, e := range entries { - name := e.Manifest.Module - if bases[name] || taken[name] { - continue - } - for base := range bases { - if standsOnModule(e, base, against) { - taken[name] = true - out = append(out, e) - grew = true - break - } - } - } - for _, e := range out { - bases[e.Manifest.Module] = true - } - } - return out -} - // moduleManifestFile is the file that says what a module is, in its directory (the build seat's // ManifestName). const moduleManifestFile = "module.json" -// deletedAtSource is whether a build failed because its module's manifest is not at its source any -// more — the build seat's own words (internal/builder) — which a merge that deleted the module causes -// when its announcer did not say which files went (ADR 0236). Such a module is not a failure of the plan. -func deletedAtSource(failed string) bool { - return strings.Contains(failed, "has no "+moduleManifestFile+" at ") && - strings.Contains(failed, "so there is nothing saying what it is") -} - // reachOfMerge is what a merge of these changed files into this branch would move, without acting: a pull request's // reach, as the check reckons it. func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository, @@ -1026,3 +642,11 @@ func sayUpgrade(module string, u inventory.Upgrade) string { return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+ "the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from) } + +// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: touchedBy's first answer +// (novox/hq issue 280, ADR 0238). +func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved, + read map[string][]inventory.ReadRepository) []inventory.Entry { + touched, _, _ := touchedBy(candidates, known, m, read) + return touched +} diff --git a/internal/inventory/build_lookups.go b/internal/inventory/build_lookups.go index 0f72e797..28647c48 100644 --- a/internal/inventory/build_lookups.go +++ b/internal/inventory/build_lookups.go @@ -24,11 +24,3 @@ func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[s } return i.chainBuildFingerprints(ctx, c, module) } - -// short is a commit as a person reads it. -func short(commit string) string { - if len(commit) > 8 { - return commit[:8] - } - return commit -} diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index ebc6ac2d..ff2a4995 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -903,17 +903,6 @@ type BusyWait struct { // BusyWaiting is the state of a container left running because it said busy (link.BusyWaiting). const BusyWaiting = "waiting" -// WaitingFor is the container of a module that waits because it said busy, and whether one does: a -// resource whose id is `.`. -func WaitingFor(busy []BusyWait, module string) (BusyWait, bool) { - for _, b := range busy { - if b.State == BusyWaiting && strings.HasPrefix(b.ID, module+".") { - return b, true - } - } - return BusyWait{}, false -} - // Waiting says whether any container waits because it said busy. func (d Doing) Waiting() bool { for _, b := range d.Busy { diff --git a/internal/link/bus.go b/internal/link/bus.go index c4fbdf27..01ba3e7d 100644 --- a/internal/link/bus.go +++ b/internal/link/bus.go @@ -26,11 +26,6 @@ type Bus interface { // nobody is obliged to act (ADR 0041). PublishEvent(ctx context.Context, key, source, node string, body []byte) error - // PublishDeclaration delivers one node what it should be. Addressed to that node alone: a - // declaration is not an event, and replaying yesterday's is actively harmful - // (design 29 §4, the *state* shape). - PublishDeclaration(ctx context.Context, node string, body []byte) error - // PublishSeatEvent states a fact under a role's own name, for the holder of that role. A // module's event is addressed to the module; a role's is addressed to the role, so it keeps // meaning when the holder changes (novox/hq ADR 0121, ADR 0129). @@ -101,11 +96,6 @@ func SeatEventSubject(seat, event string) string { return "mesh.seat." + seat + ".event." + event } -// DeclareSubject is where one node's declaration lands. Last-per-subject on the NODES stream, so -// a node that was away gets exactly the current one and a replayed older one is refused by -// sequence — the wire-level answer to novox/hq issue 107. -func DeclareSubject(node string) string { return "mesh.node." + node + ".declare" } - func (b OverNATS) PublishEvent(ctx context.Context, key, source, node string, body []byte) error { id, err := eventID() if err != nil { @@ -192,14 +182,6 @@ func (b OverNATS) MaxPayload() int64 { return b.Conn.MaxPayload() } -func (b OverNATS) PublishDeclaration(ctx context.Context, node string, body []byte) error { - _, err := b.JS.Publish(DeclareSubject(node), body, nats.Context(ctx)) - if err != nil { - return fmt.Errorf("declaring to %s: %w", node, err) - } - return nil -} - // ToolSubject is where a module answers. Derived from the module and the tool, so a caller names // what it wants rather than where it lives. func ToolSubject(module, tool string) string { return "mesh.mod." + module + ".tool." + tool } diff --git a/internal/link/bus_test.go b/internal/link/bus_test.go index 24387a45..4b99616e 100644 --- a/internal/link/bus_test.go +++ b/internal/link/bus_test.go @@ -12,16 +12,6 @@ import ( // // go test ./internal/link/ -run TestTheNatsBus (each test on a bus of its own: internal/testbus) -// The subject a declaration lands on is one node's, and nothing else's — the state shape. -func TestADeclarationIsAddressedToOneNode(t *testing.T) { - if got := DeclareSubject("anchor"); got != "mesh.node.anchor.declare" { - t.Fatalf("a declaration would go to %q", got) - } - if DeclareSubject("anchor") == DeclareSubject("laptop") { - t.Fatal("two nodes share a declaration subject, so each would apply the other's") - } -} - // A module cannot emit under another's name: the subject is derived from the source, and the // server's permissions make that subject the authority. func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) { diff --git a/internal/link/declare.go b/internal/link/declare.go deleted file mode 100644 index be53528b..00000000 --- a/internal/link/declare.go +++ /dev/null @@ -1,94 +0,0 @@ -package link - -import ( - "context" - "encoding/json" - "fmt" - "time" -) - -// Signer is whatever holds the control plane's signing key. -type Signer interface { - Sign(ctx context.Context, message []byte) ([]byte, error) -} - -// ActingGate is what every send passes before it is made (novox/hq to-be 45 §6): whether this process -// may act under the controller's lease. Set by the controller; nil passes every send — a test, a tool. -var ActingGate func(ctx context.Context) error - -// Declare sends a node what it should be, signed. -// -// The signature is over the declaration exactly as it is published — the same bytes the node -// verifies. Anything that re-encoded between here and there would produce a signature over -// something else, and the node would refuse a declaration that was genuinely the mesh's. -// -// Published to the node's own queue, which its account alone may read. -func Declare(ctx context.Context, bus Bus, signer Signer, node string, - declaration []byte, timeout time.Duration) error { - - if !json.Valid(declaration) { - return fmt.Errorf("refusing to send %s something that is not a declaration", node) - } - // **At the send, not only where it was composed**: a lease lost between the two stops this one. - if ActingGate != nil { - if err := ActingGate(ctx); err != nil { - return fmt.Errorf("%s was not sent its declaration: %w", node, err) - } - } - - signature, err := signer.Sign(ctx, declaration) - if err != nil { - return fmt.Errorf("cannot sign a declaration for %s: %w", node, err) - } - - body, err := Envelope(declaration, signature) - if err != nil { - return err - } - - // **Measured before it is sent** (novox/hq issue 1689): the bare bus error said only "maximum payload - // exceeded", on 2026-10-21 for the control node's declaration, 2 KB over. The limit is the connection's own. - if limit := payloadLimitOf(bus); limit > 0 { - size := int64(len(body)) - if DeclarationMeasured != nil { - DeclarationMeasured(ctx, node, size, limit) - } - if size > limit { - return fmt.Errorf("%s's declaration is %d bytes on the wire (%d bytes of declaration, signed and encoded), "+ - "over the bus's largest message of %d bytes by %d: it cannot be sent (novox/hq issue 1689)", - node, size, len(declaration), limit, size-limit) - } - } - - publish, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - return bus.PublishDeclaration(publish, node, body) -} - -// Envelope is the message a declaration goes on the bus as: the declaration and its signature, each as base64 in one JSON -// document. Its length is what the bus's largest message bounds. -func Envelope(declaration, signature []byte) ([]byte, error) { - return json.Marshal(Signed{Declaration: declaration, Signature: signature}) -} - -// NearTheLimit is the share of the bus's largest message at which a declaration is said to be near it: three quarters, so that -// a declaration which should stay far from the limit raises the condition while there is still room to act (novox/hq ADR 0486 -// §8; the first fix of issue 1689 warned at nine tenths while every file was still inline). -const NearTheLimit = 0.75 - -// DeclarationMeasured is told, at every send of a declaration, its size on the wire and the bus's largest message, -// so the controller raises a condition when it nears the limit and clears it when it is back under (issue 1689). -// Set by the controller; nil in a test or a tool. -var DeclarationMeasured func(ctx context.Context, node string, size, limit int64) - -// payloadLimiter is a bus that knows its largest message. -type payloadLimiter interface{ MaxPayload() int64 } - -// payloadLimitOf is the bus's largest message in bytes, or 0 when the bus does not say. -func payloadLimitOf(bus Bus) int64 { - if l, ok := bus.(payloadLimiter); ok { - return l.MaxPayload() - } - return 0 -} diff --git a/internal/link/declare_room_integration_test.go b/internal/link/declare_room_integration_test.go deleted file mode 100644 index eb607b2f..00000000 --- a/internal/link/declare_room_integration_test.go +++ /dev/null @@ -1,52 +0,0 @@ -//go:build integration - -package link - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/testbus" -) - -// Against a real bus whose largest message is small: the limit is read from the connection, the one that fits goes -// through, the one over is refused by us with its size and the limit, not by the client's bare error (issue 1689). -func TestADeclarationIsMeasuredAgainstTheRealBusLimit(t *testing.T) { - s := testbus.StartWithMaxPayload(t, 2048) - conn, err := nats.Connect(s.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - js, err := conn.JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := js.AddStream(&nats.StreamConfig{Name: "NODES", Subjects: []string{"mesh.node.*.declare"}}); err != nil { - t.Fatal(err) - } - bus := OverNATS{Conn: conn, JS: js} - if bus.MaxPayload() != 2048 { - t.Fatalf("the limit read from the connection is %d, expected 2048", bus.MaxPayload()) - } - var told []int64 - was := DeclarationMeasured - DeclarationMeasured = func(_ context.Context, _ string, size, limit int64) { told = append(told, size, limit) } - t.Cleanup(func() { DeclarationMeasured = was }) - - fits := []byte(`{"a":"` + strings.Repeat("x", 600) + `"}`) - if err := Declare(t.Context(), bus, fixedSigner{}, "novox", fits, 5*time.Second); err != nil { - t.Fatalf("a declaration under the limit: %v", err) - } - over := []byte(`{"a":"` + strings.Repeat("x", 1600) + `"}`) - err = Declare(t.Context(), bus, fixedSigner{}, "novox", over, 5*time.Second) - if err == nil || !strings.Contains(err.Error(), "2048 bytes") || strings.Contains(err.Error(), "maximum payload exceeded") { - t.Fatalf("a declaration over the limit: %v", err) - } - if len(told) != 4 || told[1] != 2048 || told[2] <= 2048 { - t.Fatalf("measured %v", told) - } -} diff --git a/internal/link/declare_room_test.go b/internal/link/declare_room_test.go deleted file mode 100644 index 4339277b..00000000 --- a/internal/link/declare_room_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package link - -import ( - "context" - "strings" - "testing" - "time" -) - -type roomBus struct { - Bus - limit int64 - sent [][]byte -} - -func (b *roomBus) MaxPayload() int64 { return b.limit } -func (b *roomBus) PublishDeclaration(_ context.Context, _ string, body []byte) error { - b.sent = append(b.sent, body) - return nil -} - -type fixedSigner struct{} - -func (fixedSigner) Sign(context.Context, []byte) ([]byte, error) { return []byte("sig"), nil } - -// A declaration is measured as it goes on the wire against the bus's own limit; one over it is refused saying size and -// limit, and what the measure sees is told to whoever raises a condition (issue 1689). -func TestADeclarationOverTheBusLimitIsRefusedSayingSizeAndLimit(t *testing.T) { - var seen []int64 - was := DeclarationMeasured - DeclarationMeasured = func(_ context.Context, node string, size, limit int64) { - if node != "novox" || limit != 400 { - t.Errorf("measured %s against %d", node, limit) - } - seen = append(seen, size) - } - t.Cleanup(func() { DeclarationMeasured = was }) - - bus := &roomBus{limit: 400} - small := []byte(`{"a":"` + strings.Repeat("x", 100) + `"}`) - if err := Declare(t.Context(), bus, fixedSigner{}, "novox", small, time.Second); err != nil || len(bus.sent) != 1 { - t.Fatalf("a small declaration: %v, %d sent", err, len(bus.sent)) - } - if int64(len(bus.sent[0])) != seen[0] { - t.Fatalf("measured %d bytes, sent %d", seen[0], len(bus.sent[0])) - } - - big := []byte(`{"a":"` + strings.Repeat("x", 300) + `"}`) - err := Declare(t.Context(), bus, fixedSigner{}, "novox", big, time.Second) - if err == nil || !strings.Contains(err.Error(), "400 bytes") || !strings.Contains(err.Error(), "novox") || - !strings.Contains(err.Error(), "over") || strings.Contains(err.Error(), "maximum payload exceeded") { - t.Fatalf("an oversize declaration: %v", err) - } - if len(bus.sent) != 1 || len(seen) != 2 || seen[1] <= 400 { - t.Fatalf("it was sent (%d) or not measured (%v)", len(bus.sent), seen) - } -} diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 9ee84314..24e4b085 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -11,7 +11,6 @@ import ( "fmt" "log" "sort" - "strings" "time" "github.com/novox/mesh-controller/internal/broker" @@ -268,16 +267,6 @@ var _ Enroller = Enrolment{} // A node states; the owning context writes (novox/hq ADR 0006). What a node says it applied is // its own account of its own machine, kept as a copy for recovery — so this writes it down and // decides nothing from it. -// Outstanding is the declaration the mesh last sent a node, so a report about an older one is not -// acted on (design 25 §3, window.go). -// -// **Here rather than on Listener.** A report is recorded by whatever keeps records, and a great -// many things that record reports have no idea what was sent — every test in this package among -// them. So the serving loop asks for this when the listener happens to be able to answer, and -// where it cannot, a report has nothing to be stale against and is simply acted on. -func (e Enrolment) Outstanding(ctx context.Context, node string) (string, error) { - return e.Inventory.Outstanding(ctx, node) -} func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err error) { // A store that could not be asked right now is said as such, so the report is kept for @@ -444,11 +433,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err // arrives; the digest the mesh last sent decides nothing for it. **By digest, from an older // node-engine**: its report claims no order, and the rule of issue 267 stands for it. // - // And whether the node-engine reads an epoch in a declaration, from every account it gives: the - // mesh sends one only to a machine that said so, and one rolled back says so no longer. - if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil { - return false, err - } if report.Ordered() { account := AccountOf(report) news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing, @@ -467,17 +451,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err return false, err } } else { - if report.Declared != "" { - sent, err := e.Inventory.Outstanding(ctx, report.Node) - if err != nil { - return false, err - } - if Superseded(report.Declared, sent) { - log.Printf("kept what %s says about the machine, and not its account of declaration %s: "+ - "the mesh has sent it %s since", report.Node, short(report.Declared), short(sent)) - return false, e.Inventory.Seen(ctx, node.ID) - } - } // **Whether this is news** is the store's answer: it holds the previous report, and a machine // that reconciles every minute says the same thing until something changes (novox/hq ADR 0134). news, err = e.Inventory.RecordDoing(ctx, node.ID, doing) @@ -493,25 +466,3 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err } return news, e.Inventory.RecordOwned(ctx, node.ID, report.Applied) } - -// busyWords is every container waiting because it said busy, in a line (novox/hq ADR 0335). -func busyWords(busy []inventory.BusyWait) string { - var out []string - for _, b := range busy { - if b.State == inventory.BusyWaiting { - out = append(out, fmt.Sprintf("%s since %s, at the latest until %s", b.Container, - b.Since.UTC().Format(time.RFC3339), b.Until.UTC().Format(time.RFC3339))) - } - } - return strings.Join(out, "; ") -} - -// replacedAfterWaiting says a report replaced a container that had waited because it was busy. -func replacedAfterWaiting(busy []inventory.BusyWait) bool { - for _, b := range busy { - if b.State != inventory.BusyWaiting { - return true - } - } - return false -} diff --git a/internal/link/generation_test.go b/internal/link/generation_test.go deleted file mode 100644 index b93e41f4..00000000 --- a/internal/link/generation_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package link - -import ( - "encoding/json" - "testing" -) - -// A report's word on the assignment generation (novox/hq issue 234), as mesh-host's report writes it: -// `reads_generation` on every report of a node-engine that reads one, and `older_generation` on a refusal -// of a declaration composed from an older generation than the machine applied. -func TestTheGenerationOnTheWire(t *testing.T) { - raw := []byte(`{"node":"anchor","refused":"older generation","declared":"d12","epoch":57,"sequence":12,` + - `"report_sequence":8,"reads_generation":true,"older_generation":{"generation":38,"applied":40}}`) - var r Report - if err := json.Unmarshal(raw, &r); err != nil { - t.Fatal(err) - } - if !r.ReadsGeneration || r.OlderGeneration == nil || *r.OlderGeneration != (GenerationRefusal{38, 40}) || - r.Sequence != 12 { - t.Fatalf("a node-engine's generation refusal reads as %+v", r) - } - // Not a refusal by the lease's epoch: S13 counts those, and this one is raised naming its sender. - if r.StaleRefusalOf() { - t.Errorf("a generation refusal reads as a stale writer's") - } - var older Report - if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil { - t.Fatal(err) - } - if older.ReadsGeneration || older.OlderGeneration != nil { - t.Fatalf("a node-engine that never said it reads a generation reads as one that does: %+v", older) - } -} - -// 1145/AC3: a report's word that the node-engine reads a module per resource and a build per module, as mesh-host's -// report writes it: `reads_modules`, on every report of a node-engine that reads them. -func TestWhetherTheNodeEngineReadsModulesIsOnItsReport(t *testing.T) { - var reads, older Report - if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":8,"reads_modules":true}`), &reads); err != nil { - t.Fatal(err) - } - if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"report_sequence":3}`), &older); err != nil { - t.Fatal(err) - } - if !reads.ReadsModules || older.ReadsModules { - t.Fatalf("1145/AC3: reads %v, an older node-engine reads %v", reads.ReadsModules, older.ReadsModules) - } -} diff --git a/internal/link/heard_order_test.go b/internal/link/heard_order_test.go index de59da59..3ef40105 100644 --- a/internal/link/heard_order_test.go +++ b/internal/link/heard_order_test.go @@ -1,7 +1,6 @@ package link_test import ( - "testing" "time" "github.com/novox/mesh-controller/internal/link" @@ -25,27 +24,6 @@ func ordered(declared string, epoch, sequence, report int64, applied ...string) // A node-engine rolled back to one that orders nothing: its account is taken by the digest rule, the // order kept is cleared, and it is no longer sent an epoch. -// A stale refusal names its writer by the epoch the refused declaration claimed, and a refusal whose -// own report was lost is still counted from the machine's own tally. -func TestAStaleRefusalIsCountedByItsWriter(t *testing.T) { - count := link.NewRefusalCount() - now := time.Now() - refusal := link.Report{Node: "anchor", Refused: "older", Order: link.Order{Epoch: 41, Sequence: 11}, - OlderThan: &link.Order{Epoch: 57, Sequence: 12}, ReportSequence: 9, RefusedOlder: 3} - if !refusal.StaleRefusalOf() { - t.Fatal("a refusal naming the order it holds is not stale") - } - count.Lifetime("anchor", 2, now) // the machine's tally, as the controller first heard it - count.Refused(link.Refusal{Writer: link.WriterEpoch(refusal.Epoch), Epoch: refusal.Epoch, Receiver: "anchor", At: now}) - count.Lifetime("anchor", refusal.RefusedOlder, now) - count.Lifetime("anchor", 5, now) // two more refused, their reports lost - got := count.Within(now.Add(-time.Minute)) - if len(got) != 2 || got[0].Count != 2 || got[0].Epoch != 0 || got[1].Writer != "the controller of epoch 41" || - got[1].Count != 1 || got[1].Receivers[0] != "anchor" { - t.Fatalf("the refusals by writer are %+v", got) - } -} - // countFor is how many refusals of a writer this process has heard in the last minute. func countFor(writer string) int { for _, w := range link.StaleRefusals.Within(time.Now().Add(-time.Minute)) { diff --git a/internal/link/order.go b/internal/link/order.go index 311bd0fd..ed8de121 100644 --- a/internal/link/order.go +++ b/internal/link/order.go @@ -2,68 +2,22 @@ package link import "strconv" -// The order a declaration carries, and the order a report about one carries back (novox/hq to-be 45 -// §6, ADR 0227 rule 2). +// The order of a node's facts (novox/hq ADR 0539 §9, to-be 45 §6, ADR 0227 rule 2). // -// **This file is the controller's side of the contract with the node-engine for Phase 2**, and the one -// place it is written here. It mirrors mesh-host internal/link/messages.go (`Order`, and the report's -// `report_sequence`, `older_than`, `refused_older`) field for field and rule for rule, as Report has -// always mirrored the host's report: a key added on one side and not the other is a key the other side -// does not know exists. -// -// **The wire.** -// -// - A declaration's order is two top-level keys of its signed envelope, beside "declaration" and -// "resources": `sequence` (since issue 107) and `epoch` (new). Inside the signed bytes, so a message -// cannot be given a newer order than the controller gave it. -// - A report carries the order of the declaration it is about as the same two top-level keys, beside -// "declared"; `report_sequence`, the node-engine's own number for the report; `older_than`, on a -// refusal of a declaration older than one it applied, the order of the one it holds; and -// `refused_older`, how many it has refused so, ever, on every report. -// -// Every key is optional and absent when zero, and zero is "no order claimed", never "first". So each -// side reads the other's older shape: -// -// - **An older node-engine with this controller.** It decodes a declaration strictly and refuses a key -// it does not know, whole — so `epoch` is sent only to a machine whose latest report carried a -// `report_sequence`, which a node-engine that reads the epoch always does. Until then it is sent the -// sequence alone, as today; its reports carry no report sequence and are judged by the digest they -// name, as today (issue 267). -// - **A newer node-engine with an older controller.** It is sent no epoch, which claims none: it -// refuses nothing by epoch. The keys it adds to a report are fields the older controller ignores. -// - **A controller rolled back to a build without the lease** sends no epoch again and is not refused -// for it: a node-engine refuses by epoch only when both declarations claim one. That gives up the -// epoch's protection while such a build runs — the price of a rollback that cannot strand every -// machine. -// -// **Epoch** is the controller lease's epoch (to-be 45 §6): the lease bucket's revision at which the -// instance that composed the declaration took the lease. It only grows: a later holder's is higher, and -// the controller never issues one at or under the highest it has issued (internal/lease, the floor). -// **Sequence** is the declaration's number for that machine, one higher every send, taken from the -// controller's store before the declaration is composed (issues 107, 204). +// A node-engine says its facts (profile, firewall found, reachable, tunnel, witness, rollbacks, rekey, health) in a report +// on the bus, and orders them itself: `epoch` and `sequence` are the node-engine's own (Writer `node-engine`), and +// `report_sequence` numbers each report it makes. The controller keeps the newest account of a machine and refuses an older +// one, said and counted (rule 2): by epoch, then sequence, then report sequence, each compared only where both claim one. +// Zero is "no order claimed", never "first". The controller sends a machine no declaration to be ordered by: the order here +// is only ever the machine's own. -// Order is where a declaration stands among everything the mesh has sent a machine: the lease epoch it -// was sent under and its sequence. Zero in either claims none. +// Order is where a node-engine's account stands among its own: the epoch and sequence the node-engine counts. Zero in either +// claims none. type Order struct { Epoch int64 `json:"epoch,omitempty"` Sequence int64 `json:"sequence,omitempty"` } -// Older is the node-engine's rule, as mesh-host states it: a declaration of this order is older than -// one of order `than` the machine applied **only when both claim an epoch** — and then when its epoch -// is lower, or its epoch is the same and its sequence lower (both claimed). A higher epoch is a new lease -// holder and is never older, whatever its sequence. Here so the controller's tests state what the -// machines will do with what it sends. -func (o Order) Older(than Order) bool { - if o.Epoch <= 0 || than.Epoch <= 0 { - return false - } - if o.Epoch != than.Epoch { - return o.Epoch < than.Epoch - } - return o.Sequence > 0 && than.Sequence > 0 && o.Sequence < than.Sequence -} - // Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a Stringer // promoted onto every report would print each one as its order alone. func (o Order) Words() string { @@ -91,17 +45,6 @@ func AccountOf(r Report) Account { return Account{Order: r.Order, ReportSequence // before (issue 267). func (r Report) Ordered() bool { return r.ReportSequence > 0 } -// ReadsEpoch says the node-engine that made the report takes an epoch in a declaration: one that orders -// its reports does (mesh-host: "its presence also says this host reads a declaration's epoch"). -func (r Report) ReadsEpoch() bool { return r.ReportSequence > 0 } - -// StaleRefusalOf says a report refuses a declaration older than one the machine applied: the node-engine -// names the order it holds (`older_than`), or, from a node-engine older than that, says so in the words -// of its sequence refusal. -func (r Report) StaleRefusalOf() bool { - return r.OlderThan != nil || (r.Refused != "" && IsStaleRefusal(r.Refused)) -} - // OlderThan is the controller's rule (to-be 45 §6): whether this account is older than the one kept for // the machine — **by epoch, then sequence, then report sequence**, each compared only where both claim // one. An account of a declaration of an older epoch is refused; of an older sequence, refused; an older diff --git a/internal/link/order_test.go b/internal/link/order_test.go index a215ba08..d0a11101 100644 --- a/internal/link/order_test.go +++ b/internal/link/order_test.go @@ -5,34 +5,8 @@ import ( "testing" ) -// The contract of order.go, case by case: what a node-engine refuses of a declaration (the rule -// mesh-host states, restated so the controller's tests say what the machines do with what it sends), -// what the controller refuses of a report, and the bytes on the wire both ways. - -func TestADeclarationIsOlderOnlyWhenBothClaimAnEpoch(t *testing.T) { - for _, c := range []struct { - name string - arriving Order - held Order - older bool - }{ - {"a newer sequence of the same epoch", Order{57, 12}, Order{57, 11}, false}, - {"the same declaration again (a reconcile)", Order{57, 12}, Order{57, 12}, false}, - {"a lower sequence of the same epoch", Order{57, 11}, Order{57, 12}, true}, - // Issue 204: a controller that lost its lease goes on sending. - {"an older epoch, whatever its sequence", Order{41, 99}, Order{57, 12}, true}, - {"a newer epoch, whatever its sequence", Order{57, 3}, Order{41, 99}, false}, - {"no epoch arriving: a rolled-back controller is not stranded", Order{0, 11}, Order{57, 12}, false}, - {"no epoch held: what the machine held before any lease", Order{57, 11}, Order{0, 12}, false}, - {"no order at all", Order{}, Order{57, 12}, false}, - } { - t.Run(c.name, func(t *testing.T) { - if got := c.arriving.Older(c.held); got != c.older { - t.Fatalf("%s against %s: older %v, want %v", c.arriving.Words(), c.held.Words(), got, c.older) - } - }) - } -} +// The contract of order.go, case by case: what the controller refuses of a node-engine's facts (the node-engine's own order) +// and the bytes on the wire. func TestAnAccountIsOlderByEpochThenSequenceThenReportSequence(t *testing.T) { for _, c := range []struct { @@ -78,7 +52,7 @@ func TestTheOrderOnTheWire(t *testing.T) { r.RefusedOlder != 3 { t.Fatalf("a node-engine's stale refusal reads as %+v", r) } - if !r.Ordered() || !r.ReadsEpoch() || !r.StaleRefusalOf() { + if !r.Ordered() { t.Fatalf("a node-engine that orders its reports reads as one that does not: %+v", r) } @@ -87,15 +61,9 @@ func TestTheOrderOnTheWire(t *testing.T) { if err := json.Unmarshal([]byte(`{"node":"anchor","applied":["a"],"declared":"d1"}`), &older); err != nil { t.Fatal(err) } - if older.Ordered() || older.ReadsEpoch() || older.StaleRefusalOf() { + if older.Ordered() { t.Fatalf("an older node-engine's report reads as ordered: %+v", older) } - // Its stale refusal, in the words it has always used, is still one. - older.Refused = "this declaration " + StaleRefusal + ": it is sequence 3" - if !older.StaleRefusalOf() { - t.Fatal("an older node-engine's refusal of an older sequence does not read as stale") - } - // And a report with no order puts no order key on the wire. raw, err := json.Marshal(Report{Node: "anchor", Declared: "d1"}) if err != nil { diff --git a/internal/link/receive.go b/internal/link/receive.go index 30e053cf..aeb12417 100644 --- a/internal/link/receive.go +++ b/internal/link/receive.go @@ -121,15 +121,3 @@ type Inbound interface { // Close lets go of whatever the implementation holds. Close() } - -// Outstanding answers which declaration the mesh last sent a node — the digest, not the -// declaration. -// -// **Asked before the store is waited on** (design 25 §3): a report about a declaration the mesh -// has already moved past is not worth holding a slot in the window that a current message needs. -// It is a separate interface from Listener rather than a method on it, because a controller that -// only publishes needs neither and something that records reports need not also be able to say -// what was sent. -type Outstanding interface { - Outstanding(ctx context.Context, node string) (string, error) -} diff --git a/internal/link/receive_nats_integration_test.go b/internal/link/receive_nats_integration_test.go index 59c89e1d..b2ee6f5a 100644 --- a/internal/link/receive_nats_integration_test.go +++ b/internal/link/receive_nats_integration_test.go @@ -102,25 +102,6 @@ func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T }) } -func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) { - js := aBus(t) - store := &sentAndHeardSafely{sent: "d2"} - _, stop := servingOn(t, js, store) - defer stop() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - eventually(t, "the superseded report leaving the stream", func() bool { - info, err := js.Context().StreamInfo("CONTROL") - return err == nil && info.State.Msgs == 0 - }) - if store.count() != 0 { - t.Fatalf("a report about a superseded declaration was acted on") - } -} - func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) { js := aBus(t) s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, diff --git a/internal/link/serve.go b/internal/link/serve.go index 4aeff2dd..3e0ccf42 100644 --- a/internal/link/serve.go +++ b/internal/link/serve.go @@ -292,26 +292,6 @@ func (s *Server) giveUpAfter() time.Duration { return s.giveUp } -// outstanding is the digest of the declaration the mesh last sent a node. -// -// Nothing is guessed when it cannot be answered: a listener that keeps no record of what was sent -// gives a report nothing to be stale against, and a store that cannot be read will hold the report -// anyway, so there is nothing for the check to decide. -func (s *Server) outstanding(ctx context.Context, node string) string { - if node == "" { - return "" - } - asks, ok := s.listener.(Outstanding) - if !ok { - return "" - } - digest, err := asks.Outstanding(ctx, node) - if err != nil { - return "" - } - return digest -} - // toolsHeartbeat records that a machine's node tools were heard from (novox/hq to-be 45 S11), and // nothing else: in this process's memory, for the watchdog — the next one is a minute away. func (s *Server) toolsHeartbeat(m Control) { @@ -343,25 +323,8 @@ func (s *Server) reported(ctx context.Context, m Control) { what := fmt.Sprintf("%s's report of declaration %s", report.Node, report.Declared) if s.listener != nil { - // **Asked before the store, not after** (design 25 §3). A report about a declaration the - // mesh has moved past would otherwise wait out a restarting store to be written and then - // overwrite what the node is doing now — and on the bus being built, where the holding is - // the server's, it comes back after the newer was applied whatever the controller does. - outstanding := s.outstanding(ctx, report.Node) - declaredIn := staleAgainst(report) - if report.Ordered() { - // Kept by its order, not by the digest the mesh last sent (novox/hq to-be 45 §6): the - // listener refuses an older account under the one rule, whatever it is about. - declaredIn = "" - } - if Superseded(declaredIn, outstanding) { - s.log.Printf("set aside %s: the mesh has moved past that declaration", what) - _ = m.Took() - return - } - news, err := s.listener.Heard(context.Background(), report) - switch s.decide(ctx, m, what, declaredIn, outstanding, err) { + switch s.decide(ctx, m, what, "", "", err) { case Hold: // Held, not settled, while the store cannot take it: the node reports an apply once, // and a report lost here is a node the mesh never hears from again — the store @@ -396,27 +359,6 @@ func (s *Server) reported(ctx context.Context, m Control) { _ = m.Took() } -// staleAgainst is the declaration a report may be judged stale against, and it is empty for a -// report that is not only an account of an apply. -// -// **A report carries two different things, and only one of them is about a declaration.** What the -// node applied is; what the machine *is* — the tunnel it took over, the ports its own bundle holds, -// what an adopted node found and is keeping, a node moving its overlay key — is not. Those reach -// the mesh on a report because a report is the message a node sends, and nowhere else: a rekey -// dropped as stale is a node whose overlay key never moves, and no retry is coming, because the -// node said it once. -// -// So staleness is asked only of a report that is purely an apply's account. The rest is acted on -// whenever it arrives, which is the behaviour the mesh has had all along. -func staleAgainst(report Report) string { - if report.Rekey != nil || report.Tunnel != nil || len(report.Held) > 0 || - report.Firewall != "" || len(report.Reachable) > 0 || len(report.Carried) > 0 || - len(report.Outward) > 0 { - return "" - } - return report.Declared -} - func (s *Server) enrolling(ctx context.Context, m Control) { reply := EnrolReply{Refusal: "that token cannot be used"} diff --git a/internal/link/signer.go b/internal/link/signer.go new file mode 100644 index 00000000..f083333c --- /dev/null +++ b/internal/link/signer.go @@ -0,0 +1,12 @@ +package link + +import "context" + +// Signer is whatever holds the control plane's signing key. +type Signer interface { + Sign(ctx context.Context, message []byte) ([]byte, error) +} + +// ActingGate is what every act passes before it is made (novox/hq to-be 45 §6): whether this process may act under the +// controller's lease. Set by the controller; nil passes every act — a test, a tool. +var ActingGate func(ctx context.Context) error diff --git a/internal/link/stale_report_test.go b/internal/link/stale_report_test.go deleted file mode 100644 index cf9f303b..00000000 --- a/internal/link/stale_report_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package link - -import ( - "context" - "errors" - "testing" -) - -// Supersession as a check rather than a memory (design 25 §3). -// -// Holding a message in memory let the controller drop an older report when a newer one for the -// same node arrived. On the bus being built the message belongs to the server and comes back -// whatever happened meanwhile — so the older report is redelivered *after* the newer was applied, -// and acting on it would undo the newer. -// -// The answer was already in the message: a report carries the digest of the declaration it is -// about, so "is this the past?" is a question the message answers. - -// sentAndHeard records reports and knows what was last sent, which is the pair the check needs. -type sentAndHeard struct { - sent string - heard []Report - err error -} - -func (s *sentAndHeard) Heard(_ context.Context, r Report) (bool, error) { - if s.err != nil { - return false, s.err - } - s.heard = append(s.heard, r) - return true, nil -} - -func (s *sentAndHeard) Outstanding(context.Context, string) (string, error) { return s.sent, nil } - -// A report about a declaration the mesh has moved past is settled and not acted on. Settled rather -// than dropped, because there is nothing wrong with the message — it is simply the past, and -// redelivering it for ever is worse than letting it go. -func TestAReportAboutASupersededDeclarationIsNotActedOn(t *testing.T) { - store := &sentAndHeard{sent: "d2"} - s, in := serving() - s.listener = store - to := &settled{} - s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1"))) - - if len(store.heard) != 0 { - t.Fatalf("a report about a superseded declaration was acted on: %+v", store.heard) - } - if !to.acked { - t.Fatalf("a superseded report was not settled, so it comes back for ever: %+v", *to) - } -} - -// The report about the declaration that *is* outstanding is acted on, and so is one from a node -// the mesh has no digest for — an older host that says nothing about which declaration it applied -// has nothing to be judged against, and refusing it would silence every node built before reports -// carried the digest. -func TestAReportAboutTheOutstandingDeclarationIsActedOn(t *testing.T) { - for _, c := range []struct{ what, sent, declared string }{ - {"the one outstanding", "d2", "d2"}, - {"a report that says nothing about which", "d2", ""}, - {"a node nothing was ever sent", "", "d1"}, - } { - store := &sentAndHeard{sent: c.sent} - s, in := serving() - s.listener = store - to := &settled{} - s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", c.declared))) - if len(store.heard) != 1 || !to.acked { - t.Errorf("%s: was not acted on and acknowledged: heard %+v, settled %+v", - c.what, store.heard, *to) - } - } -} - -// **Staleness is decided before the store is waited on**, not after: a redelivery that lost its -// race is not worth holding a slot in the window that a current message needs. -func TestASupersededReportIsNotHeldForTheStore(t *testing.T) { - store := &sentAndHeard{sent: "d2", err: errors.Join(ErrTryAgain, errors.New("starting up"))} - s, in := serving() - s.listener = store - to := &settled{} - s.act(context.Background(), in.sends(t, to, KindReport, aReport("anchor", "d1"))) - if !to.acked || len(in.held) != 0 { - t.Fatalf("a superseded report waited for the store: %+v, %d held", *to, len(in.held)) - } -} - -// **Half of a report is not about a declaration, and that half is never stale.** -// -// What the machine *is* — the tunnel it took over, the ports its own bundle holds, what an adopted -// node found and is keeping, a node moving its overlay key — reaches the mesh on a report and -// nowhere else. A rekey set aside as stale is a node whose overlay key never moves, and no retry is -// coming, because the node said it once. So a report carrying any of these is acted on whenever it -// arrives, however far the mesh has moved on. -func TestAReportCarryingWhatOnlyTheNodeKnowsIsActedOnHoweverOldItIs(t *testing.T) { - for _, c := range []struct { - what string - report Report - }{ - {"a rekey", Report{Node: "anchor", Declared: "d1", - Rekey: &Rekey{Previous: "k1", OverlayKey: "k2"}}}, - {"the tunnel it carried", Report{Node: "anchor", Declared: "d1", - Tunnel: &CarriedTunnel{Interface: "wg0", State: "taken"}}}, - {"what an adopted node holds", Report{Node: "anchor", Declared: "d1", - Held: []Held{{ID: "conf", Module: "web", Kind: "file"}}}}, - {"the firewall it found", Report{Node: "anchor", Declared: "d1", Firewall: "ufw"}}, - {"what is reachable on it", Report{Node: "anchor", Declared: "d1", - Reachable: []Reach{{Protocol: "tcp", Port: 443}}}}, - {"the ports its own bundle holds", Report{Node: "anchor", Declared: "d1", - Carried: []int{5432}}}, - } { - store := &sentAndHeard{sent: "d9"} - s, in := serving() - s.listener = store - to := &settled{} - s.act(context.Background(), in.sends(t, to, KindReport, c.report)) - if len(store.heard) != 1 { - t.Errorf("%s was set aside as stale, and the mesh will never hear it again: %+v", - c.what, *to) - } - } -}