diff --git a/cmd/mesh-controller/composed_judge.go b/cmd/mesh-controller/composed_judge.go index 742c1070..49f7db20 100644 --- a/cmd/mesh-controller/composed_judge.go +++ b/cmd/mesh-controller/composed_judge.go @@ -192,12 +192,20 @@ func resourcesByID(c sdkdelivery.Composed) map[string]*judgedResource { return out } +// compactComposed is a resource's bytes canonical: its keys sorted, its spacing dropped, its numbers as written, so +// two compositions that say the same compare equal however each was written. func compactComposed(raw []byte) []byte { - var b bytes.Buffer - if json.Compact(&b, raw) != nil { + var v any + d := json.NewDecoder(bytes.NewReader(raw)) + d.UseNumber() + if err := d.Decode(&v); err != nil { return raw } - return b.Bytes() + out, err := json.Marshal(v) + if err != nil { + return raw + } + return out } // changedOf are the ids of the resources of the kinds counted that are changed, added or removed, sorted. diff --git a/cmd/mesh-controller/composed_judge_test.go b/cmd/mesh-controller/composed_judge_test.go new file mode 100644 index 00000000..bde7bd8e --- /dev/null +++ b/cmd/mesh-controller/composed_judge_test.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// mailOn is a composition of mesh-mail on ace at fingerprint c, from the given resources, each saying its module. +func mailOn(t *testing.T, c byte, data bool, resources ...string) sdkdelivery.Composed { + t.Helper() + out := sdkdelivery.Composed{Node: "ace", Module: "mesh-mail", FP: sdkdelivery.SourcePrefix2 + strings.Repeat(string(c), 64), Data: data} + for _, r := range resources { + out.Resources = append(out.Resources, json.RawMessage(r)) + } + return out +} + +const ( + mailContainer = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:1111111111111111111111111111111111111111111111111111111111111111","volumes":["/var/lib/mesh-mail/spool:/spool"]}` + mailContainer2 = `{"id":"mesh-mail.server","type":"container","module":"mesh-mail","name":"mail","image":"mail@sha256:2222222222222222222222222222222222222222222222222222222222222222","volumes":["/var/lib/mesh-mail/spool:/spool"]}` + mailSpool = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/var/lib/mesh-mail/spool"}` + mailSpoolMoved = `{"id":"mesh-mail.spool","type":"directory","module":"mesh-mail","path":"/srv/mesh-mail/spool"}` + mailTools = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"a"}` + mailTools2 = `{"id":"mesh-mail.tools","type":"file","module":"mesh-mail","path":"/usr/lib/mesh-mail/tools.json","content":"b"}` + mailConfig = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"a"}` + mailConfig2 = `{"id":"mesh-mail.conf","type":"file","module":"mesh-mail","path":"/var/lib/mesh-mail/spool/conf","content":"b"}` +) + +// 1945/AC12: captured pairs of compositions are judged by ADR 0377 §3's tests: a changed container recreates; with no +// container changed, a directory moved, a file within the data a container mounts changed, or the data declaration +// changed moves data; a tools file nothing mounts changed is neither. +func TestAMoveIsJudgedByWhatItRecreatesOrMoves(t *testing.T) { + cases := []struct { + name string + from, to sdkdelivery.Composed + want sdkdelivery.Judgement + }{ + {"a new image", mailOn(t, 'a', true, mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools), sdkdelivery.JudgedRecreates}, + {"a container gone", mailOn(t, 'a', true, mailContainer, mailSpool), mailOn(t, 'b', true, mailSpool), sdkdelivery.JudgedRecreates}, + {"its spool moved", mailOn(t, 'a', true, mailContainer, mailSpool), mailOn(t, 'b', true, mailContainer, mailSpoolMoved), sdkdelivery.JudgedMovesData}, + {"a file in its spool", mailOn(t, 'a', true, mailContainer, mailSpool, mailConfig), mailOn(t, 'b', true, mailContainer, mailSpool, mailConfig2), sdkdelivery.JudgedMovesData}, + {"its data declaration", mailOn(t, 'a', false, mailContainer, mailSpool), mailOn(t, 'b', true, mailContainer, mailSpool), sdkdelivery.JudgedMovesData}, + {"its tools only", mailOn(t, 'a', true, mailContainer, mailSpool, mailTools), mailOn(t, 'b', true, mailContainer, mailSpool, mailTools2), sdkdelivery.JudgedNeither}, + } + for _, c := range cases { + if got, words := judged(c.from, c.to); got != c.want || words == "" { + t.Errorf("%s: judged %s (%q), want %s", c.name, got, words, c.want) + } + } +} + +// 1945/AC12: a move is judged against the composition the node's have names: one the node never ran is neither; one +// whose composition is gone cannot be judged and says why; and with A running, B held and C composed, C is judged +// against A, and judged again when the have changes. +func TestAMoveIsJudgedAgainstWhatTheNodeRuns(t *testing.T) { + at, _, _, haves := aPlace(t, "ace") + ctx := context.Background() + a := mailOn(t, 'a', true, mailContainer, mailSpool, mailTools) + b := mailOn(t, 'b', true, mailContainer2, mailSpool, mailTools) + c := mailOn(t, 'c', true, mailContainer, mailSpool, mailTools2) + write := func(comp sdkdelivery.Composed) (string, string) { + key, _ := sdkdelivery.ComposedKey("ace", comp.Module, comp.FP) + if _, err := writeComposed(ctx, at, key, comp); err != nil { + t.Fatal(err) + } + d, _ := sdkdelivery.ComposedDigestOf(comp) + return key, d + } + judgedNow := func(key string) sdkdelivery.ComposedDigest { + e, err := at.digests.Get(ctx, key) + if err != nil { + t.Fatal(err) + } + v, err := sdkdelivery.DecodeComposedDigest(e.Value()) + if err != nil { + t.Fatal(err) + } + return v + } + keyA, digestA := write(a) + if v := judgedNow(keyA); v.Judgement != sdkdelivery.JudgedNeither || v.Against != "" { + t.Errorf("a module the node never ran: %+v", v) + } + if v := judgedNow(keyA); v.Policy != "roll" || v.PolicyFrom != "default" { + t.Errorf("the policy beside the judgement: %+v", v) + } + haves["mesh-mail"] = sdkdelivery.Have{FP: a.FP, Composed: digestA, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK} + keyB, _ := write(b) + if v := judgedNow(keyB); v.Judgement != sdkdelivery.JudgedRecreates || v.Against != digestA { + t.Errorf("B against A: %+v", v) + } + // B is held (the assigner writes no want for it); the node still runs A. + keyC, digestC := write(c) + if v := judgedNow(keyC); v.Judgement != sdkdelivery.JudgedNeither || v.Against != digestA { + t.Errorf("C against A, B held: %+v", v) + } + // The have changes to C: every judgement of the module is written again, against what runs now (the have watch). + haves["mesh-mail"] = sdkdelivery.Have{FP: c.FP, Composed: digestC, Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK} + if n, err := rejudge(ctx, at, "mesh-mail"); err != nil || n == 0 { + t.Fatalf("rejudged %d (%v)", n, err) + } + if v := judgedNow(keyC); v.Against != digestC || v.Judgement != sdkdelivery.JudgedNeither { + t.Errorf("C once the node runs it: %+v", v) + } + if v := judgedNow(keyB); v.Against != digestC || v.Judgement != sdkdelivery.JudgedRecreates { + t.Errorf("B once the node runs C: %+v", v) + } + // A person sets the module's policy: written beside every composition of it. + at.policy = func(string) (string, string, string) { return "record", "person", "its spool" } + if _, err := rejudge(ctx, at, "mesh-mail"); err != nil { + t.Fatal(err) + } + if v := judgedNow(keyB); v.Policy != "record" || v.PolicyFrom != "person" || v.PolicyWhy != "its spool" { + t.Errorf("the person's policy beside B: %+v", v) + } + // A have naming a composition the buckets no longer hold. + haves["mesh-mail"] = sdkdelivery.Have{FP: sdkdelivery.SourcePrefix2 + strings.Repeat("9", 64), Composed: "sha256:" + strings.Repeat("9", 64), + Phase: sdkdelivery.PhaseRunning, Health: sdkdelivery.HealthOK} + write(b) + if v := judgedNow(keyB); v.Judgement != sdkdelivery.JudgedCannot || !strings.Contains(v.Words, "gone") { + t.Errorf("against a composition that is gone: %+v", v) + } +} diff --git a/cmd/mesh-controller/composed_operator_keys_test.go b/cmd/mesh-controller/composed_operator_keys_test.go new file mode 100644 index 00000000..f06a94ab --- /dev/null +++ b/cmd/mesh-controller/composed_operator_keys_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1945/AC13: the node-engine's composition on a node carries the operator's public keys as its file operator-keys, +// from its setting operator-signers, beside the controller's key it already trusts (ADR 0539 clause 3, decided with the +// lead 2026-10-24: the switch's last declaration carries it because today's controller composes the same file from +// the same manifest). The manifest is mesh-host's module.json at 20d0998c (testdata/mesh-host-module.json), never +// typed by hand. +func TestTheNodeEnginesCompositionCarriesTheOperatorsKeys(t *testing.T) { + raw, err := os.ReadFile("testdata/mesh-host-module.json") + if err != nil { + t.Fatal(err) + } + var m catalogue.Manifest + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + // Its archive names a build artifact, which only its registration turns into a reference: left out here, the rest of + // the manifest as captured. + var kept []map[string]any + for _, r := range m.Resources { + if r["type"] != "archive" { + kept = append(kept, r) + } + } + m.Resources = kept + const key = "MCowBQYDK2VwAyEAb3BlcmF0b3Itc2lnbmluZy1rZXktZm9yLXRlc3RzLW9ubHk=" + in := compose.Input{Catalogue: map[string]catalogue.Manifest{m.Module: m}, + Nodes: []compose.Node{{Name: "ace", Assigned: []string{m.Module}, Stated: true, + Settings: map[string][]catalogue.Layer{m.Module: {{From: "mesh", Values: map[string]any{"operator-signers": key}}}}}}} + fp := sdkdelivery.SourcePrefix2 + strings.Repeat("6", 64) + all, _, err := compose.Compositions(in, "ace", map[string]string{m.Module: fp}, compose.Reading) + if err != nil { + t.Fatal(err) + } + k, _ := sdkdelivery.ComposedKey("ace", m.Module, fp) + var file struct{ Path, Content string } + for _, r := range all[k].Resources { + var head struct{ ID, Path, Content string } + _ = json.Unmarshal(r, &head) + if strings.HasSuffix(head.ID, "operator-keys") { + file.Path, file.Content = head.Path, head.Content + } + } + if file.Path != "/var/lib/mesh-host/operator-keys" || file.Content != key+"\n" { + t.Errorf("the node-engine's composition carries operator-keys at %q with %q, want the operator's key", file.Path, file.Content) + } +} diff --git a/cmd/mesh-controller/composed_rejudge.go b/cmd/mesh-controller/composed_rejudge.go new file mode 100644 index 00000000..e550720f --- /dev/null +++ b/cmd/mesh-controller/composed_rejudge.go @@ -0,0 +1,163 @@ +package main + +import ( + "context" + "errors" + "fmt" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" +) + +// The judgement beside each composition is rewritten when what it is judged against or with changes (novox/hq ADR +// 0539 clauses 2 and 14): when the node's have of the module changes (it runs another composition, or holds), and when +// a person sets the module's policy. Heard as the have bucket's writes and the `upgrade` verb, never on a timer. + +// rejudge writes again the judgement and policy beside every composition of module on the node at, where they differ. +func rejudge(ctx context.Context, at composedPlace, module string) (int, error) { + lister, err := at.bucket.ListKeys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return 0, nil + } + if err != nil { + return 0, err + } + wrote := 0 + var errs []error + for key := range lister.Keys() { + node, m, _, isNode, err := sdkdelivery.ParseComposedKey(key) + if err != nil || isNode || node != at.node || m != module { + continue + } + c, revision, err := readComposed(ctx, at.bucket, key) + if err != nil || revision == 0 { + continue + } + c.Signature = nil + digest, err := sdkdelivery.ComposedDigestOf(c) + if err != nil { + continue + } + judged, err := judgeMove(ctx, at, key, c, digest) + if err != nil { + errs = append(errs, err) + continue + } + w, err := keepDigest(ctx, at.digests, key, judged) + if err != nil { + errs = append(errs, err) + } + if w { + wrote++ + } + } + return wrote, errors.Join(errs...) +} + +// rejudgeOn opens the node's buckets and rejudges module there, under the lease. +func rejudgeOn(ctx context.Context, open *stores, nodes []string, module string) error { + if _, err := theLease.epoch(ctx); err != nil { + return err + } + bus, err := aBus() + if err != nil { + return err + } + defer bus.Close() + js, err := jetstream.New(bus.Conn()) + if err != nil { + return err + } + upgrades, err := open.inventory.Upgrades(ctx) + if err != nil { + return err + } + var errs []error + for _, node := range nodes { + at, err := openComposedPlace(ctx, js, bus.Conn(), node) + if err != nil { + errs = append(errs, err) + continue + } + at.policy = func(m string) (string, string, string) { + u, ok := upgrades[m] + if !ok { + return "", "", "" + } + return u.Policy(), u.From, u.Why + } + if _, err := rejudge(ctx, at, module); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// rejudgeModule rejudges a module on every node it is assigned to: what a person's policy changes. +func rejudgeModule(ctx context.Context, open *stores, module string) { + entries, err := open.inventory.Catalogued(ctx) + if err != nil { + fmt.Printf("compositions: %s's policy is not written beside its compositions: %v\n", module, err) + return + } + for _, e := range entries { + if e.Manifest.Module == module && len(e.On) > 0 { + if err := rejudgeOn(ctx, open, e.On, module); err != nil { + fmt.Printf("compositions: %s's policy is not written beside its compositions: %v\n", module, err) + } + } + } +} + +// watchHavesForJudgement rejudges a node's module each time its have changes, until ctx ends: never on the watch's own +// goroutine. +func watchHavesForJudgement(ctx context.Context, open *stores) { + bus, err := aBus() + if err != nil { + fmt.Printf("compositions: the haves are not watched: %v\n", err) + return + } + js, err := jetstream.New(bus.Conn()) + if err != nil { + bus.Close() + return + } + kv, err := js.KeyValue(ctx, sdkdelivery.HaveBucket) + if err != nil { + bus.Close() + fmt.Printf("compositions: the haves are not watched: %v\n", err) + return + } + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + bus.Close() + fmt.Printf("compositions: the haves are not watched: %v\n", err) + return + } + go func() { + defer bus.Close() + defer w.Stop() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + node, module, isNode, err := sdkdelivery.ParseHaveKey(e.Key()) + if err != nil || isNode { + continue + } + inBackground(func() { + if err := rejudgeOn(ctx, open, []string{node}, module); err != nil { + fmt.Printf("compositions: %s on %s is not judged again: %v\n", module, node, err) + } + }) + } + } + }() +} diff --git a/cmd/mesh-controller/new_modules.go b/cmd/mesh-controller/new_modules.go index e00b58d7..0f66ef6a 100644 --- a/cmd/mesh-controller/new_modules.go +++ b/cmd/mesh-controller/new_modules.go @@ -208,6 +208,7 @@ func watchNewModules(ctx context.Context, open *stores) { // composed, the replaced controller composed, and from now on each new fingerprint registered, composed and granted. inBackground(func() { composeAtStart(ctx, open) }) watchDoneKeys(ctx, open) + watchHavesForJudgement(ctx, open) keys, closeKeys, err := openBuilderKeys() if err != nil { fmt.Printf("new modules: the builder's keys are not watched: %v\n", err) diff --git a/cmd/mesh-controller/testdata/mesh-host-module.json b/cmd/mesh-controller/testdata/mesh-host-module.json new file mode 100644 index 00000000..fcb6b77a --- /dev/null +++ b/cmd/mesh-controller/testdata/mesh-host-module.json @@ -0,0 +1,49 @@ +{ + "module": "mesh-host", + "version": "1", + "slug": "host", + "build": { + "artifacts": [ + { + "name": "host-arch", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-host", + "binary": "nox-mesh-host" + } + ] + }, + "consumes": [ + "restic.backed-up" + ], + "config": { + "operator-signers": { + "type": "string", + "description": "The operator's public keys, ed25519 in base64, one a line: the node applies a composition made by hand only when one of these signed it (ADR 0539 clause 3). Public keys, no secret. Empty trusts no operator key.", + "default": "" + } + }, + "resources": [ + { + "id": "launcher", + "type": "file", + "path": "/usr/lib/nox-mesh-host/launch", + "mode": "0755", + "content": "#!/bin/sh\n# Supervise the host: start it, watch it, and decide what to do when it stops.\n#\n# novox/hq ADR 0005. The init is asked for ONE thing — run this at boot — and everything else\n# lives here, in a script that can be tested. Whether to restart, how long to wait, when to give\n# up, when to roll back: all of it is policy, and policy in a unit file can only be read and\n# hoped for.\n#\n# **It is the witness of the host's own successor** (novox/hq to-be 45 §8, ADR 0227 rule 8). A\n# delivered host that is not the known-good one runs on trial: it must report its declaration\n# under its own build — which it marks by writing itself into known-good — within a bound. One\n# that crashes repeatedly, exits without standing aside for anything, or does not report in\n# bound is stopped, recorded in `rolled-back` with why, and the known-good one is run. The host\n# says every record on its reports, so the mesh raises it as a condition. One rollback per\n# version: a version in `rolled-back` is never started again; a newer delivery is.\n#\n# It does NOT exec the host. Exec would replace this process, and then only the init could\n# restart anything — which is the arrangement this exists to remove. The cost of staying is\n# signal handling, below.\n#\n# Everything a rollback does is one of: read a file, look at a directory, write a file. It shares\n# no code with the host and calls none of it: a binary that cannot start cannot be its own\n# recovery.\n#\n# POSIX sh. `set -e` is deliberately absent: this script's whole job is to inspect exit codes,\n# and -e would make it exit on the first one it is meant to handle.\nset -u\n\nSTATE_DIR=\"${MESH_HOST_STATE_DIR:-/var/lib/mesh-host}\"\nLIBEXEC=\"${MESH_HOST_LIBEXEC:-/usr/lib/nox-mesh-host}\"\n# The host that was placed by hand, used only when nothing has been delivered. The first host on a\n# machine always arrives this way; every one after it is delivered (novox/hq ADR 0141).\nFALLBACK=\"${MESH_HOST_BIN:-/usr/bin/nox-mesh-host}\"\nVERSIONS=\"$LIBEXEC/versions\"\nBINARY=\"nox-mesh-host\"\nLIMIT=\"${MESH_HOST_START_LIMIT:-3}\"\nBACKOFF=\"${MESH_HOST_BACKOFF:-5}\"\n# How long a host on trial has to report, in seconds: to-be 45 §8's ten minutes from the apply.\nBOUND=\"${MESH_HOST_TRIAL_BOUND:-600}\"\nTICK=\"${MESH_HOST_TRIAL_TICK:-5}\"\nGRACE=\"${MESH_HOST_STOP_GRACE:-20}\"\nONCE=\"${MESH_HOST_RUN_ONCE:-}\" # tests run one iteration; nothing else sets this\n\nATTEMPTS=\"$STATE_DIR/start-attempts\"\nHALTED=\"$STATE_DIR/halted\"\nPINNED=\"$STATE_DIR/rollback-pinned\"\nKNOWN_GOOD=\"$STATE_DIR/known-good\"\n# One line per verdict: from, to, when (seconds since the epoch), outcome, why — tab-separated, read\n# by the host (internal/upgrade) and said on its reports.\nROLLED=\"$STATE_DIR/rolled-back\"\nTRIAL=\"$STATE_DIR/trial\"\nEXPIRED=\"$STATE_DIR/trial-expired\"\n\nsay() { echo \"nox-mesh-host-launch: $*\" >&2; }\n\nnow() { date +%s; }\n\nknown_good() { tr -d '[:space:]' < \"$KNOWN_GOOD\" 2>/dev/null || true; }\n\ndelivered() { [ -n \"$1\" ] && [ -x \"$VERSIONS/$1/$BINARY\" ]; }\n\nrolled_back() { [ -s \"$ROLLED\" ] && cut -f1 \"$ROLLED\" 2>/dev/null | grep -qxF \"$1\"; }\n\n# The version a host path is: the directory it was delivered in, or nothing for the host placed by hand.\nversion_of() {\n\tcase \"$1\" in\n\t\t\"$VERSIONS\"/*/\"$BINARY\") v=\"${1#\"$VERSIONS\"/}\"; echo \"${v%/\"$BINARY\"}\" ;;\n\t\t*) echo \"\" ;;\n\tesac\n}\n\nrecord() { # from to outcome why\n\tprintf '%s\\t%s\\t%s\\t%s\\t%s\\n' \"$1\" \"$2\" \"$(now)\" \"$3\" \"$4\" >> \"$ROLLED\"\n}\n\n# Which host to run: the newest delivered one that was never rolled back — or, while a rollback's pin\n# stands, the version it pinned — or the one placed by hand when nothing has been delivered.\n#\n# **Asked every time round the loop, not once.** Standing aside for a successor is a clean exit, and\n# the next turn has to run what is on disk NOW — resolving this once would restart the same binary\n# for ever and the upgrade would never take.\n#\n# Newest by when it arrived, never by how its name sorts: a version string is whatever the source was\n# described as, and those do not sort — \"1.10\" orders before \"1.9\". Ordering by name would start an\n# older host and call it an upgrade.\n#\n# **A pin stands until something newer arrives.** A version delivered after the pin was written, and\n# never rolled back, is a new build the mesh asks for: the pin goes and it runs, on trial.\npick_host() {\n\tnewest=\n\t# A directory with no executable in it is not a version: an interrupted delivery leaves one, and\n\t# running \"the newest\" would then mean running nothing.\n\tfor candidate in $(ls -1t \"$VERSIONS\" 2>/dev/null || true); do\n\t\tdelivered \"$candidate\" || continue\n\t\trolled_back \"$candidate\" && continue\n\t\tnewest=\"$candidate\"\n\t\tbreak\n\tdone\n\tif [ -s \"$PINNED\" ]; then\n\t\tpinned=\"$(tr -d '[:space:]' < \"$PINNED\" 2>/dev/null || true)\"\n\t\tif [ -n \"$newest\" ] && [ \"$newest\" != \"$pinned\" ] && [ \"$VERSIONS/$newest\" -nt \"$PINNED\" ]; then\n\t\t\tsay \"host $newest arrived after the rollback to $pinned; running it\"\n\t\t\trm -f \"$PINNED\"\n\t\telif delivered \"$pinned\"; then\n\t\t\techo \"$VERSIONS/$pinned/$BINARY\"\n\t\t\treturn 0\n\t\telse\n\t\t\tsay \"the pinned version '$pinned' is not delivered; ignoring the pin\"\n\t\tfi\n\tfi\n\tif [ -n \"$newest\" ]; then\n\t\techo \"$VERSIONS/$newest/$BINARY\"\n\t\treturn 0\n\tfi\n\techo \"$FALLBACK\"\n}\n\n# Go back from `from` to the known-good version, once, and say so. False when there is nowhere to go.\nroll_back() { # from why\n\tkg=\"$(known_good)\"\n\tif [ -z \"$1\" ] || [ \"$1\" = \"$kg\" ] || ! delivered \"$kg\"; then\n\t\treturn 1\n\tfi\n\trecord \"$1\" \"$kg\" rolled-back \"$2\"\n\t# The pin is what stops the launcher starting a version newer than known-good that is not the one\n\t# rolled back; the record is what stops it starting this one again.\n\tprintf '%s\\n' \"$kg\" > \"$PINNED\"\n\trm -f \"$TRIAL\"\n\tsay \"rolled back from host $1 to $kg: $2\"\n\treturn 0\n}\n\n# Watch a host on trial: done when it writes itself into known-good, stopped when the bound passes.\n# A subshell beside the host, so the launcher's own wait is untouched.\ntrial_watch() { # pid version deadline\n\twhile kill -0 \"$1\" 2>/dev/null; do\n\t\t[ \"$(known_good)\" = \"$2\" ] && return 0\n\t\tif [ \"$(now)\" -ge \"$3\" ]; then\n\t\t\tprintf '%s\\n' \"$2\" > \"$EXPIRED\"\n\t\t\tsay \"host $2 did not report within ${BOUND}s of starting; stopping it\"\n\t\t\tkill -TERM \"$1\" 2>/dev/null\n\t\t\twaited=0\n\t\t\twhile kill -0 \"$1\" 2>/dev/null && [ \"$waited\" -lt \"$GRACE\" ]; do\n\t\t\t\tsleep 1\n\t\t\t\twaited=$((waited + 1))\n\t\t\tdone\n\t\t\tkill -KILL \"$1\" 2>/dev/null\n\t\t\treturn 0\n\t\tfi\n\t\tsleep \"$TICK\"\n\tdone\n}\n\nchild=\nwatcher=\nstopping=\n\n# The machine is shutting down. Pass it on and wait for the host to finish — a supervisor that\n# exits while its child is still running leaves the host to be killed rather than to stop, and\n# an apply interrupted that way is exactly the half-configured machine this project is about.\non_term() {\n\tstopping=yes\n\tif [ -n \"$child\" ]; then\n\t\tsay \"stopping: passing the signal to the host\"\n\t\tkill -TERM \"$child\" 2>/dev/null\n\tfi\n}\ntrap on_term TERM INT\n\nmkdir -p \"$STATE_DIR\"\n# What this launcher is, so a delivered successor of it is run at the next clean exit rather than at\n# the next boot.\nSELF=\"$(cksum < \"$0\" 2>/dev/null || true)\"\n\nwhile :; do\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -e \"$HALTED\" ]; then\n\t\tsay \"halted: $(cat \"$HALTED\" 2>/dev/null || echo 'reason not recorded')\"\n\t\tsay \"not starting the host. this node needs a person.\"\n\t\texit 0\n\tfi\n\n\t# Consecutive failed starts, not starts. Cleared by the host itself when it reports, which is\n\t# the only evidence either this or known-good has.\n\t#\n\t# Read the FIRST FIELD, then insist it is a plain integer.\n\t#\n\t# Stripping whitespace instead concatenates, and that is not hypothetical: a counter\n\t# holding \"1 2\" became \"12\", past the limit, so a healthy node rolled itself back. An\n\t# unreadable counter must fail towards \"start normally\", never towards \"give up\".\n\tcount=0\n\tif [ -s \"$ATTEMPTS\" ]; then\n\t\tread -r count _ < \"$ATTEMPTS\" 2>/dev/null || count=0\n\tfi\n\tcase \"${count:-}\" in\n\t\t'' | *[!0-9]*) count=0 ;;\n\tesac\n\n\tHOST=\"$(pick_host)\"\n\tversion=\"$(version_of \"$HOST\")\"\n\n\tif [ \"$count\" -ge \"$LIMIT\" ]; then\n\t\tif roll_back \"$version\" \"it failed $count times in a row\"; then\n\t\t\t# Fresh count for the version now run: it deserves its own attempts, and without this\n\t\t\t# it inherits a count already over the limit and halts at once. The variable too, not\n\t\t\t# only the file: resetting one and not the other made the next failure count from the\n\t\t\t# OLD value — so the rolled-back version got one attempt instead of three.\n\t\t\tcount=0\n\t\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\t\tcontinue\n\t\tfi\n\t\tkg=\"$(known_good)\"\n\t\tif [ -n \"$kg\" ] && { [ \"$version\" = \"$kg\" ] || [ -z \"$version\" ]; } && [ -s \"$ROLLED\" ]; then\n\t\t\t# Already gone back, and what it went back to fails as well: this is the machine and\n\t\t\t# not a binary. Rolling back again would flap between two versions for ever.\n\t\t\tsay \"the host failed $count times after a rollback. the version it went back to does\"\n\t\t\tsay \"not start either, so this is the machine and not the binary.\"\n\t\t\trecord \"${version:-placed-by-hand}\" \"\" halted \"the version rolled back to failed $count times as well\"\n\t\t\tprintf 'rolled back and still failing\\n' > \"$HALTED\"\n\t\t\texit 0\n\t\tfi\n\t\t# Nothing to go back to: no host has ever reported here, or the one that did was placed by\n\t\t# hand and is not delivered. An installation failure rather than an upgrade's — said, and\n\t\t# tried again slowly, never guessed at.\n\t\tsay \"the host failed $count times and there is no delivered known-good version to go back to.\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\tfi\n\n\tif [ ! -x \"$HOST\" ]; then\n\t\tsay \"no host to run: nothing delivered under $VERSIONS and $FALLBACK is not executable.\"\n\t\tprintf 'no host binary\\n' > \"$HALTED\"\n\t\texit 0\n\tfi\n\n\t# **On trial**: a delivered version that is not the known-good one, while a known-good one is\n\t# delivered to go back to. The trial starts when this version was first started, and survives\n\t# this launcher restarting.\n\ttrial=\n\tdeadline=\n\tkg=\"$(known_good)\"\n\tif [ -n \"$version\" ] && [ \"$version\" != \"$kg\" ] && delivered \"$kg\"; then\n\t\ttrial=yes\n\t\tstarted=\n\t\tif [ -s \"$TRIAL\" ]; then\n\t\t\tread -r on since _ < \"$TRIAL\" 2>/dev/null || on=\n\t\t\t[ \"${on:-}\" = \"$version\" ] && started=\"${since:-}\"\n\t\tfi\n\t\tcase \"$started\" in\n\t\t\t'' | *[!0-9]*) started=\"$(now)\"; printf '%s %s\\n' \"$version\" \"$started\" > \"$TRIAL\" ;;\n\t\tesac\n\t\tdeadline=$((started + BOUND))\n\t\tif [ \"$(now)\" -ge \"$deadline\" ]; then\n\t\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\" && continue\n\t\tfi\n\telse\n\t\trm -f \"$TRIAL\"\n\tfi\n\n\trm -f \"$EXPIRED\"\n\tsay \"running $HOST${trial:+ (on trial until it reports)}\"\n\t\"$HOST\" run &\n\tchild=$!\n\twatcher=\n\tif [ -n \"$trial\" ]; then\n\t\ttrial_watch \"$child\" \"$version\" \"$deadline\" &\n\t\twatcher=$!\n\tfi\n\tstatus=0\n\twait \"$child\" || status=$?\n\tif [ -n \"$stopping\" ]; then\n\t\t# The signal interrupted the wait, not the host: it was told, and is finishing what it was\n\t\t# doing. Waited for, so the service manager does not kill it half way through an apply.\n\t\twait \"$child\" 2>/dev/null\n\tfi\n\tchild=\n\tif [ -n \"$watcher\" ]; then\n\t\tkill \"$watcher\" 2>/dev/null\n\t\twait \"$watcher\" 2>/dev/null\n\t\twatcher=\n\tfi\n\n\tif [ -n \"$stopping\" ]; then\n\t\texit 0\n\tfi\n\n\tif [ -n \"$trial\" ] && [ -s \"$EXPIRED\" ] && [ \"$(cat \"$EXPIRED\" 2>/dev/null)\" = \"$version\" ]; then\n\t\trm -f \"$EXPIRED\"\n\t\troll_back \"$version\" \"it did not report within ${BOUND}s of starting\"\n\t\tcount=0\n\t\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\t\tcontinue\n\tfi\n\n\t# A signal the host did not survive, and we are not shutting down: treat it as a crash.\n\tcase \"$status\" in\n\t\t0)\n\t\t\t# Exited cleanly. That is how the host stands aside for a new binary after an\n\t\t\t# upgrade (novox/hq ADR 0005) — so loop and run whatever is now on disk.\n\t\t\t#\n\t\t\t# Deliberately NOT counted, and this is the whole reason the counter is\n\t\t\t# incremented here rather than before the start: counting attempts meant a host\n\t\t\t# that upgraded itself three times rolled itself back, having worked perfectly\n\t\t\t# every time.\n\t\t\t#\n\t\t\t# **Unless it stood aside for nothing.** A host on trial that exits cleanly while the\n\t\t\t# same host is still the one to run has not stood aside for a successor: it has\n\t\t\t# stopped, and a host that stops at once would otherwise loop here for ever unseen.\n\t\t\tif [ -n \"$trial\" ] && [ \"$(pick_host)\" = \"$HOST\" ] && [ \"$(known_good)\" != \"$version\" ]; then\n\t\t\t\tsay \"host $version exited cleanly on trial with nothing newer to stand aside for\"\n\t\t\telse\n\t\t\t\t# A delivered successor of this launcher runs from here on, not from the next boot.\n\t\t\t\tif [ -n \"$SELF\" ] && [ \"$(cksum < \"$0\" 2>/dev/null || true)\" != \"$SELF\" ]; then\n\t\t\t\t\tsay \"the launcher was replaced; running the new one\"\n\t\t\t\t\texec \"$0\"\n\t\t\t\tfi\n\t\t\t\tsay \"the host exited cleanly; starting it again\"\n\t\t\t\tcontinue\n\t\t\tfi\n\t\t\t;;\n\tesac\n\n\tcount=$((count + 1))\n\tprintf '%s\\n' \"$count\" > \"$ATTEMPTS\"\n\n\tsay \"the host exited $status ($count consecutive); restarting in ${BACKOFF}s\"\n\t[ -n \"$ONCE\" ] && exit \"$status\"\n\tsleep \"$BACKOFF\"\ndone\n" + }, + { + "id": "next", + "type": "archive", + "artifact": "host-arch", + "path": "/usr/lib/nox-mesh-host/versions/${version}" + }, + { + "id": "operator-keys", + "type": "file", + "path": "/var/lib/mesh-host/operator-keys", + "mode": "0644", + "content": "${config:operator-signers}\n" + } + ] +} diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index b571cc93..5256f697 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -13,6 +13,7 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -42,6 +43,30 @@ func (f following) Upgraded(ctx context.Context, u link.Upgraded) error { return nil } +// askAgainOnGrants marks a send that stopped at its grants as one to ask again (novox/hq issue 249): +// an announcement handled by a send whose memberships could not be issued is held and redelivered, +// rather than taken as handled with the machines left on the old version. +func askAgainOnGrants(err error) error { + if err != nil && errors.Is(err, errGrants) && !errors.Is(err, link.ErrTryAgain) { + return fmt.Errorf("%w: %w", link.ErrTryAgain, err) + } + return err +} + +// readableList names machines the way a sentence does, because this is read by a person deciding +// whether an upgrade went where they expected. +func readableList(names []string) string { + switch len(names) { + case 0: + return "nothing" + case 1: + return names[0] + case 2: + return names[0] + " and " + names[1] + } + return strings.Join(names[:len(names)-1], ", ") + " and " + names[len(names)-1] +} + func shortCommit(commit string) string { if len(commit) > 8 { return commit[:8] @@ -49,6 +74,13 @@ func shortCommit(commit string) string { return commit } +func isAre(n int) string { + if n == 1 { + return "is" + } + return "are" +} + // Announceable is every build this mesh recorded, in the shape the builder announces one. // // **The catalogue asks for this when it starts, and the answer is the graph's foundation** @@ -93,6 +125,25 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error return out, nil } +// notNow marks a store that could not be read right now, or plans another controller is working (issue 921), so +// the announcement is held rather than lost; anything else is returned as it was. +func notNow(err error) error { + if inventory.Unreachable(err) { + return fmt.Errorf("%w: %w", link.ErrTryAgain, err) + } + return err +} + +// unsaved marks the failure to save the plan a merge was added to, whatever it was: the merge is carried until that save +// writes it (inventory.AddMerge), so the announcement is held and handed over again, never taken (novox/hq ADR 0407 +// decision 3, ADR 0408, issue 1092). The bus acknowledges it only after the save succeeded. +func unsaved(err error) error { + if err == nil { + return nil + } + return fmt.Errorf("%w: the merge is not kept until its plan is saved: %w", link.ErrTryAgain, err) +} + // SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which // becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go. func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { @@ -102,6 +153,82 @@ func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { return keepForgeMerge(ctx, m) } +// movesOfMerge is what one merge moves, acted on: every module recorded as built from that repository and +// branch is marked as moved to the merge commit; a module the merge deleted is forgotten or said; a new module +// is asked for and sent nowhere. The names returned are what the walk builds, bases first along the graph +// (novox/hq 04-ISSUES/131). Nothing is built or pushed here: the walk asks its tiers. Called when a batch is +// cut, once per repository, with the latest merge of the repository's branch and every file the batch's +// merges of it changed. +func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry, + read map[string][]inventory.ReadRepository) ([]string, error) { + from, packaging, already := mergeCandidates(m, entries, read) + if len(from) == 0 && len(packaging) == 0 { + // "Already built from it" and "nothing reads it" are different facts, and reading the first + // as the second sends somebody looking for a broken trigger when the mesh is up to date. + if already > 0 { + fmt.Printf("%s/%s merged into %s (%.8s); %d module(s) the mesh holds are already built "+ + "from it\n", m.Owner, m.Repo, m.Base, m.Commit, already) + return nil, nil + } + fmt.Print(nothingReadsIt(m.Owner+"/"+m.Repo, m)) + return nil, nil + } + // Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows + // another branch is not part of this merge, and whoever is waiting for its change should read why. + for _, e := range entries { + if sameRepository(e.Source.Repository, m) && !sourceIs(e.Source, m) { + fmt.Printf(" %s is built from %s/%s and follows %s, not %s; this merge leaves it out\n", + e.Manifest.Module, m.Owner, m.Repo, e.Source.Ref, m.Base) + } + } + touched, added, _ := touchedBy(from, entries, m, read) + // **A module the merge deleted is not built** (novox/hq ADR 0236): its manifest is gone, so the build + // seat finds nothing saying what it is, and the plan failed on it (`has no module.json at …`) with + // every other module of its tier left unsent. It is forgotten where nothing holds it, said otherwise. + touched, deleted := splitDeleted(touched, m) + for _, e := range deleted { + retireDeleted(ctx, inv, e, m) + } + for _, e := range touched { + if err := inv.SourceMoved(ctx, e.Manifest.Module, m.Commit); err != nil { + return nil, notNow(err) + } + } + // **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan + // says so, and assigning it is a person's act, which needs it registered first. + // Nothing asks its first build (novox/hq ADR 0501 §6): the builder builds every fingerprint the indexer writes, and + // the controller registers the module from its first done key (new_modules.go). + var built []string + if len(from) > 0 { + built = added + } + moved := append(append([]inventory.Entry{}, touched...), packaging...) + if len(moved) == 0 { + if len(built) > 0 { + fmt.Printf("%s/%s merged into %s (%.8s); it changed no module the mesh holds, and adds %s: "+ + "registered from the builder's done key of each, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit, + strings.Join(built, ", ")) + return nil, nil + } + fmt.Printf("%s/%s merged into %s (%.8s); it changed nothing any module the mesh holds is "+ + "built from\n", m.Owner, m.Repo, m.Base, m.Commit) + return nil, nil + } + // Why each is in it (issue 363): the files of its build source the merge changed, or why it is read whole. + for _, e := range moved { + fmt.Printf(" %s: %s\n", e.Manifest.Module, whyMoved(e, read[e.Manifest.Module], m)) + } + for _, e := range packaging { + fmt.Printf(" %s reads %s/%s through its build context: its own source record is left where it is\n", + e.Manifest.Module, m.Owner, m.Repo) + } + var names []string + for _, e := range moved { + names = append(names, e.Manifest.Module) + } + return names, nil +} + // actingOnMerges keeps one merge acted on at a time (novox/hq issue 266). var actingOnMerges sync.Mutex @@ -165,6 +292,63 @@ func lookedAtCommit(read []inventory.ReadRepository, commit string) bool { // historyMargin is how far a packaging module's last look is taken back before a merge is history for it. const historyMargin = time.Minute +// whyMoved is why a merge moves a module, as a plan says it (novox/hq ADR 0267, issue 363): the changed +// files in its build source, or why it is read whole. For a module built from the merged repository or one +// whose build context is that repository; a dependent is in a plan for what it stands on. +func whyMoved(e inventory.Entry, read []inventory.ReadRepository, m link.SourceMoved) string { + if len(m.Paths) == 0 || m.PathsTruncated { + return "read whole: the merge's changed files were not all said" + } + whole := "no build source recorded" + for _, r := range read { + if r.Own && r.Whole != "" { + whole = r.Whole + } + } + held := func(paths []string) []string { + var in []string + for _, p := range m.Paths { + if gosource.SourceHolds(paths, p) { + in = append(in, p) + } + } + return in + } + changed := func(where string, in []string) string { + return fmt.Sprintf("its build source%s changed: %d changed file(s) in it, e.g. %s", where, len(in), in[0]) + } + if sameRepository(e.Source.Repository, m) { + if own := ownSource(read); own != nil { + if in := held(own); len(in) > 0 { + return changed("", in) + } + } + dir := strings.Trim(e.Source.Path, "/") + if dir == "" { + return "read whole: " + whole + ", so every file of its repository is its build source" + } + for _, p := range m.Paths { + if inside(p, dir) { + return "read whole: " + whole + ", so its directory is its build source; e.g. " + p + } + } + return "read whole: " + whole + } + for _, r := range read { + if r.Own || !sameRepository(r.Repository, m) || !followsBase(r.Ref, m.Base) { + continue + } + if len(r.Paths) > 0 { + if in := held(r.Paths); len(in) > 0 { + return changed(" in "+m.Owner+"/"+m.Repo, in) + } + continue + } + return "read whole: " + whole + ", so every file of " + m.Owner + "/" + m.Repo + ", which its build context is, is its build source" + } + return "read whole: " + whole +} + // lookedOf is when a module packaging another repository was last looked at, as readForPlanning says. func lookedOf(read []inventory.ReadRepository) time.Time { var at time.Time @@ -176,6 +360,21 @@ func lookedOf(read []inventory.ReadRepository) time.Time { return at } +// wouldMove is the modules acting on this merge would move and rebuild — SourceMoved's judgement, made +// without acting (novox/hq issue 266). Empty for a merge already acted on: acting marks each module built +// from the repository as looked at, so the merge then reads as history for it. +// +// **The ones packaging source from it too** (novox/hq ADR 0267): with a module moved only by the files of +// its build source, a merge can move a packaging module and nothing built from the repository, and a missed +// one of those was never acted on. A packaging module's look is its newest build or plan (lookedAt), so a +// merge acted on for it reads as history once its plan is made. +func wouldMove(m link.SourceMoved, entries []inventory.Entry, + read map[string][]inventory.ReadRepository) []inventory.Entry { + from, packaging, _ := mergeCandidates(m, entries, read) + touched, _ := splitDeleted(whatTheMergeTouched(from, entries, m, read), m) + return append(touched, packaging...) +} + // splitDeleted parts the modules a merge touched into those it changed and those whose manifest it // removed — deleted at their source (ADR 0236). func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted []inventory.Entry) { @@ -198,6 +397,39 @@ func splitDeleted(touched []inventory.Entry, m link.SourceMoved) (kept, deleted return kept, deleted } +// retireDeleted is what the mesh does with a module deleted at its source: its record says the merge +// was looked at, so it is not acted on again; it is forgotten where nothing holds it; where a machine +// runs it or the mesh holds something for it, that is said, and nothing is built. +func retireDeleted(ctx context.Context, inv *inventory.Inventory, e inventory.Entry, m link.SourceMoved) { + name := e.Manifest.Module + if err := inv.SourceMoved(ctx, name, m.Commit); err != nil { + fmt.Printf(" %s was deleted from %s/%s at %.8s, and that could not be recorded: %v\n", name, m.Owner, m.Repo, m.Commit, err) + } + err := inv.ForgetModule(ctx, name) + switch { + case err == nil: + fmt.Printf(" %s was deleted from %s/%s at %.8s: forgotten, nothing built\n", name, m.Owner, m.Repo, m.Commit) + case errors.Is(err, inventory.ErrStillAssigned) || errors.Is(err, inventory.ErrStillHolds): + fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; the mesh still runs or holds it, so it is "+ + "kept until a person unassigns it and `module forget %s`: %v\n", name, m.Owner, m.Repo, m.Commit, name, + firstLine(err.Error())) + default: + fmt.Printf(" %s was deleted from %s/%s at %.8s and is not built; forgetting it failed: %v\n", name, m.Owner, + m.Repo, m.Commit, err) + } +} + +// nothingReadsIt is what is said of a merge no module the mesh holds reads. **A merge into a branch other than the +// trunk is said to start no delivery** (novox/hq ADR 0401 clause 3, issue 889): every delivery is cut from a +// repository's trunk, and whoever merged into a feature branch should not look for a broken trigger. +func nothingReadsIt(repository string, m link.SourceMoved) string { + if m.Base != "" && m.Base != "main" { + return fmt.Sprintf("%s merged into %s (%.8s), not the trunk of %s; a merge into any other branch starts no "+ + "delivery\n", repository, m.Base, m.Commit, repository) + } + return fmt.Sprintf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit) +} + // sourceIs is whether a recorded source is the repository and branch a merge announced. A source on // the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from. // An empty recorded ref is the repository's default branch, which is what a merge into the base @@ -299,12 +531,79 @@ func readsDirectory(e inventory.Entry, p string) bool { return strings.Trim(e.Source.Path, "/") == "" || inside(p, e.Source.Path) } +// unchangedOfMerge are the modules a merge changed files for under a reading without recorded build sources, but +// whose recorded build source holds none of them, each with why (novox/hq issue 611): not built, because the build +// the mesh holds is theirs (ADR 0267 §5). That reading — a delivery plan made while another walk built the module +// (issue 606) — would have moved them; the walk says they matched what runs rather than leaving them unnamed. +// A module built from the repository whose directory, or whole repository when built from its root, holds a +// changed file, and a module packaging the repository's branch, are each read so. A merge whose changed files +// were not all said moves every one of them: none is said here. Without acting. +func unchangedOfMerge(m link.SourceMoved, entries []inventory.Entry, + read map[string][]inventory.ReadRepository) map[string]string { + if len(m.Paths) == 0 || m.PathsTruncated { + return nil + } + from, _, _ := mergeCandidates(m, entries, read) + touched, _, _ := touchedBy(from, entries, m, read) + out := map[string]string{} + for _, e := range from { + name := e.Manifest.Module + if ownSource(read[name]) == nil || slices.ContainsFunc(touched, func(t inventory.Entry) bool { + return t.Manifest.Module == name + }) { + continue + } + var in []string + for _, p := range m.Paths { + if readsDirectory(e, p) { + in = append(in, p) + } + } + if len(in) > 0 { + out[name] = fmt.Sprintf("its build source holds none of the %d changed file(s) in its directory (e.g. %s): "+ + "the build the mesh holds is its build", len(in), in[0]) + } + } + for _, e := range entries { + name := e.Manifest.Module + if sourceIs(e.Source, m) || !packagesSaid(read[name], m) || readsFrom(read[name], m) { + continue + } + out[name] = fmt.Sprintf("its build source in %s/%s holds none of the %d changed file(s) (e.g. %s): "+ + "the build the mesh holds is its build", m.Owner, m.Repo, len(m.Paths), m.Paths[0]) + } + if len(out) == 0 { + return nil + } + return out +} + +// packagesSaid is whether a module's newest trunk build said the build source it read in a merge's repository and +// branch, as a context beside its own. +func packagesSaid(read []inventory.ReadRepository, m link.SourceMoved) bool { + for _, r := range read { + if !r.Own && sameRepository(r.Repository, m) && followsBase(r.Ref, m.Base) && len(r.Paths) > 0 { + return true + } + } + return false +} + // readForPlanning is what each module's build read, as a change is mapped onto it — for a merge's reach, a pull request's // check and the facts alike (novox/hq ADR 0238), so they cannot disagree. No plan overtakes it any more (ADR 0539 §17). func readForPlanning(ctx context.Context, inv *inventory.Inventory) (map[string][]inventory.ReadRepository, error) { return inv.ReadRepositories(ctx) } +// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: **a +// changed file touches exactly the modules whose build reads it** (novox/hq issue 280, ADR 0238). It is +// touchedBy's first answer; touchedBy is the one place the mesh maps a changed file onto its modules. +func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved, + read map[string][]inventory.ReadRepository) []inventory.Entry { + touched, _, _ := touchedBy(candidates, known, m, read) + return touched +} + // touchedBy maps a merge's changed files onto the mesh's modules — **the one place it is done**, for the // merge handler, the release planner's what-if, the merge gate and a pull request's check alike (novox/hq // ADR 0238), so planning and gating cannot disagree about what a change touches. @@ -458,6 +757,49 @@ func inside(path, dir string) bool { return path == dir || strings.HasPrefix(path, dir+"/") } +// anyInside is whether any of these changed files is in a directory. +func anyInside(paths []string, dir string) bool { + for _, p := range paths { + if inside(p, dir) { + return true + } + } + return false +} + +// standsOnModule is whether an entry's build stood on the named module: by what its newest build +// recorded it was handed (`artifact-store:///@…`, the module's own artifact), or +// — for a module registered from a manifest and not yet built — by the base its manifest names. +func standsOnModule(e inventory.Entry, module string, against map[string][]string) bool { + if e.Manifest.Module == module { + return false + } + if e.Manifest.Build != nil { + for _, on := range e.Manifest.Build.On { + if on.Module == module { + return true + } + } + } + prefix := catalogue.ArtifactStoreScheme + module + "/" + for _, ref := range against[e.Manifest.Module] { + if strings.HasPrefix(ref, prefix) { + return true + } + } + return false +} + +// standsOn is whether anything in the set is built on the named module's artifacts. +func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool { + for _, e := range entries { + if standsOnModule(e, module, against) { + return true + } + } + return false +} + // isHistory is whether a merge made at mergedAt predates the last time the source was seen. A merge // with no time on it is taken as news: refusing it would silence a forge that says less. func isHistory(mergedAt string, seen time.Time) bool { @@ -471,10 +813,52 @@ func isHistory(mergedAt string, seen time.Time) bool { return at.Before(seen) } +// dependentsOf is every catalogued module that stands on one of the moved modules, directly or +// through another dependent, and is not itself among them — in the catalogue's order, so the +// answer is the same each time. A module standing on nothing that moved is left alone: a merge +// rebuilds what it changed and what is built on top of that, not the catalogue. +func dependentsOf(moved, entries []inventory.Entry, against map[string][]string) []inventory.Entry { + bases := map[string]bool{} + for _, e := range moved { + bases[e.Manifest.Module] = true + } + var out []inventory.Entry + taken := map[string]bool{} + for grew := true; grew; { + grew = false + for _, e := range entries { + name := e.Manifest.Module + if bases[name] || taken[name] { + continue + } + for base := range bases { + if standsOnModule(e, base, against) { + taken[name] = true + out = append(out, e) + grew = true + break + } + } + } + for _, e := range out { + bases[e.Manifest.Module] = true + } + } + return out +} + // moduleManifestFile is the file that says what a module is, in its directory (the build seat's // ManifestName). const moduleManifestFile = "module.json" +// deletedAtSource is whether a build failed because its module's manifest is not at its source any +// more — the build seat's own words (internal/builder) — which a merge that deleted the module causes +// when its announcer did not say which files went (ADR 0236). Such a module is not a failure of the plan. +func deletedAtSource(failed string) bool { + return strings.Contains(failed, "has no "+moduleManifestFile+" at ") && + strings.Contains(failed, "so there is nothing saying what it is") +} + // reachOfMerge is what a merge of these changed files into this branch would move, without acting: a pull request's // reach, as the check reckons it. func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository, @@ -607,6 +991,8 @@ func upgradeCommand(ctx context.Context, args []string) error { return err } fmt.Println(sayUpgrade(module, decision)) + // Beside each of its compositions, for the assigner's hold (novox/hq ADR 0539 clause 14). + rejudgeModule(ctx, open, module) return nil default: return fmt.Errorf("upgrade roll-out|record|default — not %q", positionals[1]) @@ -619,6 +1005,8 @@ func upgradeCommand(ctx context.Context, args []string) error { return err } fmt.Println(sayUpgrade(module, decision)) + // Beside each of its compositions, for the assigner's hold (novox/hq ADR 0539 clause 14). + rejudgeModule(ctx, open, module) return nil } @@ -642,11 +1030,3 @@ func sayUpgrade(module string, u inventory.Upgrade) string { return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+ "the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from) } - -// whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: touchedBy's first answer -// (novox/hq issue 280, ADR 0238). -func whatTheMergeTouched(candidates, known []inventory.Entry, m link.SourceMoved, - read map[string][]inventory.ReadRepository) []inventory.Entry { - touched, _, _ := touchedBy(candidates, known, m, read) - return touched -}